def checkForDuplicateInputs(rootnames): """ Check input files specified in ASN table for duplicate versions with multiple valid suffixes (_flt and _flc, for example). """ flist = [] duplist = [] for fname in rootnames: # Look for any recognized CTE-corrected products f1 = fileutil.buildRootname(fname,ext=['_flc.fits']) f2 = fileutil.buildRootname(fname) flist.append(f2) if os.path.exists(f1) and f1 != f2: # More than 1 valid input found for this rootname duplist.append(f1) return flist,duplist
def checkForDuplicateInputs(rootnames): """ Check input files specified in ASN table for duplicate versions with multiple valid suffixes (_flt and _flc, for example). """ flist = [] duplist = [] for fname in rootnames: # Look for any recognized CTE-corrected products f1 = fileutil.buildRootname(fname,ext=['_flc.fits']) f2 = fileutil.buildRootname(fname) flist.append(f2) if os.path.exists(f1) and f1 != f2: # More than 1 valid input found for this rootname duplist.append(f1) return flist,duplist
def define_output_name(fname): """Determines the output name based on input filename or drizzle keyword """ phdr = pyfits.getheader(fname, ext=0) if 'D001DATA' in phdr: outname = phdr['D001DATA'] if outname.find('['): outname = outname.split('[')[0] else: frootname = fname.split('_')[0] outname = fileutil.buildRootname(frootname) if outname is None: # make one from the header keywords outname = phdr['rootname'] del phdr return outname
def parse_input(input, prodonly=False, sort_wildcards=True): catlist = None if (isinstance(input, list) == False) and \ ('_asn' in input or '_asc' in input) : # Input is an association table # Get the input files oldasndict = asnutil.readASNTable(input, prodonly=prodonly) filelist = [ fileutil.buildRootname(fname) for fname in oldasndict['order'] ] elif (isinstance(input, list) == False) and \ (input[0] == '@') : # input is an @ file f = open(input[1:]) # Read the first line in order to determine whether # catalog files have been specified in a second column... line = f.readline() f.close() # Parse the @-file with irafglob to extract the input filename filelist = irafglob.irafglob(input, atfile=atfile_sci) print(line) # If there are additional columns for catalog files... if len(line.split()) > 1: # ...parse out the names of the catalog files as well catlist, catdict = parse_atfile_cat(input) elif (isinstance(input, list)): # input a python list filelist = [] for fn in input: flist, output = parse_input(fn, prodonly=prodonly) # if wild-cards are given, sort for uniform usage: if fn.find('*') > -1 and sort_wildcards: flist.sort() filelist += flist else: # input is either a string or something unrecognizable, so give it a try: try: filelist, output = parseinput.parseinput(input) # if wild-cards are given, sort for uniform usage: if input.find('*') > -1 and sort_wildcards: filelist.sort() except IOError: raise return filelist, catlist
def parse_input(input, prodonly=False, sort_wildcards=True): catlist = None if (isinstance(input, list) == False) and \ ('_asn' in input or '_asc' in input) : # Input is an association table # Get the input files oldasndict = asnutil.readASNTable(input, prodonly=prodonly) filelist = [fileutil.buildRootname(fname) for fname in oldasndict['order']] elif (isinstance(input, list) == False) and \ (input[0] == '@') : # input is an @ file f = open(input[1:]) # Read the first line in order to determine whether # catalog files have been specified in a second column... line = f.readline() f.close() # Parse the @-file with irafglob to extract the input filename filelist = irafglob.irafglob(input, atfile=atfile_sci) print(line) # If there are additional columns for catalog files... if len(line.split()) > 1: # ...parse out the names of the catalog files as well catlist,catdict = parse_atfile_cat(input) elif (isinstance(input, list)): # input a python list filelist = [] for fn in input: flist, output = parse_input(fn, prodonly=prodonly) # if wild-cards are given, sort for uniform usage: if fn.find('*') > -1 and sort_wildcards: flist.sort() filelist += flist else: # input is either a string or something unrecognizable, so give it a try: try: filelist, output = parseinput.parseinput(input) # if wild-cards are given, sort for uniform usage: if input.find('*') > -1 and sort_wildcards: filelist.sort() except IOError: raise return filelist,catlist
def wf2_pydr(filename): trl_start = "PYDRIZZLE STARTED------------ %s ------------\n" % time.asctime( ) filename = fileutil.buildRootname(filename, ext=['_c0m.fits', '_c0f.fits']) try: hdr0 = pyfits.getheader(filename) except: errormsg = "\n\nCan't find file %s \n", filename raise IOError(errormsg) try: dodrizcorr = hdr0['DRIZCORR'] except KeyError: print("Keyword DRIZCORR not found\n") dodrizcorr = None if isSupportedFilter(hdr0) and isSupportedOfftab(hdr0): useIDCModel = True else: useIDCModel = False if dodrizcorr != None: if dodrizcorr == 'PERFORM': try: # update WCS information if useIDCModel: p = pydrizzle.PyDrizzle(filename, bits_single=0, bits_final=0) else: p = pydrizzle.PyDrizzle(filename, bits_single=0, bits_final=0, updatewcs=False, idckey=None) # resample to the scale of chip #3 # this is done in the pipeline to keep the size of the # output drz product small chips = [member.chip for member in p.observation.members] """ try: chip_ind = chips.index('3') except ValueError: chip_ind = 0 oscale=p.observation.members[chip_ind].geometry.wcslin.pscale """ # Reset the plate-scale to a fixed value of 0.1 arcsecond/pixel # but NOT if the PC was the only chip read out. if len(chips) > 1 or (len(chips) == 1 and chips[0] != '1'): skf = pydrizzle.pydrizzle.SkyField() skf.set(psize=0.1) p.resetPars(skf) p.run(clean='yes') # Clean up any mask files which were created and not deleted by PyDrizzle fileutil.removeFile(glob.glob('wfpc2_inmask*.fits')) except: raise #update_header(filename) # Update header of output file, not input file update_header(p.output) trl_end = "END PYDRIZZLE------------ %s ------------\n" % time.asctime( ) else: trl_end = 'PYDRIZZLE processing not requested\n' trl_end += "END PYDRIZZLE------------ %s ------------\n" % time.asctime( ) return trl_start, trl_end else: trl_end = 'Keyword DRIZCORR missing from header.\n' trl_end += 'Pydrizzle did not run.\n' return trl_start, trl_end
def updateAsnTable(tabname, rootname, xsh=None, ysh=None, form="absolute", rot=None, scale=None, frame="input", units="pixels", output=None, mode="replace"): """ Updates an existing ASN table with shifts in arcseconds of RA and Dec. Parameters: tabname - name of ASN table to be updated rootname - name of image whose pointing needs to be updated xsh - X shift from nominal to be applied ysh - Y shift from nominal to be applied rot - additional rotation to be applied scale - scale factor to be applied form - specifies how shifts are computed: absolute (default) or delta frame - specifies whether xsh/ysh are given in terms of input (default) or output frame units - specifies units of xsh/ysh: pixels (default) or arcseconds output - filename (with or without path) of output image from which output pixels shifts were computed. mode - replace or add shifts to existing values 'replace' (default) or 'sum' The task will: - open the ASN file (tabname) - search for the row matching the rootname provided by the user (img), - convert any pixel shifts to arcseconds - update the XOFFSET, YOFFSET, ROTATION columns (or create them if they don't already exist) This task assumes that the shifts given as 'input' pixels are already distortion-corrected, but not scaled/rotated to output frame. The conversion of shifts given in terms of output pixel coordinates requires the use of the WCS from the output frame in order to correctly take into account the output frame's pixel scale and orientation. Both input and output image specifications assume '.fits' and that the WCS information appropriate for this are in either [0] or [sci,1]. If necessary a different extension can be specified in the filename. Also, the output filename can include a path which will be used. EXAMPLE: 1. Update ASN table 'j8cw03020_asn.fits' for 'j8cw03021_crj.fits' which has pixel shifts of (10.3,4.72) in (undistorted) input frame. >>> updateasn.updateAsnTable('j8cw03020_asn.fits','j8cw03021', xsh=10.3,ysh=4.72,rot=0.14) NOTE: This task still does not correctly work with 'NaN' entries in the table. For those values, it will not change the values at all. """ if frame == 'output' and output == None: print('Please specify output image needed to compute deltas...') return print('updateAsnTable Version ', __version__) _tmpname = 'buildasn_' + tabname # Start by opening the ASN table _asntab = pyfits.open(tabname, 'update') if form == 'absolute' or form == 'relative': # absolute/relative shifts _colx = 'XOFFSET' _coly = 'YOFFSET' else: # delta shifts _colx = 'XDELTA' _coly = 'YDELTA' # # Work out what needs to be done to update ASN table # Do the XOFFSET/YOFFSET/ROTATION columns already exist or not? # _update = no _add_rot = yes _add_scale = yes _colnum = 0 for name in _asntab[1].columns.names: if name == _colx: # Make the assumption that XOFFSET/YOFFSET and ROTATION # are all present if one is present... _update = yes break _colnum += 1 if _update: _tab_units = _asntab[1].columns.units[_colnum] if _tab_units == '' or _tab_units == None: _tab_units = 'pixels' else: _tab_units = units for name in _asntab[1].columns.names: if name == 'ROTATION': _add_rot = no break for name in _asntab[1].columns.names: if name == 'SCALE': _add_scale = no break # Get the shift frame and refimage if present # If keywords don't exist, assume default of 'input' shifts # with no need for refimage. try: _tab_frame = _asntab[0].header['shframe'] except KeyError: _tab_frame = 'input' try: _tab_refimg = _asntab[0].header['refimage'] except KeyError: _tab_refimg = None # Find row which corresponds to input image _rownum = 0 for row in _asntab[1].data: if rootname.find(row.field('MEMNAME')) > -1: _row = row break _rownum += 1 print('Updating table row ', _rownum, ' for image: ', rootname) # # Determine where to get the distortion coefficients # create complete filename from given rootname img = fileutil.buildRootname(rootname) if img == None: print('No valid input image for filename ', rootname) raise ValueError _instrument = fileutil.getKeyword(img, 'instrume') if _instrument not in IDCKEYS: _instrument = 'detector' _key = IDCKEYS[_instrument] if xsh == None: xsh = 0. if ysh == None: ysh = 0. # If we have any trouble computing offsets, abort and # do NOT update table. try: # Start by determining values to be used to update the table. # # Need to compute: _delta_ra, _delta_dec (undistorted arcsec) if units == _tab_units or (xsh == 0. and ysh == 0.): _delta_xoff = xsh _delta_yoff = ysh if rot == None: rot = 0. if scale == None: scale = 1.0 elif units.find('pixels') > -1 and _tab_units.find('arcsec') > -1: # # We need to convert the pixel values to arcseconds # _wcs = _getExposure(img, output, frame, _key) # If there is any additional rotation, account for it first if rot != None: if scale != None: pscale = scale * _wcs.pscale else: pscale = None _wcs.updateWCS(orient=_wcs.orient + rot, pixel_scale=pscale) # determine delta CRVALs from CRPIX+(xsh,ysh) _rd_delta = _wcs.xy2rd((_wcs.crpix1 + xsh, _wcs.crpix2 + ysh)) # Now, convert from degrees to arcseconds #### Do we need to multiply by cos(dec) for delta(RA)??? _delta_xoff = (_rd_delta[0] - _wcs.crval1) * 3600. _delta_yoff = (_rd_delta[1] - _wcs.crval2) * 3600. elif units.find('arcsec') > -1 and _tab_units.find('pixels') > -1: _wcs = _getExposure(img, _tab_refimg, _tab_frame, _key) # If there is any additional rotation, account for it first if rot != None: if scale != None: pscale = scale * _wcs.pscale else: pscale = None _wcs.updateWCS(orient=_wcs.orient + rot, pixel_scale=pscale) _xy_delta = _wcs.rd2xy((_wcs.crval1 + xsh, _wcs.crval2 + ysh)) _delta_xoff = (_xy_delta[0] - _wcs.crpix1) _delta_yoff = (_xy_delta[1] - _wcs.crpix2) except: print('ERROR: Error in updating table ', tabname) print('Closing existing table without updating it...') _asntab.close() return if _update: # Offset columns exist, just update the values in the table if mode == 'sum': _shift_xoff = _asntab[1].data.field(_colx)[_rownum] _shift_yoff = _asntab[1].data.field(_coly)[_rownum] _shift_rot = _asntab[1].data.field('ROTATION')[_rownum] _shift_scale = _asntab[1].data.field('SCALE')[_rownum] # Account for INDEF values in table # Logic: If set to INDEF, adding 1.0 will not change its value if _shift_xoff + 1.0 == _shift_xoff: _shift_xoff = 0. if _shift_yoff + 1.0 == _shift_yoff: _shift_yoff = 0. if _shift_rot + 1.0 == _shift_rot: _shift_rot = 0. if _shift_scale + 1.0 == _shift_scale: _shift_scale = 1.0 else: _shift_xoff = 0. _shift_yoff = 0. _shift_rot = 0. _shift_scale = 1. _asntab[1].data.field(_colx)[_rownum] = _shift_xoff + _delta_xoff _asntab[1].data.field(_coly)[_rownum] = _shift_yoff + _delta_yoff if rot != None: _asntab[1].data.field('ROTATION')[_rownum] = _shift_rot + rot if scale != None: _asntab[1].data.field('SCALE')[_rownum] = _shift_scale * scale else: # We need to add the extra columns to the table # Build arrays for each additional column _numrows = len(_asntab[1].data) _xsh = N.zeros(_numrows, dtype=N.float32) _ysh = N.zeros(_numrows, dtype=N.float32) # Now update entry associated with this particular image _xsh[_rownum] = _delta_xoff _ysh[_rownum] = _delta_yoff # Build the column objects for the table #_xc,_yc,_rc = buildasn._makeOffsetColumns(_xsh,_ysh,_rot) _xc = pyfits.Column(name=_colx, format='E', unit=_tab_units, array=_xsh) _yc = pyfits.Column(name=_coly, format='E', unit=_tab_units, array=_ysh) _newcols = _asntab[1].columns + _xc _newcols += _yc #_asndefs = _asntab[1].get_coldefs() #_asndefs.add_col(_xc) #_asndefs.add_col(_yc) # If rotation column does NOT exist, if _add_rot: _rot = N.zeros(_numrows, dtype=N.float32) if rot != None: _rot[_rownum] = rot _rc = pyfits.Column(name='ROTATION', format='E', unit='degrees', array=_rot) # add newly created column #_asndefs.add_col(_rc) _newcols += _rc else: # else, update column directly. _asntab[1].data.field('ROTATION')[_rownum] = rot # If scaling column does NOT exist, if _add_scale: _scale = N.zeros(_numrows, dtype=N.float32) if scale != None: _scale[_rownum] = scale _sc = pyfits.Column(name='SCALE', format='E', unit='', array=_scale) _newcols += _sc else: # else, update column directly. _asntab[1].data.field('SCALE')[_rownum] = scale _asnhdu = pyfits.new_table(_newcols) _asnhdu.writeto(_tmpname) # remove old data #del _asntab[1] # add table extension with old data and new columns #_asntab.append(_asnhdu) # Close and clean-up _asntab.close() del _asntab if os.path.exists(_tmpname): os.remove(tabname) os.rename(_tmpname, tabname)
def setCommonInput(configObj, createOutwcs=True): """ The common interface interpreter for MultiDrizzle tasks which not only runs 'process_input()' but 'createImageObject()' and 'defineOutput()' as well to fully setup all inputs for use with the rest of the MultiDrizzle steps either as stand-alone tasks or internally to MultiDrizzle itself. Parameters ---------- configObj : object configObj instance or simple dictionary of input parameters imageObjectList : list of imageObject objects list of imageObject instances, 1 for each input exposure outwcs : object imageObject instance defining the final output frame Notes ----- At a minimum, the configObj instance (dictionary) should contain: configObj = {'input':None,'output':None } If provided, the configObj should contain the values of all the multidrizzle parameters as set by the user with TEAL. If no configObj is given, it will retrieve the default values automatically. In either case, the values from the input_dict will be merged in with the configObj before being used by the rest of the code. Examples -------- You can set *createOutwcs=False* for the cases where you only want the images processed and no output wcs information in necessary; as in: >>> imageObjectList,outwcs = processInput.processCommonInput(configObj) """ # make sure 'updatewcs' is set to False when running from GUI or if missing # from configObj: if 'updatewcs' not in configObj: configObj['updatewcs'] = False if not createOutwcs or not configObj['coeffs']: # we're probably just working on single images here configObj['updatewcs']=False # maybe we can chunk this part up some more so that we can call just the # parts we want # Interpret input, read and convert and update input files, then return # list of input filenames and derived output filename asndict, ivmlist, output = process_input( configObj['input'], configObj['output'], updatewcs=configObj['updatewcs'], wcskey=configObj['wcskey'], **configObj['STATE OF INPUT FILES']) if not asndict: return None, None # convert the filenames from asndict into a list of full filenames files = [fileutil.buildRootname(f) for f in asndict['order']] original_files = asndict['original_file_names'] # interpret MDRIZTAB, if specified, and update configObj accordingly # This can be done here because MDRIZTAB does not include values for # input, output, or updatewcs. if 'mdriztab' in configObj and configObj['mdriztab']: print("Reading in MDRIZTAB parameters for {} files".format(len(files))) mdriztab_dict = mdzhandler.getMdriztabParameters(files) # Update configObj with values from mpars cfgpars.mergeConfigObj(configObj, mdriztab_dict) # Convert interpreted list of input files from process_input into a list # of imageObject instances for use by the MultiDrizzle tasks. instrpars = configObj['INSTRUMENT PARAMETERS'] # pass in 'proc_unit' to initialize unit conversions as necessary instrpars['proc_unit'] = configObj['proc_unit'] undistort = True if not configObj['coeffs']: undistort = False # determine whether parallel processing will be performed use_parallel = False if util.can_parallel: # look to see whether steps which can be run using multiprocessing # have been turned on for stepnum in parallel_steps: sname = util.getSectionName(configObj,stepnum[0]) if configObj[sname][stepnum[1]]: use_parallel = True break # interpret all 'bits' related parameters and convert them to integers configObj['resetbits'] = interpret_bit_flags(configObj['resetbits']) step3name = util.getSectionName(configObj,3) configObj[step3name]['driz_sep_bits'] = interpret_bit_flags( configObj[step3name]['driz_sep_bits'] ) step7name = util.getSectionName(configObj,7) configObj[step7name]['final_bits'] = interpret_bit_flags( configObj[step7name]['final_bits'] ) # Verify any refimage parameters to be used step3aname = util.getSectionName(configObj,'3a') if not util.verifyRefimage(configObj[step3aname]['driz_sep_refimage']): msg = 'No refimage with WCS found!\n '+\ ' This could be caused by one of 2 problems:\n'+\ ' * filename does not specify an extension with a valid WCS.\n'+\ ' * can not find the file.\n'+\ 'Please check the filename specified in the "refimage" parameter.' print(textutil.textbox(msg)) return None,None step7aname = util.getSectionName(configObj,'7a') if not util.verifyRefimage(configObj[step7aname]['final_refimage']): msg = 'No refimage with WCS found!\n '+\ ' This could be caused by one of 2 problems:\n'+\ ' * filename does not specify an extension with a valid WCS.\n'+\ ' * can not find the file.\n'+\ 'Please check the filename specified in the "refimage" parameter.' print(textutil.textbox(msg)) return None,None # Build imageObject list for all the valid, shift-updated input files log.info('-Creating imageObject List as input for processing steps.') if 'in_memory' in configObj: virtual = configObj['in_memory'] else: virtual = False imageObjectList = createImageObjectList(files, instrpars, group=configObj['group'], undistort=undistort, inmemory=virtual) # Add original file names as "hidden" attributes of imageObject assert(len(original_files) == len(imageObjectList)) #TODO: remove after extensive testing for i in range(len(imageObjectList)): imageObjectList[i]._original_file_name = original_files[i] # apply context parameter applyContextPar(imageObjectList, configObj['context']) # reset DQ bits if requested by user resetDQBits(imageObjectList, cr_bits_value=configObj['resetbits']) # Add info about input IVM files at this point to the imageObjectList addIVMInputs(imageObjectList, ivmlist) if createOutwcs: log.info('-Creating output WCS.') # Build output WCS and update imageObjectList with output WCS info outwcs = wcs_functions.make_outputwcs(imageObjectList, output, configObj=configObj, perfect=True) outwcs.final_wcs.printwcs() else: outwcs = None try: # Provide user with some information on resource usage for this run # raises ValueError Exception in interactive mode and user quits num_cores = configObj.get('num_cores') if use_parallel else 1 reportResourceUsage(imageObjectList, outwcs, num_cores) except ValueError: imageObjectList = None return imageObjectList, outwcs
def setCommonInput(configObj, createOutwcs=True): """ The common interface interpreter for MultiDrizzle tasks which not only runs 'process_input()' but 'createImageObject()' and 'defineOutput()' as well to fully setup all inputs for use with the rest of the MultiDrizzle steps either as stand-alone tasks or internally to MultiDrizzle itself. Parameters ---------- configObj : object configObj instance or simple dictionary of input parameters imageObjectList : list of imageObject objects list of imageObject instances, 1 for each input exposure outwcs : object imageObject instance defining the final output frame Notes ----- At a minimum, the configObj instance (dictionary) should contain: configObj = {'input':None,'output':None } If provided, the configObj should contain the values of all the multidrizzle parameters as set by the user with TEAL. If no configObj is given, it will retrieve the default values automatically. In either case, the values from the input_dict will be merged in with the configObj before being used by the rest of the code. Examples -------- You can set *createOutwcs=False* for the cases where you only want the images processed and no output wcs information in necessary; as in: >>> imageObjectList,outwcs = processInput.processCommonInput(configObj) """ # make sure 'updatewcs' is set to False when running from GUI or if missing # from configObj: if 'updatewcs' not in configObj: configObj['updatewcs'] = False if not createOutwcs or not configObj['coeffs']: # we're probably just working on single images here configObj['updatewcs'] = False # maybe we can chunk this part up some more so that we can call just the # parts we want # Interpret input, read and convert and update input files, then return # list of input filenames and derived output filename asndict, ivmlist, output = process_input( configObj['input'], configObj['output'], updatewcs=configObj['updatewcs'], wcskey=configObj['wcskey'], **configObj['STATE OF INPUT FILES']) if not asndict: return None, None # convert the filenames from asndict into a list of full filenames files = [fileutil.buildRootname(f) for f in asndict['order']] original_files = asndict['original_file_names'] # interpret MDRIZTAB, if specified, and update configObj accordingly # This can be done here because MDRIZTAB does not include values for # input, output, or updatewcs. if 'mdriztab' in configObj and configObj['mdriztab']: print("Reading in MDRIZTAB parameters for {} files".format(len(files))) mdriztab_dict = mdzhandler.getMdriztabParameters(files) # Update configObj with values from mpars cfgpars.mergeConfigObj(configObj, mdriztab_dict) # Convert interpreted list of input files from process_input into a list # of imageObject instances for use by the MultiDrizzle tasks. instrpars = configObj['INSTRUMENT PARAMETERS'] # pass in 'proc_unit' to initialize unit conversions as necessary instrpars['proc_unit'] = configObj['proc_unit'] undistort = True if not configObj['coeffs']: undistort = False # determine whether parallel processing will be performed use_parallel = False if util.can_parallel: # look to see whether steps which can be run using multiprocessing # have been turned on for stepnum in parallel_steps: sname = util.getSectionName(configObj, stepnum[0]) if configObj[sname][stepnum[1]]: use_parallel = True break # interpret all 'bits' related parameters and convert them to integers configObj['resetbits'] = interpret_bit_flags(configObj['resetbits']) step3name = util.getSectionName(configObj, 3) configObj[step3name]['driz_sep_bits'] = interpret_bit_flags( configObj[step3name]['driz_sep_bits']) step4name = util.getSectionName(configObj, 4) if len(files) > 5 and 'minmed' in configObj[step4name]['combine_type']: msg = '“minmed” is highly recommended for three images, \n'+\ ' and is good for four to six images, \n'+\ ' but should be avoided for ten or more images.\n' print(textutil.textbox(msg)) step7name = util.getSectionName(configObj, 7) configObj[step7name]['final_bits'] = interpret_bit_flags( configObj[step7name]['final_bits']) # Verify any refimage parameters to be used step3aname = util.getSectionName(configObj, '3a') if not util.verifyRefimage(configObj[step3aname]['driz_sep_refimage']): msg = 'No refimage with WCS found!\n '+\ ' This could be caused by one of 2 problems:\n'+\ ' * filename does not specify an extension with a valid WCS.\n'+\ ' * can not find the file.\n'+\ 'Please check the filename specified in the "refimage" parameter.' print(textutil.textbox(msg)) return None, None step7aname = util.getSectionName(configObj, '7a') if not util.verifyRefimage(configObj[step7aname]['final_refimage']): msg = 'No refimage with WCS found!\n '+\ ' This could be caused by one of 2 problems:\n'+\ ' * filename does not specify an extension with a valid WCS.\n'+\ ' * can not find the file.\n'+\ 'Please check the filename specified in the "refimage" parameter.' print(textutil.textbox(msg)) return None, None # Build imageObject list for all the valid, shift-updated input files log.info('-Creating imageObject List as input for processing steps.') if 'in_memory' in configObj: virtual = configObj['in_memory'] else: virtual = False imageObjectList = createImageObjectList(files, instrpars, output=asndict['output'], group=configObj['group'], undistort=undistort, inmemory=virtual) # Add original file names as "hidden" attributes of imageObject assert (len(original_files) == len(imageObjectList) ) #TODO: remove after extensive testing for i in range(len(imageObjectList)): imageObjectList[i]._original_file_name = original_files[i] # apply context parameter applyContextPar(imageObjectList, configObj['context']) # reset DQ bits if requested by user resetDQBits(imageObjectList, cr_bits_value=configObj['resetbits']) # Add info about input IVM files at this point to the imageObjectList addIVMInputs(imageObjectList, ivmlist) if createOutwcs: log.info('-Creating output WCS.') # Build output WCS and update imageObjectList with output WCS info outwcs = wcs_functions.make_outputwcs(imageObjectList, output, configObj=configObj, perfect=True) outwcs.final_wcs.printwcs() else: outwcs = None try: # Provide user with some information on resource usage for this run # raises ValueError Exception in interactive mode and user quits num_cores = configObj.get('num_cores') if use_parallel else 1 reportResourceUsage(imageObjectList, outwcs, num_cores) except ValueError: imageObjectList = None return imageObjectList, outwcs
def process_input(input, output=None, ivmlist=None, updatewcs=True, prodonly=False, shiftfile=None): ivmlist = None oldasndict = None if (isinstance(input, list) == False) and \ ('_asn' in input or '_asc' in input) : # Input is an association table # Get the input files, and run makewcs on them oldasndict = asnutil.readASNTable(input, prodonly=prodonly) if not output: output = oldasndict['output'] filelist = [fileutil.buildRootname(fname) for fname in oldasndict['order']] elif (isinstance(input, list) == False) and \ (input[0] == '@') : # input is an @ file f = open(input[1:]) # Read the first line in order to determine whether # IVM files have been specified in a second column... line = f.readline() f.close() # Parse the @-file with irafglob to extract the input filename filelist = irafglob.irafglob(input, atfile=atfile_sci) # If there is a second column... if len(line.split()) == 2: # ...parse out the names of the IVM files as well ivmlist = irafglob.irafglob(input, atfile=atfile_ivm) else: #input is a string or a python list try: filelist, output = parseinput.parseinput(input, outputname=output) #filelist.sort() except IOError: raise # sort the list of input files # this ensures the list of input files has the same order on all platforms # it can have ifferent order because listdir() uses inode order, not unix type order filelist.sort() newfilelist, ivmlist = checkFiles(filelist, ivmlist) if not newfilelist: buildEmptyDRZ(input,output) return None, None, output #make an asn table at the end if updatewcs: pydr_input = runmakewcs(newfilelist) else: pydr_input = newfilelist # AsnTable will handle the case when output==None if not oldasndict: oldasndict = asnutil.ASNTable(pydr_input, output=output) oldasndict.create() if shiftfile: oldasndict.update(shiftfile=shiftfile) asndict = update_member_names(oldasndict, pydr_input) # Build output filename drz_extn = '_drz.fits' for img in newfilelist: # special case logic to automatically recognize when _flc.fits files # are provided as input and produce a _drc.fits file instead if '_flc.fits' in img: drz_extn = '_drc.fits' break if output in [None,'']: output = fileutil.buildNewRootname(asndict['output'], extn=drz_extn) else: if '.fits' in output.lower(): pass elif drz_extn[:4] not in output.lower(): output = fileutil.buildNewRootname(output, extn=drz_extn) print('Setting up output name: ',output) return asndict, ivmlist, output
def process(inFile, force=False, newpath=None, inmemory=False, num_cores=None, headerlets=True): """ Run astrodrizzle on input file/ASN table using default values for astrodrizzle parameters. """ # We only need to import this package if a user run the task import drizzlepac from drizzlepac import processInput # used for creating new ASNs for _flc inputs import stwcs if headerlets: from stwcs.wcsutil import headerlet # Open the input file try: # Make sure given filename is complete and exists... inFilename = fileutil.buildRootname(inFile, ext=['.fits']) if not os.path.exists(inFilename): print("ERROR: Input file - %s - does not exist." % inFilename) return except TypeError: print("ERROR: Inappropriate input file.") return #If newpath was specified, move all files to that directory for processing if newpath: orig_processing_dir = os.getcwd() new_processing_dir = _createWorkingDir(newpath, inFilename) _copyToNewWorkingDir(new_processing_dir, inFilename) os.chdir(new_processing_dir) # Initialize for later use... _mname = None _new_asn = None _calfiles = [] # Check input file to see if [DRIZ/DITH]CORR is set to PERFORM if '_asn' in inFilename: # We are working with an ASN table. # Use asnutil code to extract filename inFilename = _lowerAsn(inFilename) _new_asn = [inFilename] _asndict = asnutil.readASNTable(inFilename, None, prodonly=False) _cal_prodname = _asndict['output'].lower() _fname = fileutil.buildRootname(_cal_prodname, ext=['_drz.fits']) # Retrieve the first member's rootname for possible use later _fimg = fits.open(inFilename) for name in _fimg[1].data.field('MEMNAME'): if name[-1] != '*': _mname = name.split('\0', 1)[0].lower() break _fimg.close() del _fimg else: # Check to see if input is a _RAW file # If it is, strip off the _raw.fits extension... _indx = inFilename.find('_raw') if _indx < 0: _indx = len(inFilename) # ... and build the CALXXX product rootname. _mname = fileutil.buildRootname(inFilename[:_indx]) _cal_prodname = inFilename[:_indx] # Reset inFilename to correspond to appropriate input for # drizzle: calibrated product name. inFilename = _mname if _mname == None: errorMsg = 'Could not find calibrated product!' raise Exception(errorMsg) # Create trailer filenames based on ASN output filename or # on input name for single exposures if '_raw' in inFile: # Output trailer file to RAW file's trailer _trlroot = inFile[:inFile.find('_raw')] elif '_asn' in inFile: # Output trailer file to ASN file's trailer, not product's trailer _trlroot = inFile[:inFile.find('_asn')] else: # Default: trim off last suffix of input filename # and replacing with .tra _indx = inFile.rfind('_') if _indx > 0: _trlroot = inFile[:_indx] else: _trlroot = inFile _trlfile = _trlroot + '.tra' # Open product and read keyword value # Check to see if product already exists... dkey = 'DRIZCORR' # ...if product does NOT exist, interrogate input file # to find out whether 'dcorr' has been set to PERFORM # Check if user wants to process again regardless of DRIZCORR keyword value if force: dcorr = 'PERFORM' else: if _mname: _fimg = fits.open(fileutil.buildRootname(_mname, ext=['_raw.fits'])) _phdr = _fimg['PRIMARY'].header if dkey in _phdr: dcorr = _phdr[dkey] else: dcorr = None _fimg.close() del _fimg else: dcorr = None time_str = _getTime() _tmptrl = _trlroot + '_tmp.tra' _drizfile = _trlroot + '_pydriz' _drizlog = _drizfile + ".log" # the '.log' gets added automatically by astrodrizzle if dcorr == 'PERFORM': if '_asn.fits' not in inFilename: # Working with a singleton # However, we always want to make sure we always use # a calibrated product as input, if available. _infile = fileutil.buildRootname(_cal_prodname) _infile_flc = fileutil.buildRootname(_cal_prodname, ext=['_flc.fits']) _cal_prodname = _infile _inlist = _calfiles = [_infile] # Add CTE corrected filename as additional input if present if os.path.exists(_infile_flc) and _infile_flc != _infile: _inlist.append(_infile_flc) else: # Working with an ASN table... _infile = inFilename flist, duplist = processInput.checkForDuplicateInputs( _asndict['order']) _calfiles = flist if len(duplist) > 0: origasn = processInput.changeSuffixinASN(inFilename, 'flt') dupasn = processInput.changeSuffixinASN(inFilename, 'flc') _inlist = [origasn, dupasn] else: _inlist = [_infile] # We want to keep the original specification of the calibration # product name, though, not a lower-case version... _cal_prodname = inFilename _new_asn.extend(_inlist) # kept so we can delete it when finished # Run astrodrizzle and send its processing statements to _trlfile _pyver = drizzlepac.astrodrizzle.__version__ for _infile in _inlist: # Run astrodrizzle for all inputs # Create trailer marker message for start of astrodrizzle processing _trlmsg = _timestamp('astrodrizzle started ') _trlmsg = _trlmsg + __trlmarker__ _trlmsg = _trlmsg + '%s: Processing %s with astrodrizzle Version %s\n' % ( time_str, _infile, _pyver) print(_trlmsg) # Write out trailer comments to trailer file... ftmp = open(_tmptrl, 'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile, _tmptrl) _pyd_err = _trlroot + '_pydriz.stderr' try: b = drizzlepac.astrodrizzle.AstroDrizzle(input=_infile, runfile=_drizfile, configobj='defaults', in_memory=inmemory, num_cores=num_cores, **pipeline_pars) except Exception as errorobj: _appendTrlFile(_trlfile, _drizlog) _appendTrlFile(_trlfile, _pyd_err) _ftrl = open(_trlfile, 'a') _ftrl.write( 'ERROR: Could not complete astrodrizzle processing of %s.\n' % _infile) _ftrl.write(str(sys.exc_info()[0]) + ': ') _ftrl.writelines(str(errorobj)) _ftrl.write('\n') _ftrl.close() print( 'ERROR: Could not complete astrodrizzle processing of %s.' % _infile) raise Exception(str(errorobj)) # Now, append comments created by PyDrizzle to CALXXX trailer file print('Updating trailer file %s with astrodrizzle comments.' % _trlfile) _appendTrlFile(_trlfile, _drizlog) # Save this for when astropy.io.fits can modify a file 'in-place' # Update calibration switch _fimg = fits.open(_cal_prodname, mode='update') _fimg['PRIMARY'].header[dkey] = 'COMPLETE' _fimg.close() del _fimg # Enforce pipeline convention of all lower-case product # names _prodlist = glob.glob('*drz.fits') for _prodname in _prodlist: _plower = _prodname.lower() if _prodname != _plower: os.rename(_prodname, _plower) else: # Create default trailer file messages when astrodrizzle is not # run on a file. This will typically apply only to BIAS,DARK # and other reference images. # Start by building up the message... _trlmsg = _timestamp('astrodrizzle skipped ') _trlmsg = _trlmsg + __trlmarker__ _trlmsg = _trlmsg + '%s: astrodrizzle processing not requested for %s.\n' % ( time_str, inFilename) _trlmsg = _trlmsg + ' astrodrizzle will not be run at this time.\n' print(_trlmsg) # Write message out to temp file and append it to full trailer file ftmp = open(_tmptrl, 'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile, _tmptrl) _fmsg = None # Append final timestamp to trailer file... _final_msg = '%s: Finished processing %s \n' % (time_str, inFilename) _final_msg += _timestamp('astrodrizzle completed ') _trlmsg += _final_msg ftmp = open(_tmptrl, 'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile, _tmptrl) # If we created a new ASN table, we need to remove it if _new_asn != None: for _name in _new_asn: fileutil.removeFile(_name) # Clean up any generated OrIg_files directory if os.path.exists("OrIg_files"): # check to see whether this directory is empty flist = glob.glob('OrIg_files/*.fits') if len(flist) == 0: os.rmdir("OrIg_files") else: print( 'OrIg_files directory NOT removed as it still contained images...' ) if headerlets: # Generate headerlets for each updated FLT image hlet_msg = _timestamp("Writing Headerlets started") for fname in _calfiles: frootname = fileutil.buildNewRootname(fname) hname = "%s_flt_hlet.fits" % frootname hlet_msg += "Created Headerlet file %s \n" % hname try: headerlet.write_headerlet( fname, 'OPUS', output='flt', wcskey='PRIMARY', author="OPUS", descrip="Default WCS from Pipeline Calibration", attach=False, clobber=True, logging=False) except ValueError: hlet_msg += _timestamp( "SKIPPED: Headerlet not created for %s \n" % fname) # update trailer file to log creation of headerlet files hlet_msg += _timestamp("Writing Headerlets completed") ftrl = open(_trlfile, 'a') ftrl.write(hlet_msg) ftrl.close() # If processing was done in a temp working dir, restore results to original # processing directory, return to original working dir and remove temp dir if newpath: _restoreResults(new_processing_dir, orig_processing_dir) os.chdir(orig_processing_dir) _removeWorkingDir(new_processing_dir) # Provide feedback to user print(_final_msg)
def process(inFile,force=False,newpath=None, inmemory=False, num_cores=None, headerlets=True): """ Run astrodrizzle on input file/ASN table using default values for astrodrizzle parameters. """ # We only need to import this package if a user run the task import drizzlepac from drizzlepac import processInput # used for creating new ASNs for _flc inputs import stwcs if headerlets: from stwcs.wcsutil import headerlet # Open the input file try: # Make sure given filename is complete and exists... inFilename = fileutil.buildRootname(inFile,ext=['.fits']) if not os.path.exists(inFilename): print("ERROR: Input file - %s - does not exist." % inFilename) return except TypeError: print("ERROR: Inappropriate input file.") return #If newpath was specified, move all files to that directory for processing if newpath: orig_processing_dir = os.getcwd() new_processing_dir = _createWorkingDir(newpath,inFilename) _copyToNewWorkingDir(new_processing_dir,inFilename) os.chdir(new_processing_dir) # Initialize for later use... _mname = None _new_asn = None _calfiles = [] # Identify WFPC2 inputs to account for differences in WFPC2 inputs wfpc2_input = fits.getval(inFilename, 'instrume') == 'WFPC2' cal_ext = None # Check input file to see if [DRIZ/DITH]CORR is set to PERFORM if '_asn' in inFilename: # We are working with an ASN table. # Use asnutil code to extract filename inFilename = _lowerAsn(inFilename) _new_asn = [inFilename] _asndict = asnutil.readASNTable(inFilename,None,prodonly=False) _cal_prodname = _asndict['output'].lower() _fname = fileutil.buildRootname(_cal_prodname,ext=['_drz.fits']) # Retrieve the first member's rootname for possible use later _fimg = fits.open(inFilename, memmap=False) for name in _fimg[1].data.field('MEMNAME'): if name[-1] != '*': _mname = name.split('\0', 1)[0].lower() break _fimg.close() del _fimg else: # Check to see if input is a _RAW file # If it is, strip off the _raw.fits extension... _indx = inFilename.find('_raw') if _indx < 0: _indx = len(inFilename) # ... and build the CALXXX product rootname. if wfpc2_input: # force code to define _c0m file as calibrated product to be used cal_ext = ['_c0m.fits'] _mname = fileutil.buildRootname(inFilename[:_indx], ext=cal_ext) _cal_prodname = inFilename[:_indx] # Reset inFilename to correspond to appropriate input for # drizzle: calibrated product name. inFilename = _mname if _mname is None: errorMsg = 'Could not find calibrated product!' raise Exception(errorMsg) # Create trailer filenames based on ASN output filename or # on input name for single exposures if '_raw' in inFile: # Output trailer file to RAW file's trailer _trlroot = inFile[:inFile.find('_raw')] elif '_asn' in inFile: # Output trailer file to ASN file's trailer, not product's trailer _trlroot = inFile[:inFile.find('_asn')] else: # Default: trim off last suffix of input filename # and replacing with .tra _indx = inFile.rfind('_') if _indx > 0: _trlroot = inFile[:_indx] else: _trlroot = inFile _trlfile = _trlroot + '.tra' # Open product and read keyword value # Check to see if product already exists... dkey = 'DRIZCORR' # ...if product does NOT exist, interrogate input file # to find out whether 'dcorr' has been set to PERFORM # Check if user wants to process again regardless of DRIZCORR keyword value if force: dcorr = 'PERFORM' else: if _mname : _fimg = fits.open(fileutil.buildRootname(_mname,ext=['_raw.fits']), memmap=False) _phdr = _fimg['PRIMARY'].header if dkey in _phdr: dcorr = _phdr[dkey] else: dcorr = None _fimg.close() del _fimg else: dcorr = None time_str = _getTime() _tmptrl = _trlroot + '_tmp.tra' _drizfile = _trlroot + '_pydriz' _drizlog = _drizfile + ".log" # the '.log' gets added automatically by astrodrizzle if dcorr == 'PERFORM': if '_asn.fits' not in inFilename: # Working with a singleton # However, we always want to make sure we always use # a calibrated product as input, if available. _infile = fileutil.buildRootname(_cal_prodname, ext=cal_ext) _infile_flc = fileutil.buildRootname(_cal_prodname,ext=['_flc.fits']) _cal_prodname = _infile _inlist = _calfiles = [_infile] # Add CTE corrected filename as additional input if present if os.path.exists(_infile_flc) and _infile_flc != _infile: _inlist.append(_infile_flc) else: # Working with an ASN table... _infile = inFilename flist,duplist = processInput.checkForDuplicateInputs(_asndict['order']) _calfiles = flist if len(duplist) > 0: origasn = processInput.changeSuffixinASN(inFilename,'flt') dupasn = processInput.changeSuffixinASN(inFilename,'flc') _inlist = [origasn,dupasn] else: _inlist = [_infile] # We want to keep the original specification of the calibration # product name, though, not a lower-case version... _cal_prodname = inFilename _new_asn.extend(_inlist) # kept so we can delete it when finished # Run astrodrizzle and send its processing statements to _trlfile _pyver = drizzlepac.astrodrizzle.__version__ for _infile in _inlist: # Run astrodrizzle for all inputs # Create trailer marker message for start of astrodrizzle processing _trlmsg = _timestamp('astrodrizzle started ') _trlmsg = _trlmsg+ __trlmarker__ _trlmsg = _trlmsg + '%s: Processing %s with astrodrizzle Version %s\n' % (time_str,_infile,_pyver) print(_trlmsg) # Write out trailer comments to trailer file... ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) _pyd_err = _trlroot+'_pydriz.stderr' try: b = drizzlepac.astrodrizzle.AstroDrizzle(input=_infile,runfile=_drizfile, configobj='defaults',in_memory=inmemory, num_cores=num_cores, **pipeline_pars) except Exception as errorobj: _appendTrlFile(_trlfile,_drizlog) _appendTrlFile(_trlfile,_pyd_err) _ftrl = open(_trlfile,'a') _ftrl.write('ERROR: Could not complete astrodrizzle processing of %s.\n' % _infile) _ftrl.write(str(sys.exc_info()[0])+': ') _ftrl.writelines(str(errorobj)) _ftrl.write('\n') _ftrl.close() print('ERROR: Could not complete astrodrizzle processing of %s.' % _infile) raise Exception(str(errorobj)) # Now, append comments created by PyDrizzle to CALXXX trailer file print('Updating trailer file %s with astrodrizzle comments.' % _trlfile) _appendTrlFile(_trlfile,_drizlog) # Save this for when astropy.io.fits can modify a file 'in-place' # Update calibration switch _fimg = fits.open(_cal_prodname, mode='update', memmap=False) _fimg['PRIMARY'].header[dkey] = 'COMPLETE' _fimg.close() del _fimg # Enforce pipeline convention of all lower-case product # names _prodlist = glob.glob('*drz.fits') for _prodname in _prodlist: _plower = _prodname.lower() if _prodname != _plower: os.rename(_prodname,_plower) else: # Create default trailer file messages when astrodrizzle is not # run on a file. This will typically apply only to BIAS,DARK # and other reference images. # Start by building up the message... _trlmsg = _timestamp('astrodrizzle skipped ') _trlmsg = _trlmsg + __trlmarker__ _trlmsg = _trlmsg + '%s: astrodrizzle processing not requested for %s.\n' % (time_str,inFilename) _trlmsg = _trlmsg + ' astrodrizzle will not be run at this time.\n' print(_trlmsg) # Write message out to temp file and append it to full trailer file ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) _fmsg = None # Append final timestamp to trailer file... _final_msg = '%s: Finished processing %s \n' % (time_str,inFilename) _final_msg += _timestamp('astrodrizzle completed ') _trlmsg += _final_msg ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) # If we created a new ASN table, we need to remove it if _new_asn is not None: for _name in _new_asn: fileutil.removeFile(_name) # Clean up any generated OrIg_files directory if os.path.exists("OrIg_files"): # check to see whether this directory is empty flist = glob.glob('OrIg_files/*.fits') if len(flist) == 0: os.rmdir("OrIg_files") else: print('OrIg_files directory NOT removed as it still contained images...') if headerlets: # Generate headerlets for each updated FLT image hlet_msg = _timestamp("Writing Headerlets started") for fname in _calfiles: frootname = fileutil.buildNewRootname(fname) hname = "%s_flt_hlet.fits"%frootname hlet_msg += "Created Headerlet file %s \n"%hname try: headerlet.write_headerlet(fname,'OPUS',output='flt', wcskey='PRIMARY', author="OPUS",descrip="Default WCS from Pipeline Calibration", attach=False,clobber=True,logging=False) except ValueError: hlet_msg += _timestamp("SKIPPED: Headerlet not created for %s \n"%fname) # update trailer file to log creation of headerlet files hlet_msg += _timestamp("Writing Headerlets completed") ftrl = open(_trlfile,'a') ftrl.write(hlet_msg) ftrl.close() # If processing was done in a temp working dir, restore results to original # processing directory, return to original working dir and remove temp dir if newpath: _restoreResults(new_processing_dir,orig_processing_dir) os.chdir(orig_processing_dir) _removeWorkingDir(new_processing_dir) # Provide feedback to user print(_final_msg)
def process(inFile,force=False,newpath=None, inmemory=False, num_cores=None, headerlets=True, align_to_gaia=True): """ Run astrodrizzle on input file/ASN table using default values for astrodrizzle parameters. """ # We only need to import this package if a user run the task import drizzlepac from drizzlepac import processInput # used for creating new ASNs for _flc inputs from stwcs import updatewcs from drizzlepac import alignimages # interpret envvar variable, if specified if envvar_compute_name in os.environ: val = os.environ[envvar_compute_name].lower() if val not in envvar_bool_dict: msg = "ERROR: invalid value for {}.".format(envvar_compute_name) msg += " \n Valid Values: on, off, yes, no, true, false" raise ValueError(msg) align_to_gaia = envvar_bool_dict[val] if envvar_new_apriori_name in os.environ: # Reset ASTROMETRY_STEP_CONTROL based on this variable # This provides backward-compatibility until ASTROMETRY_STEP_CONTROL # gets removed entirely. val = os.environ[envvar_new_apriori_name].lower() if val not in envvar_dict: msg = "ERROR: invalid value for {}.".format(envvar_new_apriori_name) msg += " \n Valid Values: on, off, yes, no, true, false" raise ValueError(msg) os.environ[envvar_old_apriori_name] = envvar_dict[val] if headerlets or align_to_gaia: from stwcs.wcsutil import headerlet # Open the input file try: # Make sure given filename is complete and exists... inFilename = fileutil.buildRootname(inFile,ext=['.fits']) if not os.path.exists(inFilename): print("ERROR: Input file - %s - does not exist." % inFilename) return except TypeError: print("ERROR: Inappropriate input file.") return #If newpath was specified, move all files to that directory for processing if newpath: orig_processing_dir = os.getcwd() new_processing_dir = _createWorkingDir(newpath,inFilename) _copyToNewWorkingDir(new_processing_dir,inFilename) os.chdir(new_processing_dir) # Initialize for later use... _mname = None _new_asn = None _calfiles = [] # Identify WFPC2 inputs to account for differences in WFPC2 inputs wfpc2_input = fits.getval(inFilename, 'instrume') == 'WFPC2' cal_ext = None # Check input file to see if [DRIZ/DITH]CORR is set to PERFORM if '_asn' in inFilename: # We are working with an ASN table. # Use asnutil code to extract filename inFilename = _lowerAsn(inFilename) _new_asn = [inFilename] _asndict = asnutil.readASNTable(inFilename,None,prodonly=False) _cal_prodname = _asndict['output'].lower() #_fname = fileutil.buildRootname(_cal_prodname,ext=['_drz.fits']) # Retrieve the first member's rootname for possible use later _fimg = fits.open(inFilename, memmap=False) for name in _fimg[1].data.field('MEMNAME'): if name[-1] != '*': _mname = name.split('\0', 1)[0].lower() break _fimg.close() del _fimg else: # Check to see if input is a _RAW file # If it is, strip off the _raw.fits extension... _indx = inFilename.find('_raw') if _indx < 0: _indx = len(inFilename) # ... and build the CALXXX product rootname. if wfpc2_input: # force code to define _c0m file as calibrated product to be used cal_ext = ['_c0m.fits'] _mname = fileutil.buildRootname(inFilename[:_indx], ext=cal_ext) _cal_prodname = inFilename[:_indx] # Reset inFilename to correspond to appropriate input for # drizzle: calibrated product name. inFilename = _mname if _mname is None: errorMsg = 'Could not find calibrated product!' raise Exception(errorMsg) # Create trailer filenames based on ASN output filename or # on input name for single exposures if '_raw' in inFile: # Output trailer file to RAW file's trailer _trlroot = inFile[:inFile.find('_raw')] elif '_asn' in inFile: # Output trailer file to ASN file's trailer, not product's trailer _trlroot = inFile[:inFile.find('_asn')] else: # Default: trim off last suffix of input filename # and replacing with .tra _indx = inFile.rfind('_') if _indx > 0: _trlroot = inFile[:_indx] else: _trlroot = inFile _trlfile = _trlroot + '.tra' # Open product and read keyword value # Check to see if product already exists... dkey = 'DRIZCORR' # ...if product does NOT exist, interrogate input file # to find out whether 'dcorr' has been set to PERFORM # Check if user wants to process again regardless of DRIZCORR keyword value if force: dcorr = 'PERFORM' else: if _mname : _fimg = fits.open(fileutil.buildRootname(_mname,ext=['_raw.fits']), memmap=False) _phdr = _fimg['PRIMARY'].header if dkey in _phdr: dcorr = _phdr[dkey] else: dcorr = None _fimg.close() del _fimg else: dcorr = None time_str = _getTime() _tmptrl = _trlroot + '_tmp.tra' _drizfile = _trlroot + '_pydriz' _drizlog = _drizfile + ".log" # the '.log' gets added automatically by astrodrizzle _alignlog = _trlroot + '_align.log' if dcorr == 'PERFORM': if '_asn.fits' not in inFilename: # Working with a singleton # However, we always want to make sure we always use # a calibrated product as input, if available. _infile = fileutil.buildRootname(_cal_prodname, ext=cal_ext) _infile_flc = fileutil.buildRootname(_cal_prodname,ext=['_flc.fits']) _cal_prodname = _infile _inlist = _calfiles = [_infile] # Add CTE corrected filename as additional input if present if os.path.exists(_infile_flc) and _infile_flc != _infile: _inlist.append(_infile_flc) else: # Working with an ASN table... _infile = inFilename flist,duplist = processInput.checkForDuplicateInputs(_asndict['order']) _calfiles = flist if len(duplist) > 0: origasn = processInput.changeSuffixinASN(inFilename,'flt') dupasn = processInput.changeSuffixinASN(inFilename,'flc') _inlist = [origasn,dupasn] else: _inlist = [_infile] # We want to keep the original specification of the calibration # product name, though, not a lower-case version... _cal_prodname = inFilename _new_asn.extend(_inlist) # kept so we can delete it when finished # check to see whether FLC files are also present, and need to be updated # generate list of FLC files align_files = None _calfiles_flc = [f.replace('_flt.fits','_flc.fits') for f in _calfiles] # insure these files exist, if not, blank them out # Also pick out what files will be used for additional alignment to GAIA if not os.path.exists(_calfiles_flc[0]): _calfiles_flc = None align_files = _calfiles align_update_files = None else: align_files = _calfiles_flc align_update_files = _calfiles # Run updatewcs on each list of images updatewcs.updatewcs(_calfiles) if _calfiles_flc: updatewcs.updatewcs(_calfiles_flc) if align_to_gaia: # Perform additional alignment on the FLC files, if present ############### # # call hlapipeline code here on align_files list of files # ############### # Create trailer marker message for start of align_to_GAIA processing _trlmsg = _timestamp("Align_to_GAIA started ") print(_trlmsg) ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) _trlmsg = "" # Create an empty astropy table so it can be used as input/output for the perform_align function #align_table = Table() try: align_table = alignimages.perform_align(align_files,update_hdr_wcs=True, runfile=_alignlog) for row in align_table: if row['status'] == 0: trlstr = "Successfully aligned {} to {} astrometric frame\n" _trlmsg += trlstr.format(row['imageName'], row['catalog']) else: trlstr = "Could not align {} to absolute astrometric frame\n" _trlmsg += trlstr.format(row['imageName']) except Exception: # Something went wrong with alignment to GAIA, so report this in # trailer file _trlmsg = "EXCEPTION encountered in alignimages...\n" _trlmsg += " No correction to absolute astrometric frame applied!\n" # Write the perform_align log to the trailer file...(this will delete the _alignlog) _appendTrlFile(_trlfile,_alignlog) # Append messages from this calling routine post-perform_align ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) _trlmsg = "" #Check to see whether there are any additional input files that need to # be aligned (namely, FLT images) if align_update_files and align_table: # Apply headerlets from alignment to FLT version of the files for fltfile, flcfile in zip(align_update_files, align_files): row = align_table[align_table['imageName']==flcfile] headerletFile = row['headerletFile'][0] if headerletFile != "None": headerlet.apply_headerlet_as_primary(fltfile, headerletFile, attach=True, archive=True) # append log file contents to _trlmsg for inclusion in trailer file _trlstr = "Applying headerlet {} as Primary WCS to {}\n" _trlmsg += _trlstr.format(headerletFile, fltfile) else: _trlmsg += "No absolute astrometric headerlet applied to {}\n".format(fltfile) # Finally, append any further messages associated with alignement from this calling routine _trlmsg += _timestamp('Align_to_GAIA completed ') print(_trlmsg) ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) # Run astrodrizzle and send its processing statements to _trlfile _pyver = drizzlepac.astrodrizzle.__version__ for _infile in _inlist: # Run astrodrizzle for all inputs # Create trailer marker message for start of astrodrizzle processing _trlmsg = _timestamp('astrodrizzle started ') _trlmsg += __trlmarker__ _trlmsg += '%s: Processing %s with astrodrizzle Version %s\n' % (time_str,_infile,_pyver) print(_trlmsg) # Write out trailer comments to trailer file... ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) _pyd_err = _trlroot+'_pydriz.stderr' try: b = drizzlepac.astrodrizzle.AstroDrizzle(input=_infile,runfile=_drizfile, configobj='defaults',in_memory=inmemory, num_cores=num_cores, **pipeline_pars) except Exception as errorobj: _appendTrlFile(_trlfile,_drizlog) _appendTrlFile(_trlfile,_pyd_err) _ftrl = open(_trlfile,'a') _ftrl.write('ERROR: Could not complete astrodrizzle processing of %s.\n' % _infile) _ftrl.write(str(sys.exc_info()[0])+': ') _ftrl.writelines(str(errorobj)) _ftrl.write('\n') _ftrl.close() print('ERROR: Could not complete astrodrizzle processing of %s.' % _infile) raise Exception(str(errorobj)) # Now, append comments created by PyDrizzle to CALXXX trailer file print('Updating trailer file %s with astrodrizzle comments.' % _trlfile) _appendTrlFile(_trlfile,_drizlog) # Save this for when astropy.io.fits can modify a file 'in-place' # Update calibration switch _fimg = fits.open(_cal_prodname, mode='update', memmap=False) _fimg['PRIMARY'].header[dkey] = 'COMPLETE' _fimg.close() del _fimg # Enforce pipeline convention of all lower-case product # names _prodlist = glob.glob('*drz.fits') for _prodname in _prodlist: _plower = _prodname.lower() if _prodname != _plower: os.rename(_prodname,_plower) else: # Create default trailer file messages when astrodrizzle is not # run on a file. This will typically apply only to BIAS,DARK # and other reference images. # Start by building up the message... _trlmsg = _timestamp('astrodrizzle skipped ') _trlmsg = _trlmsg + __trlmarker__ _trlmsg = _trlmsg + '%s: astrodrizzle processing not requested for %s.\n' % (time_str,inFilename) _trlmsg = _trlmsg + ' astrodrizzle will not be run at this time.\n' print(_trlmsg) # Write message out to temp file and append it to full trailer file ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) # Append final timestamp to trailer file... _final_msg = '%s: Finished processing %s \n' % (time_str,inFilename) _final_msg += _timestamp('astrodrizzle completed ') _trlmsg += _final_msg ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) # If we created a new ASN table, we need to remove it if _new_asn is not None: for _name in _new_asn: fileutil.removeFile(_name) # Clean up any generated OrIg_files directory if os.path.exists("OrIg_files"): # check to see whether this directory is empty flist = glob.glob('OrIg_files/*.fits') if len(flist) == 0: os.rmdir("OrIg_files") else: print('OrIg_files directory NOT removed as it still contained images...') # If headerlets have already been written out by alignment code, # do NOT write out this version of the headerlets if headerlets: # Generate headerlets for each updated FLT image hlet_msg = _timestamp("Writing Headerlets started") for fname in _calfiles: frootname = fileutil.buildNewRootname(fname) hname = "%s_flt_hlet.fits"%frootname # Write out headerlet file used by astrodrizzle, however, # do not overwrite any that was already written out by alignimages if not os.path.exists(hname): hlet_msg += "Created Headerlet file %s \n"%hname try: headerlet.write_headerlet(fname,'OPUS',output='flt', wcskey='PRIMARY', author="OPUS",descrip="Default WCS from Pipeline Calibration", attach=False,clobber=True,logging=False) except ValueError: hlet_msg += _timestamp("SKIPPED: Headerlet not created for %s \n"%fname) # update trailer file to log creation of headerlet files hlet_msg += _timestamp("Writing Headerlets completed") ftrl = open(_trlfile,'a') ftrl.write(hlet_msg) ftrl.close() # If processing was done in a temp working dir, restore results to original # processing directory, return to original working dir and remove temp dir if newpath: _restoreResults(new_processing_dir,orig_processing_dir) os.chdir(orig_processing_dir) _removeWorkingDir(new_processing_dir) # Provide feedback to user print(_final_msg)
def writeAsnDict(asndict, output=None): """ writeAsnDict: ============= Write out a new ASN table using a dictionary in memory. The input dictionary should be read in using 'readAsnTable', and can be modified as needed before writing out the new table. SYNTAX: buildasn.writeAsnDict(asndict,output=None) PARAMETERS: asndict: dictionary from 'readAsnTable output: rootname or filename for output ASN file if output == None (default), product name from asndict will be used to define the output filename for the table. """ # Convert the ASN dictionary from 'readAsnTable' # into a format usable by 'buildasn' functions. tbldict = makeTableDict(asndict) # Extract info from table necessary for writing it out if output == None: outfile = asndict['output'] + '_asn.fits' else: if output.find('_asn.fits') < 0: outfile = output + '_asn.fits' else: outfile = output # Delete the file if it exists. if os.path.exists(outfile): warningmsg = "\n#########################################\n" warningmsg += "# #\n" warningmsg += "# WARNING: #\n" warningmsg += "# The exisiting assocation table, #\n" warningmsg += " " + str(outfile) + '\n' warningmsg += "# is being replaced by buildasn. #\n" warningmsg += "# #\n" warningmsg += "#########################################\n\n" print(warningmsg) os.remove(outfile) mem0name = asndict['order'][0] refimg = asndict['members'][mem0name]['refimage'] shframe = tbldict['frame'] # Input image to be used as template fname = buildRootname(mem0name) # Open output ASN table file fasn = pyfits.HDUList() # Build ASN file primary header _buildAsnPrimary(fasn, outfile, fname, frame=shframe, refimage=refimg) # Create an extension HDU which contains a table exthdu = _makeTableHDU(tbldict) fasn.append(exthdu) # Close ASN table file fasn.writeto(outfile) fasn.close() del fasn return outfile