def wf2_pydr(filename): trl_start = "PYDRIZZLE STARTED------------ %s ------------\n" % time.asctime( ) filename = fileutil.buildRootname(filename, ext=['_c0m.fits', '_c0f.fits']) try: hdr0 = pyfits.getheader(filename) except: errormsg = "\n\nCan't find file %s \n", filename raise IOError(errormsg) try: dodrizcorr = hdr0['DRIZCORR'] except KeyError: print("Keyword DRIZCORR not found\n") dodrizcorr = None if isSupportedFilter(hdr0) and isSupportedOfftab(hdr0): useIDCModel = True else: useIDCModel = False if dodrizcorr != None: if dodrizcorr == 'PERFORM': try: # update WCS information if useIDCModel: p = pydrizzle.PyDrizzle(filename, bits_single=0, bits_final=0) else: p = pydrizzle.PyDrizzle(filename, bits_single=0, bits_final=0, updatewcs=False, idckey=None) # resample to the scale of chip #3 # this is done in the pipeline to keep the size of the # output drz product small chips = [member.chip for member in p.observation.members] """ try: chip_ind = chips.index('3') except ValueError: chip_ind = 0 oscale=p.observation.members[chip_ind].geometry.wcslin.pscale """ # Reset the plate-scale to a fixed value of 0.1 arcsecond/pixel # but NOT if the PC was the only chip read out. if len(chips) > 1 or (len(chips) == 1 and chips[0] != '1'): skf = pydrizzle.pydrizzle.SkyField() skf.set(psize=0.1) p.resetPars(skf) p.run(clean='yes') # Clean up any mask files which were created and not deleted by PyDrizzle fileutil.removeFile(glob.glob('wfpc2_inmask*.fits')) except: raise #update_header(filename) # Update header of output file, not input file update_header(p.output) trl_end = "END PYDRIZZLE------------ %s ------------\n" % time.asctime( ) else: trl_end = 'PYDRIZZLE processing not requested\n' trl_end += "END PYDRIZZLE------------ %s ------------\n" % time.asctime( ) return trl_start, trl_end else: trl_end = 'Keyword DRIZCORR missing from header.\n' trl_end += 'Pydrizzle did not run.\n' return trl_start, trl_end
def writeFITS(self, template, sciarr, whtarr, ctxarr=None, versions=None, overwrite=yes, blend=True, virtual=False, rules_file=None, logfile=None): """ Generate PyFITS objects for each output extension using the file given by 'template' for populating headers. The arrays will have the size specified by 'shape'. """ if not isinstance(template, list): template = [template] if fileutil.findFile(self.output): if overwrite: log.info('Deleting previous output product: %s' % self.output) fileutil.removeFile(self.output) else: log.warning('Output file %s already exists and overwrite not ' 'specified!' % self.output) log.error('Quitting... Please remove before resuming ' 'operations.') raise IOError # initialize output value for this method outputFITS = {} # Default value for NEXTEND when 'build'== True nextend = 3 if not self.build: nextend = 0 if self.outweight: if overwrite: if fileutil.findFile(self.outweight): log.info('Deleting previous output WHT product: %s' % self.outweight) fileutil.removeFile(self.outweight) else: log.warning('Output file %s already exists and overwrite ' 'not specified!' % self.outweight) log.error('Quitting... Please remove before resuming ' 'operations.') raise IOError if self.outcontext: if overwrite: if fileutil.findFile(self.outcontext): log.info('Deleting previous output CTX product: %s' % self.outcontext) fileutil.removeFile(self.outcontext) else: log.warning('Output file %s already exists and overwrite ' 'not specified!' % self.outcontext) log.error('Quitting... Please remove before resuming ' 'operations.') raise IOError # Get default headers from multi-extension FITS file # If only writing out single drizzle product, blending needs to be # forced off as there is only 1 input to report, no blending needed if self.single: blend = False # If input data is not in MEF FITS format, it will return 'None' # and those headers will have to be generated from drizzle output # file FITS headers. # NOTE: These are HEADER objects, not HDUs self.fullhdrs, intab = getTemplates(template, blend=False, rules_file=rules_file) newhdrs, newtab = getTemplates(template, blend=blend, rules_file=rules_file) if newtab is not None: nextend += 1 # account for new table extn prihdr = newhdrs[0] scihdr = newhdrs[1] errhdr = newhdrs[2] dqhdr = newhdrs[3] # Setup primary header as an HDU ready for appending to output FITS file prihdu = fits.PrimaryHDU(header=prihdr, data=None) # Start by updating PRIMARY header keywords... prihdu.header.set('EXTEND', value=True, after='NAXIS') prihdu.header['NEXTEND'] = nextend prihdu.header['FILENAME'] = self.output prihdu.header['PROD_VER'] = 'DrizzlePac {}'.format(__version__) prihdu.header['DRIZPARS'] = (logfile, "Logfile for processing") # Update the ROOTNAME with the new value as well _indx = self.output.find('_drz') if _indx < 0: rootname_val = self.output else: rootname_val = self.output[:_indx] prihdu.header['ROOTNAME'] = rootname_val # Get the total exposure time for the image # If not calculated by PyDrizzle and passed through # the pardict, then leave value from the template image. if self.texptime: prihdu.header['EXPTIME'] = self.texptime prihdu.header.set('TEXPTIME', value=self.texptime, after='EXPTIME') prihdu.header['EXPSTART'] = self.expstart prihdu.header['EXPEND'] = self.expend # Update ASN_MTYPE to reflect the fact that this is a product # Currently hard-wired to always output 'PROD-DTH' as MTYPE prihdu.header['ASN_MTYP'] = 'PROD-DTH' # Update DITHCORR calibration keyword if present # Remove when we can modify FITS headers in place... if 'DRIZCORR' in prihdu.header: prihdu.header['DRIZCORR'] = 'COMPLETE' if 'DITHCORR' in prihdu.header: prihdu.header['DITHCORR'] = 'COMPLETE' prihdu.header['NDRIZIM'] = (len(self.parlist), 'Drizzle, No. images drizzled onto output') # Only a subset of these keywords makes sense for the new WCS based # transformations. They need to be reviewed to decide what to keep # and what to leave out. if not self.blot: self.addDrizKeywords(prihdu.header, versions) if scihdr: scihdr.pop('OBJECT', None) if 'CCDCHIP' in scihdr: scihdr['CCDCHIP'] = '-999' if 'NCOMBINE' in scihdr: scihdr['NCOMBINE'] = self.parlist[0]['nimages'] # If BUNIT keyword was found and reset, then bunit_last_kw = self.find_kwupdate_location(scihdr, 'bunit') if self.bunit is not None: comment_str = "Units of science product" if self.bunit.lower()[:5] == 'count': comment_str = "counts * gain = electrons" scihdr.set('BUNIT', value=self.bunit, comment=comment_str, after=bunit_last_kw) else: # check to see whether to update already present BUNIT comment if 'bunit' in scihdr and scihdr['bunit'].lower( )[:5] == 'count': comment_str = "counts * gain = electrons" scihdr.set('BUNIT', value=scihdr['bunit'], comment=comment_str, after=bunit_last_kw) # Add WCS keywords to SCI header if self.wcs: pre_wcs_kw = self.find_kwupdate_location(scihdr, 'CD1_1') addWCSKeywords(self.wcs, scihdr, blot=self.blot, single=self.single, after=pre_wcs_kw) # Recompute this after removing distortion kws pre_wcs_kw = self.find_kwupdate_location(scihdr, 'CD1_1') ########## # Now, build the output file ########## if self.build: print('-Generating multi-extension output file: ', self.output) fo = fits.HDUList() # Add primary header to output file... fo.append(prihdu) if self.single and self.compress: hdu = fits.CompImageHDU(data=sciarr, header=scihdr, name=EXTLIST[0]) else: hdu = fits.ImageHDU(data=sciarr, header=scihdr, name=EXTLIST[0]) last_kw = self.find_kwupdate_location(scihdr, 'EXTNAME') hdu.header.set('EXTNAME', value='SCI', after=last_kw) hdu.header.set('EXTVER', value=1, after='EXTNAME') fo.append(hdu) # Build WHT extension here, if requested... if errhdr: errhdr['CCDCHIP'] = '-999' if self.single and self.compress: hdu = fits.CompImageHDU(data=whtarr, header=errhdr, name=EXTLIST[1]) else: hdu = fits.ImageHDU(data=whtarr, header=errhdr, name=EXTLIST[1]) last_kw = self.find_kwupdate_location(errhdr, 'EXTNAME') hdu.header.set('EXTNAME', value='WHT', after=last_kw) hdu.header.set('EXTVER', value=1, after='EXTNAME') if self.wcs: pre_wcs_kw = self.find_kwupdate_location(hdu.header, 'CD1_1') # Update WCS Keywords based on PyDrizzle product's value # since 'drizzle' itself doesn't update that keyword. addWCSKeywords(self.wcs, hdu.header, blot=self.blot, single=self.single, after=pre_wcs_kw) fo.append(hdu) # Build CTX extension here # If there is only 1 plane, write it out as a 2-D extension if self.outcontext: if ctxarr.shape[0] == 1: _ctxarr = ctxarr[0] else: _ctxarr = ctxarr else: _ctxarr = None if self.single and self.compress: hdu = fits.CompImageHDU(data=_ctxarr, header=dqhdr, name=EXTLIST[2]) else: hdu = fits.ImageHDU(data=_ctxarr, header=dqhdr, name=EXTLIST[2]) last_kw = self.find_kwupdate_location(dqhdr, 'EXTNAME') hdu.header.set('EXTNAME', value='CTX', after=last_kw) hdu.header.set('EXTVER', value=1, after='EXTNAME') if self.wcs: pre_wcs_kw = self.find_kwupdate_location(hdu.header, 'CD1_1') # Update WCS Keywords based on PyDrizzle product's value # since 'drizzle' itself doesn't update that keyword. addWCSKeywords(self.wcs, hdu.header, blot=self.blot, single=self.single, after=pre_wcs_kw) fo.append(hdu) # remove all alternate WCS solutions from headers of this product wcs_functions.removeAllAltWCS(fo, [1]) # add table of combined header keyword values to FITS file if newtab is not None: fo.append(newtab) if not virtual: print('Writing out to disk:', self.output) # write out file to disk fo.writeto(self.output) fo.close() del fo, hdu fo = None # End 'if not virtual' outputFITS[self.output] = fo else: print('-Generating simple FITS output: %s' % self.outdata) fo = fits.HDUList() hdu_header = prihdu.header.copy() del hdu_header['nextend'] # Append remaining unique header keywords from template DQ # header to Primary header... if scihdr: for _card in scihdr.cards: if _card.keyword not in RESERVED_KEYS and _card.keyword not in hdu_header: hdu_header.append(_card) hdu_header['filename'] = self.outdata if self.compress: hdu = fits.CompImageHDU(data=sciarr, header=hdu_header) wcs_ext = [1] else: hdu = fits.PrimaryHDU(data=sciarr, header=hdu_header) wcs_ext = [0] # explicitly set EXTEND to FALSE for simple FITS files. dim = len(sciarr.shape) hdu.header.set('extend', value=False, after='NAXIS%s' % dim) # explicitly remove EXTNAME, EXTVER from header # since this header may have been used # to create a CompImageHDU instance instead of a PrimaryHDU instance for kw in ['EXTNAME', 'EXTVER', 'PCOUNT', 'GCOUNT']: hdu.header.pop(kw, None) hdu.header.set('filetype', 'SCI', before='TELESCOP', comment='Type of data in array') # Add primary header to output file... fo.append(hdu) if not self.blot: # remove all alternate WCS solutions from headers of this product logutil.logging.disable(logutil.logging.INFO) wcs_functions.removeAllAltWCS(fo, wcs_ext) logutil.logging.disable(logutil.logging.NOTSET) # add table of combined header keyword values to FITS file if newtab is not None: fo.append(newtab) if not virtual or "single_sci" in self.outdata: print('Writing out image to disk:', self.outdata) # write out file to disk fo.writeto(self.outdata, overwrite=True) del hdu if "single_sci" not in self.outdata: del fo fo = None # End 'if not virtual' outputFITS[self.outdata] = fo if self.outweight and whtarr is not None: # We need to build new PyFITS objects for each WHT array fwht = fits.HDUList() if errhdr: errhdr['CCDCHIP'] = '-999' if self.compress: hdu = fits.CompImageHDU(data=whtarr, header=prihdu.header) else: hdu = fits.PrimaryHDU(data=whtarr, header=prihdu.header) # Append remaining unique header keywords from template DQ # header to Primary header... if errhdr: for _card in errhdr.cards: if _card.keyword not in RESERVED_KEYS and _card.keyword not in hdu.header: hdu.header.append(_card) hdu.header['filename'] = self.outweight hdu.header['CCDCHIP'] = '-999' if self.wcs: pre_wcs_kw = self.find_kwupdate_location( hdu.header, 'CD1_1') # Update WCS Keywords based on PyDrizzle product's value # since 'drizzle' itself doesn't update that keyword. addWCSKeywords(self.wcs, hdu.header, blot=self.blot, single=self.single, after=pre_wcs_kw) # explicitly remove EXTNAME, EXTVER from header # since this header may have been used # to create a CompImageHDU instance instead of a PrimaryHDU instance for kw in ['EXTNAME', 'EXTVER', 'PCOUNT', 'GCOUNT']: hdu.header.pop(kw, None) hdu.header.set('filetype', 'WHT', before='TELESCOP', comment='Type of data in array') # Add primary header to output file... fwht.append(hdu) # remove all alternate WCS solutions from headers of this product wcs_functions.removeAllAltWCS(fwht, wcs_ext) if not virtual: print('Writing out image to disk:', self.outweight) fwht.writeto(self.outweight, overwrite=True) del fwht, hdu fwht = None # End 'if not virtual' outputFITS[self.outweight] = fwht # If a context image was specified, build a PyFITS object # for it as well... if self.outcontext and ctxarr is not None: fctx = fits.HDUList() # If there is only 1 plane, write it out as a 2-D extension if ctxarr.shape[0] == 1: _ctxarr = ctxarr[0] else: _ctxarr = ctxarr if self.compress: hdu = fits.CompImageHDU(data=_ctxarr, header=prihdu.header) else: hdu = fits.PrimaryHDU(data=_ctxarr, header=prihdu.header) # Append remaining unique header keywords from template DQ # header to Primary header... if dqhdr: for _card in dqhdr.cards: if ((_card.keyword not in RESERVED_KEYS) and _card.keyword not in hdu.header): hdu.header.append(_card) hdu.header['filename'] = self.outcontext if self.wcs: pre_wcs_kw = self.find_kwupdate_location( hdu.header, 'CD1_1') # Update WCS Keywords based on PyDrizzle product's value # since 'drizzle' itself doesn't update that keyword. addWCSKeywords(self.wcs, hdu.header, blot=self.blot, single=self.single, after=pre_wcs_kw) # explicitly remove EXTNAME, EXTVER from header # since this header may have been used # to create a CompImageHDU instance instead of a PrimaryHDU instance for kw in ['EXTNAME', 'EXTVER', 'PCOUNT', 'GCOUNT']: hdu.header.pop(kw, None) hdu.header.set('filetype', 'CTX', before='TELESCOP', comment='Type of data in array') fctx.append(hdu) # remove all alternate WCS solutions from headers of this product wcs_functions.removeAllAltWCS(fctx, wcs_ext) if not virtual: print('Writing out image to disk:', self.outcontext) fctx.writeto(self.outcontext, overwrite=True) del fctx, hdu fctx = None # End 'if not virtual' outputFITS[self.outcontext] = fctx return outputFITS
def writeSingleFITS(data, wcs, output, template, clobber=True, verbose=True, rules_file=None): """ Write out a simple FITS file given a numpy array and the name of another FITS file to use as a template for the output image header. """ outname, outextn = fileutil.parseFilename(output) outextname, outextver = fileutil.parseExtn(outextn) if fileutil.findFile(outname): if clobber: log.info('Deleting previous output product: %s' % outname) fileutil.removeFile(outname) else: log.warning('Output file %s already exists and overwrite not ' 'specified!' % outname) log.error('Quitting... Please remove before resuming operations.') raise IOError # Now update WCS keywords with values from provided WCS if hasattr(wcs.sip, 'a_order'): siphdr = True else: siphdr = False wcshdr = wcs.wcs2header(sip2hdr=siphdr) if template is not None: # Get default headers from multi-extension FITS file # If input data is not in MEF FITS format, it will return 'None' # NOTE: These are HEADER objects, not HDUs (prihdr, scihdr, errhdr, dqhdr), newtab = getTemplates(template, EXTLIST, rules_file=rules_file) if scihdr is None: scihdr = fits.Header() indx = 0 for c in prihdr.cards: if c.keyword not in ['INHERIT', 'EXPNAME']: indx += 1 else: break for i in range(indx, len(prihdr)): scihdr.append(prihdr.cards[i]) for i in range(indx, len(prihdr)): del prihdr[indx] else: scihdr = fits.Header() prihdr = fits.Header() # Start by updating PRIMARY header keywords... prihdr.set('EXTEND', value=True, after='NAXIS') prihdr['FILENAME'] = outname if outextname == '': outextname = 'sci' if outextver == 0: outextver = 1 scihdr['EXTNAME'] = outextname.upper() scihdr['EXTVER'] = outextver scihdr.update(wcshdr) # Create PyFITS HDUList for all extensions outhdu = fits.HDUList() # Setup primary header as an HDU ready for appending to output FITS file prihdu = fits.PrimaryHDU(header=prihdr) scihdu = fits.ImageHDU(header=scihdr, data=data) outhdu.append(prihdu) outhdu.append(scihdu) outhdu.writeto(outname) if verbose: print('Created output image: %s' % outname)
def buildMaskImage(rootname,bitvalue,output,extname='DQ',extver=1): """ Builds mask image from rootname's DQ array If there is no valid 'DQ' array in image, then return an empty string. """ # If no bitvalue is set or rootname given, assume no mask is desired # However, this name would be useful as the output mask from # other processing, such as MultiDrizzle, so return it anyway. #if bitvalue == None or rootname == None: # return None # build output name maskname = output # If an old version of the maskfile was present, remove it and rebuild it. if fileutil.findFile(maskname): fileutil.removeFile(maskname) # Open input file with DQ array fdq = fileutil.openImage(rootname,memmap=0,mode='readonly') try: _extn = fileutil.findExtname(fdq,extname,extver=extver) if _extn != None: # Read in DQ array dqarr = fdq[_extn].data else: dqarr = None # For the case where there is no DQ array, # create a mask image of all ones. if dqarr == None: # We need to get the dimensions of the output DQ array # Since the DQ array is non-existent, look for the SCI extension _sci_extn = fileutil.findExtname(fdq,'SCI',extver=extver) if _sci_extn != None: _shape = fdq[_sci_extn].data.shape dqarr = np.zeros(_shape,dtype=np.uint16) else: raise Exception # Build mask array from DQ array maskarr = buildMask(dqarr,bitvalue) #Write out the mask file as simple FITS file fmask = fits.open(maskname, 'append') maskhdu = fits.PrimaryHDU(data = maskarr) fmask.append(maskhdu) #Close files fmask.close() del fmask fdq.close() del fdq except: fdq.close() del fdq # Safeguard against leaving behind an incomplete file if fileutil.findFile(maskname): os.remove(maskname) _errstr = "\nWarning: Problem creating MASK file for "+rootname+".\n" #raise IOError, _errstr print(_errstr) return None # Return the name of the mask image written out return maskname
def process(inFile, force=False, newpath=None, inmemory=False, num_cores=None, headerlets=True): """ Run astrodrizzle on input file/ASN table using default values for astrodrizzle parameters. """ # We only need to import this package if a user run the task import drizzlepac from drizzlepac import processInput # used for creating new ASNs for _flc inputs import stwcs if headerlets: from stwcs.wcsutil import headerlet # Open the input file try: # Make sure given filename is complete and exists... inFilename = fileutil.buildRootname(inFile, ext=['.fits']) if not os.path.exists(inFilename): print("ERROR: Input file - %s - does not exist." % inFilename) return except TypeError: print("ERROR: Inappropriate input file.") return #If newpath was specified, move all files to that directory for processing if newpath: orig_processing_dir = os.getcwd() new_processing_dir = _createWorkingDir(newpath, inFilename) _copyToNewWorkingDir(new_processing_dir, inFilename) os.chdir(new_processing_dir) # Initialize for later use... _mname = None _new_asn = None _calfiles = [] # Check input file to see if [DRIZ/DITH]CORR is set to PERFORM if '_asn' in inFilename: # We are working with an ASN table. # Use asnutil code to extract filename inFilename = _lowerAsn(inFilename) _new_asn = [inFilename] _asndict = asnutil.readASNTable(inFilename, None, prodonly=False) _cal_prodname = _asndict['output'].lower() _fname = fileutil.buildRootname(_cal_prodname, ext=['_drz.fits']) # Retrieve the first member's rootname for possible use later _fimg = fits.open(inFilename) for name in _fimg[1].data.field('MEMNAME'): if name[-1] != '*': _mname = name.split('\0', 1)[0].lower() break _fimg.close() del _fimg else: # Check to see if input is a _RAW file # If it is, strip off the _raw.fits extension... _indx = inFilename.find('_raw') if _indx < 0: _indx = len(inFilename) # ... and build the CALXXX product rootname. _mname = fileutil.buildRootname(inFilename[:_indx]) _cal_prodname = inFilename[:_indx] # Reset inFilename to correspond to appropriate input for # drizzle: calibrated product name. inFilename = _mname if _mname == None: errorMsg = 'Could not find calibrated product!' raise Exception(errorMsg) # Create trailer filenames based on ASN output filename or # on input name for single exposures if '_raw' in inFile: # Output trailer file to RAW file's trailer _trlroot = inFile[:inFile.find('_raw')] elif '_asn' in inFile: # Output trailer file to ASN file's trailer, not product's trailer _trlroot = inFile[:inFile.find('_asn')] else: # Default: trim off last suffix of input filename # and replacing with .tra _indx = inFile.rfind('_') if _indx > 0: _trlroot = inFile[:_indx] else: _trlroot = inFile _trlfile = _trlroot + '.tra' # Open product and read keyword value # Check to see if product already exists... dkey = 'DRIZCORR' # ...if product does NOT exist, interrogate input file # to find out whether 'dcorr' has been set to PERFORM # Check if user wants to process again regardless of DRIZCORR keyword value if force: dcorr = 'PERFORM' else: if _mname: _fimg = fits.open(fileutil.buildRootname(_mname, ext=['_raw.fits'])) _phdr = _fimg['PRIMARY'].header if dkey in _phdr: dcorr = _phdr[dkey] else: dcorr = None _fimg.close() del _fimg else: dcorr = None time_str = _getTime() _tmptrl = _trlroot + '_tmp.tra' _drizfile = _trlroot + '_pydriz' _drizlog = _drizfile + ".log" # the '.log' gets added automatically by astrodrizzle if dcorr == 'PERFORM': if '_asn.fits' not in inFilename: # Working with a singleton # However, we always want to make sure we always use # a calibrated product as input, if available. _infile = fileutil.buildRootname(_cal_prodname) _infile_flc = fileutil.buildRootname(_cal_prodname, ext=['_flc.fits']) _cal_prodname = _infile _inlist = _calfiles = [_infile] # Add CTE corrected filename as additional input if present if os.path.exists(_infile_flc) and _infile_flc != _infile: _inlist.append(_infile_flc) else: # Working with an ASN table... _infile = inFilename flist, duplist = processInput.checkForDuplicateInputs( _asndict['order']) _calfiles = flist if len(duplist) > 0: origasn = processInput.changeSuffixinASN(inFilename, 'flt') dupasn = processInput.changeSuffixinASN(inFilename, 'flc') _inlist = [origasn, dupasn] else: _inlist = [_infile] # We want to keep the original specification of the calibration # product name, though, not a lower-case version... _cal_prodname = inFilename _new_asn.extend(_inlist) # kept so we can delete it when finished # Run astrodrizzle and send its processing statements to _trlfile _pyver = drizzlepac.astrodrizzle.__version__ for _infile in _inlist: # Run astrodrizzle for all inputs # Create trailer marker message for start of astrodrizzle processing _trlmsg = _timestamp('astrodrizzle started ') _trlmsg = _trlmsg + __trlmarker__ _trlmsg = _trlmsg + '%s: Processing %s with astrodrizzle Version %s\n' % ( time_str, _infile, _pyver) print(_trlmsg) # Write out trailer comments to trailer file... ftmp = open(_tmptrl, 'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile, _tmptrl) _pyd_err = _trlroot + '_pydriz.stderr' try: b = drizzlepac.astrodrizzle.AstroDrizzle(input=_infile, runfile=_drizfile, configobj='defaults', in_memory=inmemory, num_cores=num_cores, **pipeline_pars) except Exception as errorobj: _appendTrlFile(_trlfile, _drizlog) _appendTrlFile(_trlfile, _pyd_err) _ftrl = open(_trlfile, 'a') _ftrl.write( 'ERROR: Could not complete astrodrizzle processing of %s.\n' % _infile) _ftrl.write(str(sys.exc_info()[0]) + ': ') _ftrl.writelines(str(errorobj)) _ftrl.write('\n') _ftrl.close() print( 'ERROR: Could not complete astrodrizzle processing of %s.' % _infile) raise Exception(str(errorobj)) # Now, append comments created by PyDrizzle to CALXXX trailer file print('Updating trailer file %s with astrodrizzle comments.' % _trlfile) _appendTrlFile(_trlfile, _drizlog) # Save this for when astropy.io.fits can modify a file 'in-place' # Update calibration switch _fimg = fits.open(_cal_prodname, mode='update') _fimg['PRIMARY'].header[dkey] = 'COMPLETE' _fimg.close() del _fimg # Enforce pipeline convention of all lower-case product # names _prodlist = glob.glob('*drz.fits') for _prodname in _prodlist: _plower = _prodname.lower() if _prodname != _plower: os.rename(_prodname, _plower) else: # Create default trailer file messages when astrodrizzle is not # run on a file. This will typically apply only to BIAS,DARK # and other reference images. # Start by building up the message... _trlmsg = _timestamp('astrodrizzle skipped ') _trlmsg = _trlmsg + __trlmarker__ _trlmsg = _trlmsg + '%s: astrodrizzle processing not requested for %s.\n' % ( time_str, inFilename) _trlmsg = _trlmsg + ' astrodrizzle will not be run at this time.\n' print(_trlmsg) # Write message out to temp file and append it to full trailer file ftmp = open(_tmptrl, 'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile, _tmptrl) _fmsg = None # Append final timestamp to trailer file... _final_msg = '%s: Finished processing %s \n' % (time_str, inFilename) _final_msg += _timestamp('astrodrizzle completed ') _trlmsg += _final_msg ftmp = open(_tmptrl, 'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile, _tmptrl) # If we created a new ASN table, we need to remove it if _new_asn != None: for _name in _new_asn: fileutil.removeFile(_name) # Clean up any generated OrIg_files directory if os.path.exists("OrIg_files"): # check to see whether this directory is empty flist = glob.glob('OrIg_files/*.fits') if len(flist) == 0: os.rmdir("OrIg_files") else: print( 'OrIg_files directory NOT removed as it still contained images...' ) if headerlets: # Generate headerlets for each updated FLT image hlet_msg = _timestamp("Writing Headerlets started") for fname in _calfiles: frootname = fileutil.buildNewRootname(fname) hname = "%s_flt_hlet.fits" % frootname hlet_msg += "Created Headerlet file %s \n" % hname try: headerlet.write_headerlet( fname, 'OPUS', output='flt', wcskey='PRIMARY', author="OPUS", descrip="Default WCS from Pipeline Calibration", attach=False, clobber=True, logging=False) except ValueError: hlet_msg += _timestamp( "SKIPPED: Headerlet not created for %s \n" % fname) # update trailer file to log creation of headerlet files hlet_msg += _timestamp("Writing Headerlets completed") ftrl = open(_trlfile, 'a') ftrl.write(hlet_msg) ftrl.close() # If processing was done in a temp working dir, restore results to original # processing directory, return to original working dir and remove temp dir if newpath: _restoreResults(new_processing_dir, orig_processing_dir) os.chdir(orig_processing_dir) _removeWorkingDir(new_processing_dir) # Provide feedback to user print(_final_msg)
def buildMaskImage(rootname, bitvalue, output, extname='DQ', extver=1): """ Builds mask image from rootname's DQ array If there is no valid 'DQ' array in image, then return an empty string. """ # If no bitvalue is set or rootname given, assume no mask is desired # However, this name would be useful as the output mask from # other processing, such as MultiDrizzle, so return it anyway. #if bitvalue == None or rootname == None: # return None # build output name maskname = output # If an old version of the maskfile was present, remove it and rebuild it. if fileutil.findFile(maskname): fileutil.removeFile(maskname) # Open input file with DQ array fdq = fileutil.openImage(rootname, mode='readonly', memmap=False) try: _extn = fileutil.findExtname(fdq, extname, extver=extver) if _extn is not None: # Read in DQ array dqarr = fdq[_extn].data else: dqarr = None # For the case where there is no DQ array, # create a mask image of all ones. if dqarr is None: # We need to get the dimensions of the output DQ array # Since the DQ array is non-existent, look for the SCI extension _sci_extn = fileutil.findExtname(fdq, 'SCI', extver=extver) if _sci_extn is not None: _shape = fdq[_sci_extn].data.shape dqarr = np.zeros(_shape, dtype=np.uint16) else: raise Exception # Build mask array from DQ array maskarr = buildMask(dqarr, bitvalue) #Write out the mask file as simple FITS file fmask = fits.open(maskname, mode='append', memmap=False) maskhdu = fits.PrimaryHDU(data=maskarr) fmask.append(maskhdu) #Close files fmask.close() del fmask fdq.close() del fdq except: fdq.close() del fdq # Safeguard against leaving behind an incomplete file if fileutil.findFile(maskname): os.remove(maskname) _errstr = "\nWarning: Problem creating MASK file for " + rootname + ".\n" #raise IOError, _errstr print(_errstr) return None # Return the name of the mask image written out return maskname
def buildShadowMaskImage(dqfile,detnum,extnum,maskname,bitvalue=None,binned=1): """ Builds mask image from WFPC2 shadow calibrations. detnum - string value for 'DETECTOR' detector """ # insure detnum is a string if type(detnum) != type(''): detnum = repr(detnum) _funcroot = '_func_Shadow_WF' # build template shadow mask's filename # If an old version of the maskfile was present, remove it and rebuild it. if fileutil.findFile(maskname): fileutil.removeFile(maskname) _use_inmask = False if fileutil.findFile(dqfile) != True or bitvalue == None: _use_inmask = True # Check for existance of input .c1h file for use in making inmask file if fileutil.findFile(dqfile) != True or bitvalue is None: #_mask = 'wfpc2_inmask'+detnum+'.fits' _mask = maskname # Check to see if file exists... if _use_inmask and not fileutil.findFile(_mask): # If not, create the file. # This takes a long time to run, so it should be done # only when absolutely necessary... try: _funcx = _funcroot+detnum+'x' _funcy = _funcroot+detnum+'y' _xarr = np.clip(np.fromfunction(eval(_funcx),(800,800)),0.0,1.0).astype(np.uint8) _yarr = np.clip(np.fromfunction(eval(_funcy),(800,800)),0.0,1.0).astype(np.uint8) maskarr = _xarr * _yarr if binned !=1: bmaskarr = maskarr[::2,::2] bmaskarr *= maskarr[1::2,::2] bmaskarr *= maskarr[::2,1::2] bmaskarr *= maskarr[1::2,1::2] maskarr = bmaskarr.copy() del bmaskarr #Write out the mask file as simple FITS file fmask = fits.open(_mask,'append') maskhdu = fits.PrimaryHDU(data=maskarr) fmask.append(maskhdu) #Close files fmask.close() del fmask except: return None else: # # Build full mask based on .c1h and shadow mask # fdq = fileutil.openImage(dqfile) #fsmask = fits.open(_mask,memmap=1,mode='readonly') try: # Read in DQ array from .c1h and from shadow mask files dqarr = fdq[int(extnum)].data #maskarr = fsmask[0].data # Build mask array from DQ array dqmaskarr = buildMask(dqarr,bitvalue) #Write out the mask file as simple FITS file fdqmask = fits.open(maskname,'append') maskhdu = fits.PrimaryHDU(data=dqmaskarr) fdqmask.append(maskhdu) #Close files fdqmask.close() del fdqmask fdq.close() del fdq except: fdq.close() del fdq # Safeguard against leaving behind an incomplete file if fileutil.findFile(maskname): os.remove(maskname) _errstr = "\nWarning: Problem creating DQMASK file for "+rootname+".\n" #raise IOError, _errstr print(_errstr) return None # Return the name of the mask image written out return maskname
def writeSingleFITS(data,wcs,output,template,clobber=True,verbose=True): """ Write out a simple FITS file given a numpy array and the name of another FITS file to use as a template for the output image header. """ outname,outextn = fileutil.parseFilename(output) outextname,outextver = fileutil.parseExtn(outextn) if fileutil.findFile(outname): if clobber: log.info('Deleting previous output product: %s' % outname) fileutil.removeFile(outname) else: log.warning('Output file %s already exists and overwrite not ' 'specified!' % outname) log.error('Quitting... Please remove before resuming operations.') raise IOError # Now update WCS keywords with values from provided WCS if hasattr(wcs.sip,'a_order'): siphdr = True else: siphdr = False wcshdr = wcs.wcs2header(sip2hdr=siphdr) if template is not None: # Get default headers from multi-extension FITS file # If input data is not in MEF FITS format, it will return 'None' # NOTE: These are HEADER objects, not HDUs (prihdr,scihdr,errhdr,dqhdr),newtab = getTemplates(template,EXTLIST) if scihdr is None: scihdr = fits.Header() indx = 0 for c in prihdr.cards: if c.keyword not in ['INHERIT','EXPNAME']: indx += 1 else: break for i in range(indx,len(prihdr)): scihdr.append(prihdr.cards[i]) for i in range(indx, len(prihdr)): del prihdr[indx] else: scihdr = fits.Header() prihdr = fits.Header() # Start by updating PRIMARY header keywords... prihdr.set('EXTEND', value=True, after='NAXIS') prihdr['FILENAME'] = outname if outextname == '': outextname = 'sci' if outextver == 0: outextver = 1 scihdr['EXTNAME'] = outextname.upper() scihdr['EXTVER'] = outextver for card in wcshdr.cards: scihdr[card.keyword] = (card.value, card.comment) # Create PyFITS HDUList for all extensions outhdu = fits.HDUList() # Setup primary header as an HDU ready for appending to output FITS file prihdu = fits.PrimaryHDU(header=prihdr) scihdu = fits.ImageHDU(header=scihdr,data=data) outhdu.append(prihdu) outhdu.append(scihdu) outhdu.writeto(outname) if verbose: print('Created output image: %s' % outname)
def buildShadowMaskImage(dqfile, detnum, extnum, maskname, bitvalue=None, binned=1): """ Builds mask image from WFPC2 shadow calibrations. detnum - string value for 'DETECTOR' detector """ # insure detnum is a string if type(detnum) != type(''): detnum = repr(detnum) _funcroot = '_func_Shadow_WF' # build template shadow mask's filename # If an old version of the maskfile was present, remove it and rebuild it. if fileutil.findFile(maskname): fileutil.removeFile(maskname) _use_inmask = not fileutil.findFile(dqfile) or bitvalue is None # Check for existance of input .c1h file for use in making inmask file if _use_inmask: #_mask = 'wfpc2_inmask'+detnum+'.fits' _mask = maskname # Check to see if file exists... if not fileutil.findFile(_mask): # If not, create the file. # This takes a long time to run, so it should be done # only when absolutely necessary... try: _funcx = _funcroot + detnum + 'x' _funcy = _funcroot + detnum + 'y' _xarr = np.clip(np.fromfunction(eval(_funcx), (800, 800)), 0.0, 1.0).astype(np.uint8) _yarr = np.clip(np.fromfunction(eval(_funcy), (800, 800)), 0.0, 1.0).astype(np.uint8) maskarr = _xarr * _yarr if binned != 1: bmaskarr = maskarr[::2, ::2] bmaskarr *= maskarr[1::2, ::2] bmaskarr *= maskarr[::2, 1::2] bmaskarr *= maskarr[1::2, 1::2] maskarr = bmaskarr.copy() del bmaskarr #Write out the mask file as simple FITS file fmask = fits.open(_mask, mode='append', memmap=False) maskhdu = fits.PrimaryHDU(data=maskarr) fmask.append(maskhdu) #Close files fmask.close() del fmask except: return None else: # # Build full mask based on .c1h and shadow mask # fdq = fileutil.openImage(dqfile, mode='readonly', memmap=False) try: # Read in DQ array from .c1h and from shadow mask files dqarr = fdq[int(extnum)].data #maskarr = fsmask[0].data # Build mask array from DQ array dqmaskarr = buildMask(dqarr, bitvalue) #Write out the mask file as simple FITS file fdqmask = fits.open(maskname, mode='append', memmap=False) maskhdu = fits.PrimaryHDU(data=dqmaskarr) fdqmask.append(maskhdu) #Close files fdqmask.close() del fdqmask fdq.close() del fdq except: fdq.close() del fdq # Safeguard against leaving behind an incomplete file if fileutil.findFile(maskname): os.remove(maskname) _errstr = "\nWarning: Problem creating DQMASK file for " + rootname + ".\n" #raise IOError, _errstr print(_errstr) return None # Return the name of the mask image written out return maskname
def writeFITS(self, template, sciarr, whtarr, ctxarr=None, versions=None, overwrite=yes, blend=True, virtual=False): """ Generate PyFITS objects for each output extension using the file given by 'template' for populating headers. The arrays will have the size specified by 'shape'. """ if not isinstance(template, list): template = [template] if fileutil.findFile(self.output): if overwrite: log.info('Deleting previous output product: %s' % self.output) fileutil.removeFile(self.output) else: log.warning('Output file %s already exists and overwrite not ' 'specified!' % self.output) log.error('Quitting... Please remove before resuming ' 'operations.') raise IOError # initialize output value for this method outputFITS = {} # Default value for NEXTEND when 'build'== True nextend = 3 if not self.build: nextend = 0 if self.outweight: if overwrite: if fileutil.findFile(self.outweight): log.info('Deleting previous output WHT product: %s' % self.outweight) fileutil.removeFile(self.outweight) else: log.warning('Output file %s already exists and overwrite ' 'not specified!' % self.outweight) log.error('Quitting... Please remove before resuming ' 'operations.') raise IOError if self.outcontext: if overwrite: if fileutil.findFile(self.outcontext): log.info('Deleting previous output CTX product: %s' % self.outcontext) fileutil.removeFile(self.outcontext) else: log.warning('Output file %s already exists and overwrite ' 'not specified!' % self.outcontext) log.error('Quitting... Please remove before resuming ' 'operations.') raise IOError # Get default headers from multi-extension FITS file # If only writing out single drizzle product, blending needs to be # forced off as there is only 1 input to report, no blending needed if self.single: blend=False # If input data is not in MEF FITS format, it will return 'None' # and those headers will have to be generated from drizzle output # file FITS headers. # NOTE: These are HEADER objects, not HDUs #prihdr,scihdr,errhdr,dqhdr = getTemplates(template) self.fullhdrs, intab = getTemplates(template, blend=False) newhdrs, newtab = getTemplates(template,blend=blend) if newtab is not None: nextend += 1 # account for new table extn prihdr = newhdrs[0] scihdr = newhdrs[1] errhdr = newhdrs[2] dqhdr = newhdrs[3] # Setup primary header as an HDU ready for appending to output FITS file prihdu = fits.PrimaryHDU(header=prihdr, data=None) # Start by updating PRIMARY header keywords... prihdu.header.set('EXTEND', value=True, after='NAXIS') prihdu.header['NEXTEND'] = nextend prihdu.header['FILENAME'] = self.output prihdu.header['PROD_VER'] = 'DrizzlePac {}'.format(version.__version__) # Update the ROOTNAME with the new value as well _indx = self.output.find('_drz') if _indx < 0: rootname_val = self.output else: rootname_val = self.output[:_indx] prihdu.header['ROOTNAME'] = rootname_val # Get the total exposure time for the image # If not calculated by PyDrizzle and passed through # the pardict, then leave value from the template image. if self.texptime: prihdu.header['EXPTIME'] = self.texptime prihdu.header.set('TEXPTIME', value=self.texptime, after='EXPTIME') prihdu.header['EXPSTART'] = self.expstart prihdu.header['EXPEND'] = self.expend #Update ASN_MTYPE to reflect the fact that this is a product # Currently hard-wired to always output 'PROD-DTH' as MTYPE prihdu.header['ASN_MTYP'] = 'PROD-DTH' # Update DITHCORR calibration keyword if present # Remove when we can modify FITS headers in place... if 'DRIZCORR' in prihdu.header: prihdu.header['DRIZCORR'] = 'COMPLETE' if 'DITHCORR' in prihdu.header: prihdu.header['DITHCORR'] = 'COMPLETE' prihdu.header['NDRIZIM'] =(len(self.parlist), 'Drizzle, No. images drizzled onto output') # Only a subset of these keywords makes sense for the new WCS based # transformations. They need to be reviewed to decide what to keep # and what to leave out. if not self.blot: self.addDrizKeywords(prihdu.header,versions) if scihdr: try: del scihdr['OBJECT'] except KeyError: pass if 'CCDCHIP' in scihdr: scihdr['CCDCHIP'] = '-999' if 'NCOMBINE' in scihdr: scihdr['NCOMBINE'] = self.parlist[0]['nimages'] # If BUNIT keyword was found and reset, then bunit_last_kw = self.find_kwupdate_location(scihdr,'bunit') if self.bunit is not None: comment_str = "Units of science product" if self.bunit.lower()[:5] == 'count': comment_str = "counts * gain = electrons" scihdr.set('BUNIT', value=self.bunit, comment=comment_str, after=bunit_last_kw) else: # check to see whether to update already present BUNIT comment if 'bunit' in scihdr and scihdr['bunit'].lower()[:5] == 'count': comment_str = "counts * gain = electrons" scihdr.set('BUNIT', value=scihdr['bunit'], comment=comment_str, after=bunit_last_kw) # Add WCS keywords to SCI header if self.wcs: pre_wcs_kw = self.find_kwupdate_location(scihdr,'CD1_1') addWCSKeywords(self.wcs,scihdr,blot=self.blot, single=self.single, after=pre_wcs_kw) # Recompute this after removing distortion kws pre_wcs_kw = self.find_kwupdate_location(scihdr,'CD1_1') ########## # Now, build the output file ########## if self.build: print('-Generating multi-extension output file: ',self.output) fo = fits.HDUList() # Add primary header to output file... fo.append(prihdu) if self.single and self.compress: hdu = fits.CompImageHDU(data=sciarr, header=scihdr, name=EXTLIST[0]) else: hdu = fits.ImageHDU(data=sciarr, header=scihdr, name=EXTLIST[0]) last_kw = self.find_kwupdate_location(scihdr,'EXTNAME') hdu.header.set('EXTNAME', value='SCI', after=last_kw) hdu.header.set('EXTVER', value=1, after='EXTNAME') fo.append(hdu) # Build WHT extension here, if requested... if errhdr: errhdr['CCDCHIP'] = '-999' if self.single and self.compress: hdu = fits.CompImageHDU(data=whtarr, header=errhdr, name=EXTLIST[1]) else: hdu = fits.ImageHDU(data=whtarr, header=errhdr, name=EXTLIST[1]) last_kw = self.find_kwupdate_location(errhdr,'EXTNAME') hdu.header.set('EXTNAME', value='WHT', after=last_kw) hdu.header.set('EXTVER', value=1, after='EXTNAME') if self.wcs: pre_wcs_kw = self.find_kwupdate_location(hdu.header,'CD1_1') # Update WCS Keywords based on PyDrizzle product's value # since 'drizzle' itself doesn't update that keyword. addWCSKeywords(self.wcs,hdu.header,blot=self.blot, single=self.single, after=pre_wcs_kw) fo.append(hdu) # Build CTX extension here # If there is only 1 plane, write it out as a 2-D extension if self.outcontext: if ctxarr.shape[0] == 1: _ctxarr = ctxarr[0] else: _ctxarr = ctxarr else: _ctxarr = None if self.single and self.compress: hdu = fits.CompImageHDU(data=_ctxarr, header=dqhdr, name=EXTLIST[2]) else: hdu = fits.ImageHDU(data=_ctxarr, header=dqhdr, name=EXTLIST[2]) last_kw = self.find_kwupdate_location(dqhdr,'EXTNAME') hdu.header.set('EXTNAME', value='CTX', after=last_kw) hdu.header.set('EXTVER', value=1, after='EXTNAME') if self.wcs: pre_wcs_kw = self.find_kwupdate_location(hdu.header,'CD1_1') # Update WCS Keywords based on PyDrizzle product's value # since 'drizzle' itself doesn't update that keyword. addWCSKeywords(self.wcs,hdu.header,blot=self.blot, single=self.single, after=pre_wcs_kw) fo.append(hdu) # remove all alternate WCS solutions from headers of this product wcs_functions.removeAllAltWCS(fo,[1]) # add table of combined header keyword values to FITS file if newtab is not None: fo.append(newtab) if not virtual: print('Writing out to disk:',self.output) # write out file to disk fo.writeto(self.output) fo.close() del fo, hdu fo = None # End 'if not virtual' outputFITS[self.output]= fo else: print('-Generating simple FITS output: %s' % self.outdata) fo = fits.HDUList() hdu_header = prihdu.header.copy() del hdu_header['nextend'] # Append remaining unique header keywords from template DQ # header to Primary header... if scihdr: for _card in scihdr.cards: if _card.keyword not in RESERVED_KEYS and _card.keyword not in hdu_header: hdu_header.append(_card) for kw in ['PCOUNT', 'GCOUNT']: try: del kw except KeyError: pass hdu_header['filename'] = self.outdata if self.compress: hdu = fits.CompImageHDU(data=sciarr, header=hdu_header) wcs_ext = [1] else: hdu = fits.ImageHDU(data=sciarr, header=hdu_header) wcs_ext = [0] # explicitly set EXTEND to FALSE for simple FITS files. dim = len(sciarr.shape) hdu.header.set('extend', value=False, after='NAXIS%s'%dim) # Add primary header to output file... fo.append(hdu) # remove all alternate WCS solutions from headers of this product logutil.logging.disable(logutil.logging.INFO) wcs_functions.removeAllAltWCS(fo,wcs_ext) logutil.logging.disable(logutil.logging.NOTSET) # add table of combined header keyword values to FITS file if newtab is not None: fo.append(newtab) if not virtual: print('Writing out image to disk:',self.outdata) # write out file to disk fo.writeto(self.outdata) del fo,hdu fo = None # End 'if not virtual' outputFITS[self.outdata]= fo if self.outweight and whtarr is not None: # We need to build new PyFITS objects for each WHT array fwht = fits.HDUList() if errhdr: errhdr['CCDCHIP'] = '-999' if self.compress: hdu = fits.CompImageHDU(data=whtarr, header=prihdu.header) else: hdu = fits.ImageHDU(data=whtarr, header=prihdu.header) # Append remaining unique header keywords from template DQ # header to Primary header... if errhdr: for _card in errhdr.cards: if _card.keyword not in RESERVED_KEYS and _card.keyword not in hdu.header: hdu.header.append(_card) hdu.header['filename'] = self.outweight hdu.header['CCDCHIP'] = '-999' if self.wcs: pre_wcs_kw = self.find_kwupdate_location(hdu.header,'CD1_1') # Update WCS Keywords based on PyDrizzle product's value # since 'drizzle' itself doesn't update that keyword. addWCSKeywords(self.wcs,hdu.header, blot=self.blot, single=self.single, after=pre_wcs_kw) # Add primary header to output file... fwht.append(hdu) # remove all alternate WCS solutions from headers of this product wcs_functions.removeAllAltWCS(fwht,wcs_ext) if not virtual: print('Writing out image to disk:',self.outweight) fwht.writeto(self.outweight) del fwht,hdu fwht = None # End 'if not virtual' outputFITS[self.outweight]= fwht # If a context image was specified, build a PyFITS object # for it as well... if self.outcontext and ctxarr is not None: fctx = fits.HDUList() # If there is only 1 plane, write it out as a 2-D extension if ctxarr.shape[0] == 1: _ctxarr = ctxarr[0] else: _ctxarr = ctxarr if self.compress: hdu = fits.CompImageHDU(data=_ctxarr, header=prihdu.header) else: hdu = fits.ImageHDU(data=_ctxarr, header=prihdu.header) # Append remaining unique header keywords from template DQ # header to Primary header... if dqhdr: for _card in dqhdr.cards: if ( (_card.keyword not in RESERVED_KEYS) and _card.keyword not in hdu.header): hdu.header.append(_card) hdu.header['filename'] = self.outcontext if self.wcs: pre_wcs_kw = self.find_kwupdate_location(hdu.header,'CD1_1') # Update WCS Keywords based on PyDrizzle product's value # since 'drizzle' itself doesn't update that keyword. addWCSKeywords(self.wcs,hdu.header, blot=self.blot, single=self.single, after=pre_wcs_kw) fctx.append(hdu) # remove all alternate WCS solutions from headers of this product wcs_functions.removeAllAltWCS(fctx,wcs_ext) if not virtual: print('Writing out image to disk:',self.outcontext) fctx.writeto(self.outcontext) del fctx,hdu fctx = None # End 'if not virtual' outputFITS[self.outcontext]= fctx return outputFITS
def process(inFile,force=False,newpath=None, inmemory=False, num_cores=None, headerlets=True): """ Run astrodrizzle on input file/ASN table using default values for astrodrizzle parameters. """ # We only need to import this package if a user run the task import drizzlepac from drizzlepac import processInput # used for creating new ASNs for _flc inputs import stwcs if headerlets: from stwcs.wcsutil import headerlet # Open the input file try: # Make sure given filename is complete and exists... inFilename = fileutil.buildRootname(inFile,ext=['.fits']) if not os.path.exists(inFilename): print("ERROR: Input file - %s - does not exist." % inFilename) return except TypeError: print("ERROR: Inappropriate input file.") return #If newpath was specified, move all files to that directory for processing if newpath: orig_processing_dir = os.getcwd() new_processing_dir = _createWorkingDir(newpath,inFilename) _copyToNewWorkingDir(new_processing_dir,inFilename) os.chdir(new_processing_dir) # Initialize for later use... _mname = None _new_asn = None _calfiles = [] # Identify WFPC2 inputs to account for differences in WFPC2 inputs wfpc2_input = fits.getval(inFilename, 'instrume') == 'WFPC2' cal_ext = None # Check input file to see if [DRIZ/DITH]CORR is set to PERFORM if '_asn' in inFilename: # We are working with an ASN table. # Use asnutil code to extract filename inFilename = _lowerAsn(inFilename) _new_asn = [inFilename] _asndict = asnutil.readASNTable(inFilename,None,prodonly=False) _cal_prodname = _asndict['output'].lower() _fname = fileutil.buildRootname(_cal_prodname,ext=['_drz.fits']) # Retrieve the first member's rootname for possible use later _fimg = fits.open(inFilename, memmap=False) for name in _fimg[1].data.field('MEMNAME'): if name[-1] != '*': _mname = name.split('\0', 1)[0].lower() break _fimg.close() del _fimg else: # Check to see if input is a _RAW file # If it is, strip off the _raw.fits extension... _indx = inFilename.find('_raw') if _indx < 0: _indx = len(inFilename) # ... and build the CALXXX product rootname. if wfpc2_input: # force code to define _c0m file as calibrated product to be used cal_ext = ['_c0m.fits'] _mname = fileutil.buildRootname(inFilename[:_indx], ext=cal_ext) _cal_prodname = inFilename[:_indx] # Reset inFilename to correspond to appropriate input for # drizzle: calibrated product name. inFilename = _mname if _mname is None: errorMsg = 'Could not find calibrated product!' raise Exception(errorMsg) # Create trailer filenames based on ASN output filename or # on input name for single exposures if '_raw' in inFile: # Output trailer file to RAW file's trailer _trlroot = inFile[:inFile.find('_raw')] elif '_asn' in inFile: # Output trailer file to ASN file's trailer, not product's trailer _trlroot = inFile[:inFile.find('_asn')] else: # Default: trim off last suffix of input filename # and replacing with .tra _indx = inFile.rfind('_') if _indx > 0: _trlroot = inFile[:_indx] else: _trlroot = inFile _trlfile = _trlroot + '.tra' # Open product and read keyword value # Check to see if product already exists... dkey = 'DRIZCORR' # ...if product does NOT exist, interrogate input file # to find out whether 'dcorr' has been set to PERFORM # Check if user wants to process again regardless of DRIZCORR keyword value if force: dcorr = 'PERFORM' else: if _mname : _fimg = fits.open(fileutil.buildRootname(_mname,ext=['_raw.fits']), memmap=False) _phdr = _fimg['PRIMARY'].header if dkey in _phdr: dcorr = _phdr[dkey] else: dcorr = None _fimg.close() del _fimg else: dcorr = None time_str = _getTime() _tmptrl = _trlroot + '_tmp.tra' _drizfile = _trlroot + '_pydriz' _drizlog = _drizfile + ".log" # the '.log' gets added automatically by astrodrizzle if dcorr == 'PERFORM': if '_asn.fits' not in inFilename: # Working with a singleton # However, we always want to make sure we always use # a calibrated product as input, if available. _infile = fileutil.buildRootname(_cal_prodname, ext=cal_ext) _infile_flc = fileutil.buildRootname(_cal_prodname,ext=['_flc.fits']) _cal_prodname = _infile _inlist = _calfiles = [_infile] # Add CTE corrected filename as additional input if present if os.path.exists(_infile_flc) and _infile_flc != _infile: _inlist.append(_infile_flc) else: # Working with an ASN table... _infile = inFilename flist,duplist = processInput.checkForDuplicateInputs(_asndict['order']) _calfiles = flist if len(duplist) > 0: origasn = processInput.changeSuffixinASN(inFilename,'flt') dupasn = processInput.changeSuffixinASN(inFilename,'flc') _inlist = [origasn,dupasn] else: _inlist = [_infile] # We want to keep the original specification of the calibration # product name, though, not a lower-case version... _cal_prodname = inFilename _new_asn.extend(_inlist) # kept so we can delete it when finished # Run astrodrizzle and send its processing statements to _trlfile _pyver = drizzlepac.astrodrizzle.__version__ for _infile in _inlist: # Run astrodrizzle for all inputs # Create trailer marker message for start of astrodrizzle processing _trlmsg = _timestamp('astrodrizzle started ') _trlmsg = _trlmsg+ __trlmarker__ _trlmsg = _trlmsg + '%s: Processing %s with astrodrizzle Version %s\n' % (time_str,_infile,_pyver) print(_trlmsg) # Write out trailer comments to trailer file... ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) _pyd_err = _trlroot+'_pydriz.stderr' try: b = drizzlepac.astrodrizzle.AstroDrizzle(input=_infile,runfile=_drizfile, configobj='defaults',in_memory=inmemory, num_cores=num_cores, **pipeline_pars) except Exception as errorobj: _appendTrlFile(_trlfile,_drizlog) _appendTrlFile(_trlfile,_pyd_err) _ftrl = open(_trlfile,'a') _ftrl.write('ERROR: Could not complete astrodrizzle processing of %s.\n' % _infile) _ftrl.write(str(sys.exc_info()[0])+': ') _ftrl.writelines(str(errorobj)) _ftrl.write('\n') _ftrl.close() print('ERROR: Could not complete astrodrizzle processing of %s.' % _infile) raise Exception(str(errorobj)) # Now, append comments created by PyDrizzle to CALXXX trailer file print('Updating trailer file %s with astrodrizzle comments.' % _trlfile) _appendTrlFile(_trlfile,_drizlog) # Save this for when astropy.io.fits can modify a file 'in-place' # Update calibration switch _fimg = fits.open(_cal_prodname, mode='update', memmap=False) _fimg['PRIMARY'].header[dkey] = 'COMPLETE' _fimg.close() del _fimg # Enforce pipeline convention of all lower-case product # names _prodlist = glob.glob('*drz.fits') for _prodname in _prodlist: _plower = _prodname.lower() if _prodname != _plower: os.rename(_prodname,_plower) else: # Create default trailer file messages when astrodrizzle is not # run on a file. This will typically apply only to BIAS,DARK # and other reference images. # Start by building up the message... _trlmsg = _timestamp('astrodrizzle skipped ') _trlmsg = _trlmsg + __trlmarker__ _trlmsg = _trlmsg + '%s: astrodrizzle processing not requested for %s.\n' % (time_str,inFilename) _trlmsg = _trlmsg + ' astrodrizzle will not be run at this time.\n' print(_trlmsg) # Write message out to temp file and append it to full trailer file ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) _fmsg = None # Append final timestamp to trailer file... _final_msg = '%s: Finished processing %s \n' % (time_str,inFilename) _final_msg += _timestamp('astrodrizzle completed ') _trlmsg += _final_msg ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) # If we created a new ASN table, we need to remove it if _new_asn is not None: for _name in _new_asn: fileutil.removeFile(_name) # Clean up any generated OrIg_files directory if os.path.exists("OrIg_files"): # check to see whether this directory is empty flist = glob.glob('OrIg_files/*.fits') if len(flist) == 0: os.rmdir("OrIg_files") else: print('OrIg_files directory NOT removed as it still contained images...') if headerlets: # Generate headerlets for each updated FLT image hlet_msg = _timestamp("Writing Headerlets started") for fname in _calfiles: frootname = fileutil.buildNewRootname(fname) hname = "%s_flt_hlet.fits"%frootname hlet_msg += "Created Headerlet file %s \n"%hname try: headerlet.write_headerlet(fname,'OPUS',output='flt', wcskey='PRIMARY', author="OPUS",descrip="Default WCS from Pipeline Calibration", attach=False,clobber=True,logging=False) except ValueError: hlet_msg += _timestamp("SKIPPED: Headerlet not created for %s \n"%fname) # update trailer file to log creation of headerlet files hlet_msg += _timestamp("Writing Headerlets completed") ftrl = open(_trlfile,'a') ftrl.write(hlet_msg) ftrl.close() # If processing was done in a temp working dir, restore results to original # processing directory, return to original working dir and remove temp dir if newpath: _restoreResults(new_processing_dir,orig_processing_dir) os.chdir(orig_processing_dir) _removeWorkingDir(new_processing_dir) # Provide feedback to user print(_final_msg)
def buildShadowMaskImage(rootname,detnum,extnum,maskname,replace=yes,bitvalue=None,binned=1): """ Builds mask image from WFPC2 shadow calibrations. detnum - string value for 'DETECTOR' detector """ # insure detnum is a string if type(detnum) != type(''): detnum = repr(detnum) _funcroot = '_func_Shadow_WF' # build template shadow mask's filename _mask = 'wfpc2_inmask'+detnum+'.fits' """ if rootname != None: maskname = buildMaskName(fileutil.buildNewRootname(rootname),detnum) else: maskname = None """ # If an old version of the maskfile was present, remove it and rebuild it. if fileutil.findFile(maskname) and replace: fileutil.removeFile(maskname) # Read in info from .c1h file to add flagged bad pixels to final mask _indx = rootname.find('.c1h') if _indx < 0: _indx = len(rootname) if rootname.find('.fits') < 0: _dqname = rootname[:_indx]+'.c1h' else: _dqname = rootname _use_inmask = False if fileutil.findFile(_dqname) != yes or bitvalue == None: _use_inmask = True # Check to see if file exists... if _use_inmask and not fileutil.findFile(_mask): # If not, create the file. # This takes a long time to run, so it should be done # only when absolutely necessary... try: _funcx = _funcroot+detnum+'x' _funcy = _funcroot+detnum+'y' _xarr = np.clip(np.fromfunction(eval(_funcx),(800,800)),0.0,1.0).astype(np.uint8) _yarr = np.clip(np.fromfunction(eval(_funcy),(800,800)),0.0,1.0).astype(np.uint8) maskarr = _xarr * _yarr if binned !=1: print('in buildmask', binned) bmaskarr = maskarr[::2,::2] bmaskarr *= maskarr[1::2,::2] bmaskarr *= maskarr[::2,1::2] bmaskarr *= maskarr[1::2,1::2] maskarr = bmaskarr.copy() del bmaskarr #Write out the mask file as simple FITS file fmask = pyfits.open(_mask,'append') maskhdu = pyfits.PrimaryHDU(data=maskarr) fmask.append(maskhdu) #Close files fmask.close() del fmask except: return None # Check for existance of input .c1h file for use in making inmask file if fileutil.findFile(_dqname) != yes: print('DQ file ',_dqname,' NOT found...') print('Copying ',_mask,'to ',maskname,' as input mask file.') # Now, copy template mask file to output file, if necessary fileutil.copyFile(_mask,maskname,replace=yes) elif bitvalue == None: # If bitvalue was not set, then do not use anything but shadow mask fileutil.copyFile(_mask,maskname,replace=yes) else: # # Build full mask based on .c1h and shadow mask # fdq = fileutil.openImage(_dqname) #fsmask = pyfits.open(_mask,memmap=1,mode='readonly') try: # Read in DQ array from .c1h and from shadow mask files dqarr = fdq[int(extnum)].data #maskarr = fsmask[0].data # Build mask array from DQ array dqmaskarr = buildMask(dqarr,bitvalue) #Write out the mask file as simple FITS file fdqmask = pyfits.open(maskname,'append') maskhdu = pyfits.PrimaryHDU(data=dqmaskarr) fdqmask.append(maskhdu) #Close files fdqmask.close() del fdqmask fdq.close() del fdq except: fdq.close() del fdq # Safeguard against leaving behind an incomplete file if fileutil.findFile(maskname): os.remove(maskname) _errstr = "\nWarning: Problem creating DQMASK file for "+rootname+".\n" #raise IOError, _errstr print(_errstr) return None # Return the name of the mask image written out return maskname
def process(inFile,force=False,newpath=None, inmemory=False, num_cores=None, headerlets=True, align_to_gaia=True): """ Run astrodrizzle on input file/ASN table using default values for astrodrizzle parameters. """ # We only need to import this package if a user run the task import drizzlepac from drizzlepac import processInput # used for creating new ASNs for _flc inputs from stwcs import updatewcs from drizzlepac import alignimages # interpret envvar variable, if specified if envvar_compute_name in os.environ: val = os.environ[envvar_compute_name].lower() if val not in envvar_bool_dict: msg = "ERROR: invalid value for {}.".format(envvar_compute_name) msg += " \n Valid Values: on, off, yes, no, true, false" raise ValueError(msg) align_to_gaia = envvar_bool_dict[val] if envvar_new_apriori_name in os.environ: # Reset ASTROMETRY_STEP_CONTROL based on this variable # This provides backward-compatibility until ASTROMETRY_STEP_CONTROL # gets removed entirely. val = os.environ[envvar_new_apriori_name].lower() if val not in envvar_dict: msg = "ERROR: invalid value for {}.".format(envvar_new_apriori_name) msg += " \n Valid Values: on, off, yes, no, true, false" raise ValueError(msg) os.environ[envvar_old_apriori_name] = envvar_dict[val] if headerlets or align_to_gaia: from stwcs.wcsutil import headerlet # Open the input file try: # Make sure given filename is complete and exists... inFilename = fileutil.buildRootname(inFile,ext=['.fits']) if not os.path.exists(inFilename): print("ERROR: Input file - %s - does not exist." % inFilename) return except TypeError: print("ERROR: Inappropriate input file.") return #If newpath was specified, move all files to that directory for processing if newpath: orig_processing_dir = os.getcwd() new_processing_dir = _createWorkingDir(newpath,inFilename) _copyToNewWorkingDir(new_processing_dir,inFilename) os.chdir(new_processing_dir) # Initialize for later use... _mname = None _new_asn = None _calfiles = [] # Identify WFPC2 inputs to account for differences in WFPC2 inputs wfpc2_input = fits.getval(inFilename, 'instrume') == 'WFPC2' cal_ext = None # Check input file to see if [DRIZ/DITH]CORR is set to PERFORM if '_asn' in inFilename: # We are working with an ASN table. # Use asnutil code to extract filename inFilename = _lowerAsn(inFilename) _new_asn = [inFilename] _asndict = asnutil.readASNTable(inFilename,None,prodonly=False) _cal_prodname = _asndict['output'].lower() #_fname = fileutil.buildRootname(_cal_prodname,ext=['_drz.fits']) # Retrieve the first member's rootname for possible use later _fimg = fits.open(inFilename, memmap=False) for name in _fimg[1].data.field('MEMNAME'): if name[-1] != '*': _mname = name.split('\0', 1)[0].lower() break _fimg.close() del _fimg else: # Check to see if input is a _RAW file # If it is, strip off the _raw.fits extension... _indx = inFilename.find('_raw') if _indx < 0: _indx = len(inFilename) # ... and build the CALXXX product rootname. if wfpc2_input: # force code to define _c0m file as calibrated product to be used cal_ext = ['_c0m.fits'] _mname = fileutil.buildRootname(inFilename[:_indx], ext=cal_ext) _cal_prodname = inFilename[:_indx] # Reset inFilename to correspond to appropriate input for # drizzle: calibrated product name. inFilename = _mname if _mname is None: errorMsg = 'Could not find calibrated product!' raise Exception(errorMsg) # Create trailer filenames based on ASN output filename or # on input name for single exposures if '_raw' in inFile: # Output trailer file to RAW file's trailer _trlroot = inFile[:inFile.find('_raw')] elif '_asn' in inFile: # Output trailer file to ASN file's trailer, not product's trailer _trlroot = inFile[:inFile.find('_asn')] else: # Default: trim off last suffix of input filename # and replacing with .tra _indx = inFile.rfind('_') if _indx > 0: _trlroot = inFile[:_indx] else: _trlroot = inFile _trlfile = _trlroot + '.tra' # Open product and read keyword value # Check to see if product already exists... dkey = 'DRIZCORR' # ...if product does NOT exist, interrogate input file # to find out whether 'dcorr' has been set to PERFORM # Check if user wants to process again regardless of DRIZCORR keyword value if force: dcorr = 'PERFORM' else: if _mname : _fimg = fits.open(fileutil.buildRootname(_mname,ext=['_raw.fits']), memmap=False) _phdr = _fimg['PRIMARY'].header if dkey in _phdr: dcorr = _phdr[dkey] else: dcorr = None _fimg.close() del _fimg else: dcorr = None time_str = _getTime() _tmptrl = _trlroot + '_tmp.tra' _drizfile = _trlroot + '_pydriz' _drizlog = _drizfile + ".log" # the '.log' gets added automatically by astrodrizzle _alignlog = _trlroot + '_align.log' if dcorr == 'PERFORM': if '_asn.fits' not in inFilename: # Working with a singleton # However, we always want to make sure we always use # a calibrated product as input, if available. _infile = fileutil.buildRootname(_cal_prodname, ext=cal_ext) _infile_flc = fileutil.buildRootname(_cal_prodname,ext=['_flc.fits']) _cal_prodname = _infile _inlist = _calfiles = [_infile] # Add CTE corrected filename as additional input if present if os.path.exists(_infile_flc) and _infile_flc != _infile: _inlist.append(_infile_flc) else: # Working with an ASN table... _infile = inFilename flist,duplist = processInput.checkForDuplicateInputs(_asndict['order']) _calfiles = flist if len(duplist) > 0: origasn = processInput.changeSuffixinASN(inFilename,'flt') dupasn = processInput.changeSuffixinASN(inFilename,'flc') _inlist = [origasn,dupasn] else: _inlist = [_infile] # We want to keep the original specification of the calibration # product name, though, not a lower-case version... _cal_prodname = inFilename _new_asn.extend(_inlist) # kept so we can delete it when finished # check to see whether FLC files are also present, and need to be updated # generate list of FLC files align_files = None _calfiles_flc = [f.replace('_flt.fits','_flc.fits') for f in _calfiles] # insure these files exist, if not, blank them out # Also pick out what files will be used for additional alignment to GAIA if not os.path.exists(_calfiles_flc[0]): _calfiles_flc = None align_files = _calfiles align_update_files = None else: align_files = _calfiles_flc align_update_files = _calfiles # Run updatewcs on each list of images updatewcs.updatewcs(_calfiles) if _calfiles_flc: updatewcs.updatewcs(_calfiles_flc) if align_to_gaia: # Perform additional alignment on the FLC files, if present ############### # # call hlapipeline code here on align_files list of files # ############### # Create trailer marker message for start of align_to_GAIA processing _trlmsg = _timestamp("Align_to_GAIA started ") print(_trlmsg) ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) _trlmsg = "" # Create an empty astropy table so it can be used as input/output for the perform_align function #align_table = Table() try: align_table = alignimages.perform_align(align_files,update_hdr_wcs=True, runfile=_alignlog) for row in align_table: if row['status'] == 0: trlstr = "Successfully aligned {} to {} astrometric frame\n" _trlmsg += trlstr.format(row['imageName'], row['catalog']) else: trlstr = "Could not align {} to absolute astrometric frame\n" _trlmsg += trlstr.format(row['imageName']) except Exception: # Something went wrong with alignment to GAIA, so report this in # trailer file _trlmsg = "EXCEPTION encountered in alignimages...\n" _trlmsg += " No correction to absolute astrometric frame applied!\n" # Write the perform_align log to the trailer file...(this will delete the _alignlog) _appendTrlFile(_trlfile,_alignlog) # Append messages from this calling routine post-perform_align ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) _trlmsg = "" #Check to see whether there are any additional input files that need to # be aligned (namely, FLT images) if align_update_files and align_table: # Apply headerlets from alignment to FLT version of the files for fltfile, flcfile in zip(align_update_files, align_files): row = align_table[align_table['imageName']==flcfile] headerletFile = row['headerletFile'][0] if headerletFile != "None": headerlet.apply_headerlet_as_primary(fltfile, headerletFile, attach=True, archive=True) # append log file contents to _trlmsg for inclusion in trailer file _trlstr = "Applying headerlet {} as Primary WCS to {}\n" _trlmsg += _trlstr.format(headerletFile, fltfile) else: _trlmsg += "No absolute astrometric headerlet applied to {}\n".format(fltfile) # Finally, append any further messages associated with alignement from this calling routine _trlmsg += _timestamp('Align_to_GAIA completed ') print(_trlmsg) ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) # Run astrodrizzle and send its processing statements to _trlfile _pyver = drizzlepac.astrodrizzle.__version__ for _infile in _inlist: # Run astrodrizzle for all inputs # Create trailer marker message for start of astrodrizzle processing _trlmsg = _timestamp('astrodrizzle started ') _trlmsg += __trlmarker__ _trlmsg += '%s: Processing %s with astrodrizzle Version %s\n' % (time_str,_infile,_pyver) print(_trlmsg) # Write out trailer comments to trailer file... ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) _pyd_err = _trlroot+'_pydriz.stderr' try: b = drizzlepac.astrodrizzle.AstroDrizzle(input=_infile,runfile=_drizfile, configobj='defaults',in_memory=inmemory, num_cores=num_cores, **pipeline_pars) except Exception as errorobj: _appendTrlFile(_trlfile,_drizlog) _appendTrlFile(_trlfile,_pyd_err) _ftrl = open(_trlfile,'a') _ftrl.write('ERROR: Could not complete astrodrizzle processing of %s.\n' % _infile) _ftrl.write(str(sys.exc_info()[0])+': ') _ftrl.writelines(str(errorobj)) _ftrl.write('\n') _ftrl.close() print('ERROR: Could not complete astrodrizzle processing of %s.' % _infile) raise Exception(str(errorobj)) # Now, append comments created by PyDrizzle to CALXXX trailer file print('Updating trailer file %s with astrodrizzle comments.' % _trlfile) _appendTrlFile(_trlfile,_drizlog) # Save this for when astropy.io.fits can modify a file 'in-place' # Update calibration switch _fimg = fits.open(_cal_prodname, mode='update', memmap=False) _fimg['PRIMARY'].header[dkey] = 'COMPLETE' _fimg.close() del _fimg # Enforce pipeline convention of all lower-case product # names _prodlist = glob.glob('*drz.fits') for _prodname in _prodlist: _plower = _prodname.lower() if _prodname != _plower: os.rename(_prodname,_plower) else: # Create default trailer file messages when astrodrizzle is not # run on a file. This will typically apply only to BIAS,DARK # and other reference images. # Start by building up the message... _trlmsg = _timestamp('astrodrizzle skipped ') _trlmsg = _trlmsg + __trlmarker__ _trlmsg = _trlmsg + '%s: astrodrizzle processing not requested for %s.\n' % (time_str,inFilename) _trlmsg = _trlmsg + ' astrodrizzle will not be run at this time.\n' print(_trlmsg) # Write message out to temp file and append it to full trailer file ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) # Append final timestamp to trailer file... _final_msg = '%s: Finished processing %s \n' % (time_str,inFilename) _final_msg += _timestamp('astrodrizzle completed ') _trlmsg += _final_msg ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) # If we created a new ASN table, we need to remove it if _new_asn is not None: for _name in _new_asn: fileutil.removeFile(_name) # Clean up any generated OrIg_files directory if os.path.exists("OrIg_files"): # check to see whether this directory is empty flist = glob.glob('OrIg_files/*.fits') if len(flist) == 0: os.rmdir("OrIg_files") else: print('OrIg_files directory NOT removed as it still contained images...') # If headerlets have already been written out by alignment code, # do NOT write out this version of the headerlets if headerlets: # Generate headerlets for each updated FLT image hlet_msg = _timestamp("Writing Headerlets started") for fname in _calfiles: frootname = fileutil.buildNewRootname(fname) hname = "%s_flt_hlet.fits"%frootname # Write out headerlet file used by astrodrizzle, however, # do not overwrite any that was already written out by alignimages if not os.path.exists(hname): hlet_msg += "Created Headerlet file %s \n"%hname try: headerlet.write_headerlet(fname,'OPUS',output='flt', wcskey='PRIMARY', author="OPUS",descrip="Default WCS from Pipeline Calibration", attach=False,clobber=True,logging=False) except ValueError: hlet_msg += _timestamp("SKIPPED: Headerlet not created for %s \n"%fname) # update trailer file to log creation of headerlet files hlet_msg += _timestamp("Writing Headerlets completed") ftrl = open(_trlfile,'a') ftrl.write(hlet_msg) ftrl.close() # If processing was done in a temp working dir, restore results to original # processing directory, return to original working dir and remove temp dir if newpath: _restoreResults(new_processing_dir,orig_processing_dir) os.chdir(orig_processing_dir) _removeWorkingDir(new_processing_dir) # Provide feedback to user print(_final_msg)
def writeFITS(self, template, sciarr, whtarr, ctxarr=None, versions=None, extlist=EXTLIST, overwrite=yes): """ Generate PyFITS objects for each output extension using the file given by 'template' for populating headers. The arrays will have the size specified by 'shape'. """ if fileutil.findFile(self.output): if overwrite: print('Deleting previous output product: ',self.output) fileutil.removeFile(self.output) else: print('WARNING: Output file ',self.output,' already exists and overwrite not specified!') print('Quitting... Please remove before resuming operations.') raise IOError # Default value for NEXTEND when 'build'== True nextend = 3 if not self.build: nextend = 0 if self.outweight: if overwrite: if fileutil.findFile(self.outweight): print('Deleting previous output WHT product: ',self.outweight) fileutil.removeFile(self.outweight) else: print('WARNING: Output file ',self.outweight,' already exists and overwrite not specified!') print('Quitting... Please remove before resuming operations.') raise IOError if self.outcontext: if overwrite: if fileutil.findFile(self.outcontext): print('Deleting previous output CTX product: ',self.outcontext) fileutil.removeFile(self.outcontext) else: print('WARNING: Output file ',self.outcontext,' already exists and overwrite not specified!') print('Quitting... Please remove before resuming operations.') raise IOError # Get default headers from multi-extension FITS file # If input data is not in MEF FITS format, it will return 'None' # and those headers will have to be generated from drizzle output # file FITS headers. # NOTE: These are HEADER objects, not HDUs prihdr,scihdr,errhdr,dqhdr = getTemplates(template,extlist) if prihdr == None: # Use readgeis to get header for use as Primary header. _indx = template.find('[') if _indx < 0: _data = template else: _data = template[:_indx] fpri = readgeis.readgeis(_data) prihdr = fpri[0].header.copy() fpri.close() del fpri # Setup primary header as an HDU ready for appending to output FITS file prihdu = pyfits.PrimaryHDU(header=prihdr,data=None) # Start by updating PRIMARY header keywords... prihdu.header.update('EXTEND',pyfits.TRUE,after='NAXIS') prihdu.header.update('NEXTEND',nextend) prihdu.header.update('FILENAME', self.output) # Update the ROOTNAME with the new value as well _indx = self.output.find('_drz') if _indx < 0: prihdu.header.update('ROOTNAME', self.output) else: prihdu.header.update('ROOTNAME', self.output[:_indx]) # Get the total exposure time for the image # If not calculated by PyDrizzle and passed through # the pardict, then leave value from the template image. if self.texptime: prihdu.header.update('EXPTIME', self.texptime) prihdu.header.update('EXPSTART', self.expstart) prihdu.header.update('EXPEND', self.expend) #Update ASN_MTYPE to reflect the fact that this is a product # Currently hard-wired to always output 'PROD-DTH' as MTYPE prihdu.header.update('ASN_MTYP', 'PROD-DTH') # Update DITHCORR calibration keyword if present # Remove when we can modify FITS headers in place... if 'DRIZCORR' in prihdu.header: prihdu.header.update('DRIZCORR','COMPLETE') if 'DITHCORR' in prihdu.header: prihdu.header.update('DITHCORR','COMPLETE') prihdu.header.update('NDRIZIM',len(self.parlist), comment='Drizzle, No. images drizzled onto output') self.addDrizKeywords(prihdu.header,versions) if scihdr: del scihdr['OBJECT'] if 'CCDCHIP' in scihdr: scihdr.update('CCDCHIP','-999') if 'NCOMBINE' in scihdr: scihdr.update('NCOMBINE', self.parlist[0]['nimages']) # If BUNIT keyword was found and reset, then if self.bunit is not None: scihdr.update('BUNIT',self.bunit,comment="Units of science product") if self.wcs: # Update WCS Keywords based on PyDrizzle product's value # since 'drizzle' itself doesn't update that keyword. scihdr.update('ORIENTAT',self.wcs.orient) scihdr.update('CD1_1',self.wcs.cd11) scihdr.update('CD1_2',self.wcs.cd12) scihdr.update('CD2_1',self.wcs.cd21) scihdr.update('CD2_2',self.wcs.cd22) scihdr.update('CRVAL1',self.wcs.crval1) scihdr.update('CRVAL2',self.wcs.crval2) scihdr.update('CRPIX1',self.wcs.crpix1) scihdr.update('CRPIX2',self.wcs.crpix2) scihdr.update('VAFACTOR',1.0) # Remove any reference to TDD correction if 'TDDALPHA' in scihdr: del scihdr['TDDALPHA'] del scihdr['TDDBETA'] # Remove '-SIP' from CTYPE for output product if scihdr['ctype1'].find('SIP') > -1: scihdr.update('ctype1', scihdr['ctype1'][:-4]) scihdr.update('ctype2',scihdr['ctype2'][:-4]) # Remove SIP coefficients from DRZ product for k in scihdr.items(): if (k[0][:2] in ['A_','B_']) or (k[0][:3] in ['IDC','SCD'] and k[0] != 'IDCTAB') or \ (k[0][:6] in ['SCTYPE','SCRVAL','SNAXIS','SCRPIX']): del scihdr[k[0]] self.addPhotKeywords(scihdr,prihdu.header) ########## # Now, build the output file ########## if self.build: print('-Generating multi-extension output file: ',self.output) fo = pyfits.HDUList() # Add primary header to output file... fo.append(prihdu) hdu = pyfits.ImageHDU(data=sciarr,header=scihdr,name=extlist[0]) fo.append(hdu) # Build WHT extension here, if requested... if errhdr: errhdr.update('CCDCHIP','-999') hdu = pyfits.ImageHDU(data=whtarr,header=errhdr,name=extlist[1]) hdu.header.update('EXTVER',1) if self.wcs: # Update WCS Keywords based on PyDrizzle product's value # since 'drizzle' itself doesn't update that keyword. hdu.header.update('ORIENTAT',self.wcs.orient) hdu.header.update('CD1_1',self.wcs.cd11) hdu.header.update('CD1_2',self.wcs.cd12) hdu.header.update('CD2_1',self.wcs.cd21) hdu.header.update('CD2_2',self.wcs.cd22) hdu.header.update('CRVAL1',self.wcs.crval1) hdu.header.update('CRVAL2',self.wcs.crval2) hdu.header.update('CRPIX1',self.wcs.crpix1) hdu.header.update('CRPIX2',self.wcs.crpix2) hdu.header.update('VAFACTOR',1.0) fo.append(hdu) # Build CTX extension here # If there is only 1 plane, write it out as a 2-D extension if self.outcontext: if ctxarr.shape[0] == 1: _ctxarr = ctxarr[0] else: _ctxarr = ctxarr else: _ctxarr = None hdu = pyfits.ImageHDU(data=_ctxarr,header=dqhdr,name=extlist[2]) hdu.header.update('EXTVER',1) if self.wcs: # Update WCS Keywords based on PyDrizzle product's value # since 'drizzle' itself doesn't update that keyword. hdu.header.update('ORIENTAT',self.wcs.orient) hdu.header.update('CD1_1',self.wcs.cd11) hdu.header.update('CD1_2',self.wcs.cd12) hdu.header.update('CD2_1',self.wcs.cd21) hdu.header.update('CD2_2',self.wcs.cd22) hdu.header.update('CRVAL1',self.wcs.crval1) hdu.header.update('CRVAL2',self.wcs.crval2) hdu.header.update('CRPIX1',self.wcs.crpix1) hdu.header.update('CRPIX2',self.wcs.crpix2) hdu.header.update('VAFACTOR',1.0) fo.append(hdu) fo.writeto(self.output) fo.close() del fo, hdu else: print('-Generating simple FITS output: ',self.outdata) fo = pyfits.HDUList() hdu = pyfits.PrimaryHDU(data=sciarr, header=prihdu.header) # Append remaining unique header keywords from template DQ # header to Primary header... if scihdr: for _card in scihdr.ascard: if (_card.key not in RESERVED_KEYS and _card.key not in hdu.header): hdu.header.ascard.append(_card) del hdu.header['PCOUNT'] del hdu.header['GCOUNT'] self.addPhotKeywords(hdu.header, prihdu.header) hdu.header.update('filename', self.outdata) # Add primary header to output file... fo.append(hdu) fo.writeto(self.outdata) del fo,hdu if self.outweight and whtarr != None: # We need to build new PyFITS objects for each WHT array fwht = pyfits.HDUList() if errhdr: errhdr.update('CCDCHIP','-999') hdu = pyfits.PrimaryHDU(data=whtarr, header=prihdu.header) # Append remaining unique header keywords from template DQ # header to Primary header... if errhdr: for _card in errhdr.ascard: if (_card.key not in RESERVED_KEYS and _card.key not in hdu.header): hdu.header.ascard.append(_card) hdu.header.update('filename', self.outweight) hdu.header.update('CCDCHIP','-999') if self.wcs: # Update WCS Keywords based on PyDrizzle product's value # since 'drizzle' itself doesn't update that keyword. hdu.header.update('ORIENTAT',self.wcs.orient) hdu.header.update('CD1_1',self.wcs.cd11) hdu.header.update('CD1_2',self.wcs.cd12) hdu.header.update('CD2_1',self.wcs.cd21) hdu.header.update('CD2_2',self.wcs.cd22) hdu.header.update('CRVAL1',self.wcs.crval1) hdu.header.update('CRVAL2',self.wcs.crval2) hdu.header.update('CRPIX1',self.wcs.crpix1) hdu.header.update('CRPIX2',self.wcs.crpix2) hdu.header.update('VAFACTOR',1.0) # Add primary header to output file... fwht.append(hdu) fwht.writeto(self.outweight) del fwht,hdu # If a context image was specified, build a PyFITS object # for it as well... if self.outcontext and ctxarr != None: fctx = pyfits.HDUList() # If there is only 1 plane, write it out as a 2-D extension if ctxarr.shape[0] == 1: _ctxarr = ctxarr[0] else: _ctxarr = ctxarr hdu = pyfits.PrimaryHDU(data=_ctxarr, header=prihdu.header) # Append remaining unique header keywords from template DQ # header to Primary header... if dqhdr: for _card in dqhdr.ascard: if (_card.key not in RESERVED_KEYS and _card.key not in hdu.header): hdu.header.ascard.append(_card) hdu.header.update('filename', self.outcontext) if self.wcs: # Update WCS Keywords based on PyDrizzle product's value # since 'drizzle' itself doesn't update that keyword. hdu.header.update('ORIENTAT',self.wcs.orient) hdu.header.update('CD1_1',self.wcs.cd11) hdu.header.update('CD1_2',self.wcs.cd12) hdu.header.update('CD2_1',self.wcs.cd21) hdu.header.update('CD2_2',self.wcs.cd22) hdu.header.update('CRVAL1',self.wcs.crval1) hdu.header.update('CRVAL2',self.wcs.crval2) hdu.header.update('CRPIX1',self.wcs.crpix1) hdu.header.update('CRPIX2',self.wcs.crpix2) hdu.header.update('VAFACTOR',1.0) fctx.append(hdu) fctx.writeto(self.outcontext) del fctx,hdu