def buildMask(dqarr, bitvalue): """ Builds a bit-mask from an input DQ array and a bitvalue flag""" bitvalue = bitmask.interpret_bits_value(bitvalue) if bitvalue is None: return (np.ones(dqarr.shape, dtype=np.uint32)) return np.logical_not(np.bitwise_and(dqarr, ~bitvalue)).astype(np.uint32)
def buildMask(dqarr, bitvalue): """ Builds a bit-mask from an input DQ array and a bitvalue flag""" bitvalue = bitmask.interpret_bits_value(bitvalue) if bitvalue is None: return (np.ones(dqarr.shape, dtype=np.uint32)) return np.logical_not(np.bitwise_and(dqarr, ~bitvalue)).astype(np.uint32)
def buildMask(dqarr, bitvalue): """ Builds a bit-mask from an input DQ array and a bitvalue flag""" bitvalue = bitmask.interpret_bits_value(bitvalue) #if bitvalue == None: #return ((dqarr * 0.0) + 1.0).astype(np.uint8) #_maskarr = np.bitwise_or(dqarr,np.array([bitvalue])) #return np.choose(np.greater(_maskarr,bitvalue),(1,0)).astype(np.uint8) if bitvalue is None: return (np.ones(dqarr.shape, dtype=np.uint8)) return np.logical_not(np.bitwise_and(dqarr, ~bitvalue)).astype(np.uint8)
def buildMask(dqarr,bitvalue): """ Builds a bit-mask from an input DQ array and a bitvalue flag""" bitvalue = bitmask.interpret_bits_value(bitvalue) #if bitvalue == None: #return ((dqarr * 0.0) + 1.0).astype(np.uint8) #_maskarr = np.bitwise_or(dqarr,np.array([bitvalue])) #return np.choose(np.greater(_maskarr,bitvalue),(1,0)).astype(np.uint8) if bitvalue is None: return (np.ones(dqarr.shape,dtype=np.uint8)) return np.logical_not(np.bitwise_and(dqarr,~bitvalue)).astype(np.uint8)
def _mergeUserMaskAndDQ(dq, mask, dqbits): dqbits = bitmask.interpret_bits_value(dqbits) if dqbits is None: if mask is None: return None else: return mask.copy().astype(dtype=np.uint8) if dq is None: raise ValueError("DQ array is None while 'dqbits' is not None.") dqmask = bitmask.bitmask2mask(bitmask=dq, ignore_bits=dqbits, good_mask_value=1, dtype=np.uint8) if mask is None: return dqmask # merge user mask with DQ mask: dqmask[mask == 0] = 0 return dqmask
def process(self, input): img = datamodels.ModelContainer(input) self._dqbits = bitmask.interpret_bits_value(self.dqbits) # set sky stattistics: self._skystat = SkyStats(skystat=self.skystat, lower=self.lower, upper=self.upper, nclip=self.nclip, lsig=self.lsigma, usig=self.usigma, binwidth=self.binwidth) # group images by their "group id": grp_img = img.models_grouped # create a list of "Sky" Images and/or Groups: images = [] for g in grp_img: if len(g) > 1: images.append(SkyGroup(list(map(self._imodel2skyim, g)))) elif len(g) == 1: images.append(self._imodel2skyim(g[0])) else: raise AssertionError("Logical error in the pipeline code.") match(images, skymethod=self.skymethod, match_down=self.match_down, subtract=self.subtract) # set sky background value in each image's meta: for im in images: if isinstance(im, SkyImage): pass return img
def process(self, input): img = models.ModelContainer(input) self._dqbits = bitmask.interpret_bits_value(self.dqbits) # set sky stattistics: self._skystat = SkyStats( skystat = self.skystat, lower = self.lower, upper = self.upper, nclip = self.nclip, lsig = self.lsigma, usig = self.usigma, binwidth = self.binwidth ) # group images by their "group id": grp_img = img.models_grouped # create a list of "Sky" Images and/or Groups: images = [] for g in grp_img: if len(g) > 1: images.append(SkyGroup(list(map(self._imodel2skyim, g)))) elif len(g) == 1: images.append(self._imodel2skyim(g[0])) else: raise AssertionError("Logical error in the pipeline code.") match(images, skymethod=self.skymethod, match_down=self.match_down, subtract=self.subtract) # set sky background value in each image's meta: for im in images: if isinstance(im, SkyImage): pass return img
def _skymatch(imageList, paramDict, in_memory, clean, logfile): # '_skymatch' converts input imageList and other parameters to # data structures accepted by the "skymatch" package. # It also creates a temporary mask by combining 'static' mask, # DQ image, and user-supplied mask. The combined mask is then # passed to 'skymatch' to be used for excluding "bad" pixels. skyKW="MDRIZSKY" #header keyword that contains the sky that's been subtracted nimg = len(imageList) if nimg == 0: ml.logentry("Skymatch needs at least one images to perform{0}" \ "sky matching. Nothing to be done.",os.linesep) return # create a list of input file names as provided by the user: user_fnames = [] loaded_fnames = [] filemaskinfos = nimg * [ None ] for img in imageList: user_fnames.append(img._original_file_name) loaded_fnames.append(img._filename) # parse sky mask catalog file (if any): catfile = paramDict['skymask_cat'] if catfile: #extname = imageList[0].scienceExt #assert(extname is not None and extname != '') catfile = catfile.strip() mfindx = parse_at_file(fname = catfile, default_ext = ('SCI','*'), default_mask_ext = 0, clobber = False, fnamesOnly = True, doNotOpenDQ = True, match2Images = user_fnames, im_fmode = 'update', dq_fmode = 'readonly', msk_fmode = 'readonly', logfile = MultiFileLog(console=True), verbose = True) for p in mfindx: filemaskinfos[p[1]] = p[0] # step through the list of input images and create # combined (static + DQ + user supplied, if any) mask, and # create a list of FileExtMaskInfo objects to be passed # to 'skymatch' function. # # This needs to be done in several steps, mostly due to the fact that # the mask catalogs use "original" (e.g., GEIS, WAIVER FITS) file names # while ultimately we want to open the version converted to MEF. Second # reason is that we want to combine user supplied masks with DQ+static # masks provided by astrodrizzle. new_fi = [] sky_bits = bitmask.interpret_bits_value(paramDict['sky_bits']) for i in range(nimg): # extract extension information: extname = imageList[i].scienceExt extver = imageList[i].group if extver is None: extver = imageList[i].getExtensions() assert(extname is not None and extname != '') assert(extver) # create a new FileExtMaskInfo object fi = FileExtMaskInfo(default_ext=(extname,'*'), default_mask_ext=0, clobber=False, doNotOpenDQ=True, fnamesOnly=False, im_fmode='update', dq_fmode='readonly', msk_fmode='readonly') # set image file and extensions: fi.image = loaded_fnames[i] extlist = [ (extname,ev) for ev in extver ] fi.append_ext(extlist) # set user masks if any (this will open the files for a later use): fi0 = filemaskinfos[i] if fi0 is not None: nmask = len(fi0.mask_images) for m in range(nmask): mask = fi0.mask_images[m] ext = fi0.maskext[m] fi.append_mask(mask, ext) fi.finalize() # combine user masks with static masks: assert(len(extlist) == fi.count) #TODO: <-- remove after thorough testing masklist = [] mextlist = [] for k in range(fi.count): if fi.mask_images[k].closed: umask = None else: umask = fi.mask_images[k].hdu[fi.maskext[k]].data (mask, mext) = _buildStaticDQUserMask(imageList[i], extlist[k], sky_bits, paramDict['use_static'], fi.mask_images[k], fi.maskext[k], in_memory) masklist.append(mask) mextlist.append(mext) # replace the original user-supplied masks with the # newly computed combined static+DQ+user masks: fi.clear_masks() for k in range(fi.count): if in_memory and mask is not None: # os.stat() on the "original_fname" of the mask will fail # since this is a "virtual" mask. Therefore we need to compute # mask_stat ourselves. We will simply use id(data) for this: mstat = os.stat_result((0,id(mask.hdu)) + 8*(0,)) fi.append_mask(masklist[k], mextlist[k], mask_stat=mstat) else: fi.append_mask(masklist[k], mextlist[k]) if masklist[k]: masklist[k].release() fi.finalize() new_fi.append(fi) # Run skymatch algorithm: skymatch(new_fi, skymethod = paramDict['skymethod'], skystat = paramDict['skystat'], lower = paramDict['skylower'], upper = paramDict['skyupper'], nclip = paramDict['skyclip'], lsigma = paramDict['skylsigma'], usigma = paramDict['skyusigma'], binwidth = paramDict['skywidth'], skyuser_kwd = skyKW, units_kwd = 'BUNIT', readonly = not paramDict['skysub'], dq_bits = None, optimize = 'inmemory' if in_memory else 'balanced', clobber = True, clean = clean, verbose = True, flog = MultiFileLog(console = True, enableBold = False), _taskname4history = 'AstroDrizzle') # Populate 'subtractedSky' and 'computedSky' of input image objects: for i in range(nimg): assert(not new_fi[i].fnamesOnly and not new_fi[i].image.closed) image = imageList[i] skysubimage = new_fi[i].image.hdu numchips = image._numchips extname = image.scienceExt assert(os.path.samefile(image._filename, skysubimage.filename())) for extver in range(1,numchips+1,1): chip = image[extname,extver] if not chip.group_member: continue subtracted_sky = skysubimage[extname,extver].header[skyKW] chip.subtractedSky = subtracted_sky chip.computedSky = subtracted_sky # clean-up: for fi in new_fi: fi.release_all_images()
def setCommonInput(configObj, createOutwcs=True): """ The common interface interpreter for MultiDrizzle tasks which not only runs 'process_input()' but 'createImageObject()' and 'defineOutput()' as well to fully setup all inputs for use with the rest of the MultiDrizzle steps either as stand-alone tasks or internally to MultiDrizzle itself. Parameters ---------- configObj : object configObj instance or simple dictionary of input parameters imageObjectList : list of imageObject objects list of imageObject instances, 1 for each input exposure outwcs : object imageObject instance defining the final output frame Notes ----- At a minimum, the configObj instance (dictionary) should contain: configObj = {'input':None,'output':None } If provided, the configObj should contain the values of all the multidrizzle parameters as set by the user with TEAL. If no configObj is given, it will retrieve the default values automatically. In either case, the values from the input_dict will be merged in with the configObj before being used by the rest of the code. Examples -------- You can set *createOutwcs=False* for the cases where you only want the images processed and no output wcs information in necessary; as in: >>>imageObjectList,outwcs = processInput.processCommonInput(configObj) """ # make sure 'updatewcs' is set to False when running from GUI or if missing # from configObj: if 'updatewcs' not in configObj: configObj['updatewcs'] = False if not createOutwcs or not configObj['coeffs']: # we're probably just working on single images here configObj['updatewcs']=False # maybe we can chunk this part up some more so that we can call just the # parts we want # Interpret input, read and convert and update input files, then return # list of input filenames and derived output filename asndict, ivmlist, output = process_input( configObj['input'], configObj['output'], updatewcs=configObj['updatewcs'], wcskey=configObj['wcskey'], **configObj['STATE OF INPUT FILES']) if not asndict: return None, None # convert the filenames from asndict into a list of full filenames files = [fileutil.buildRootname(f) for f in asndict['order']] original_files = asndict['original_file_names'] # interpret MDRIZTAB, if specified, and update configObj accordingly # This can be done here because MDRIZTAB does not include values for # input, output, or updatewcs. if 'mdriztab' in configObj and configObj['mdriztab']: print("Reading in MDRIZTAB parameters for {} files".format(len(files))) mdriztab_dict = mdzhandler.getMdriztabParameters(files) # Update configObj with values from mpars cfgpars.mergeConfigObj(configObj, mdriztab_dict) # Convert interpreted list of input files from process_input into a list # of imageObject instances for use by the MultiDrizzle tasks. instrpars = configObj['INSTRUMENT PARAMETERS'] # pass in 'proc_unit' to initialize unit conversions as necessary instrpars['proc_unit'] = configObj['proc_unit'] undistort = True if not configObj['coeffs']: undistort = False # determine whether parallel processing will be performed use_parallel = False if util.can_parallel: # look to see whether steps which can be run using multiprocessing # have been turned on for stepnum in parallel_steps: sname = util.getSectionName(configObj,stepnum[0]) if configObj[sname][stepnum[1]]: use_parallel = True break # interpret all 'bits' related parameters and convert them to integers configObj['resetbits'] = bitmask.interpret_bits_value(configObj['resetbits']) step3name = util.getSectionName(configObj,3) configObj[step3name]['driz_sep_bits'] = bitmask.interpret_bits_value( configObj[step3name]['driz_sep_bits']) step7name = util.getSectionName(configObj,7) configObj[step7name]['final_bits'] = bitmask.interpret_bits_value( configObj[step7name]['final_bits']) # Verify any refimage parameters to be used step3aname = util.getSectionName(configObj,'3a') if not util.verifyRefimage(configObj[step3aname]['driz_sep_refimage']): msg = 'No refimage with WCS found!\n '+\ ' This could be caused by one of 2 problems:\n'+\ ' * filename does not specify an extension with a valid WCS.\n'+\ ' * can not find the file.\n'+\ 'Please check the filename specified in the "refimage" parameter.' print(textutil.textbox(msg)) return None,None step7aname = util.getSectionName(configObj,'7a') if not util.verifyRefimage(configObj[step7aname]['final_refimage']): msg = 'No refimage with WCS found!\n '+\ ' This could be caused by one of 2 problems:\n'+\ ' * filename does not specify an extension with a valid WCS.\n'+\ ' * can not find the file.\n'+\ 'Please check the filename specified in the "refimage" parameter.' print(textutil.textbox(msg)) return None,None # Build imageObject list for all the valid, shift-updated input files log.info('-Creating imageObject List as input for processing steps.') if 'in_memory' in configObj: virtual = configObj['in_memory'] else: virtual = False imageObjectList = createImageObjectList(files, instrpars, group=configObj['group'], undistort=undistort, inmemory=virtual) # Add original file names as "hidden" attributes of imageObject assert(len(original_files) == len(imageObjectList)) #TODO: remove after extensive testing for i in range(len(imageObjectList)): imageObjectList[i]._original_file_name = original_files[i] # apply context parameter applyContextPar(imageObjectList, configObj['context']) # reset DQ bits if requested by user resetDQBits(imageObjectList, cr_bits_value=configObj['resetbits']) # Add info about input IVM files at this point to the imageObjectList addIVMInputs(imageObjectList, ivmlist) if createOutwcs: log.info('-Creating output WCS.') # Build output WCS and update imageObjectList with output WCS info outwcs = wcs_functions.make_outputwcs(imageObjectList, output, configObj=configObj, perfect=True) outwcs.final_wcs.printwcs() else: outwcs = None try: # Provide user with some information on resource usage for this run # raises ValueError Exception in interactive mode and user quits num_cores = configObj.get('num_cores') if use_parallel else 1 reportResourceUsage(imageObjectList, outwcs, num_cores) except ValueError: imageObjectList = None return imageObjectList, outwcs
def process(self, input1, input2): cube_models = datamodels.ModelContainer(input1) models2d = datamodels.ModelContainer(input2) dqbits = bitmask.interpret_bits_value(self.dqbits) # set sky stattistics: self._skystat = SkyStats( skystat=self.skystat, lower=self.lower, upper=self.upper, nclip=self.nclip, lsig=self.lsigma, usig=self.usigma, binwidth=self.binwidth ) # create a list of SkyCubes: skycubes = [] for cm in cube_models: # process weights and combine with DQ: if not hasattr(cm, 'weightmap') or cm.weightmap is None: weights = np.ones_like(cm.data, dtype=np.float64) else: weights = cm.weightmap.copy() if dqbits is not None: dq = bitmask.bitmask2mask(bitmask=cm.dq, ignore_bits=self._dqbits, good_mask_value=0, dtype=np.bool) weights[dq] = 0.0 wcs = cm.meta.wcs if hasattr(cm.meta, 'wcs') else None wcsinfo = cm.meta.wcsinfo if hasattr(cm.meta, 'wcsinfo') else None exptime = cm.meta.exposure.exposure_time if exptime is None: exptime = 1.0 sc = SkyCube( data=cm.data, wcs=wcs, wcsinfo=wcsinfo, weights=weights, cube_weight=exptime, bkg_deg=self.bkg_degree, bkg_center=None, id=None, meta={'original_cube_model': cm} ) skycubes.append(sc) skymatch_cubes, nsubspace = match(skycubes, subtract=self.subtract) if nsubspace > 1: self.log.warning("Not all cubes have been sky matched as " "some of them do not overlap.") # save background info in 'meta' and subtract sky from 2D images # if requested: ##### model.meta.instrument.channel for c in skymatch_cubes: self._set_cube_bkg_meta(c.meta['original_cube_model'], c) model2d, channel = _find_associated_2d_image( c.meta['original_cube_model'], models2d ) if model2d is None: continue self._set_model2d_bkg_meta( c.meta['original_cube_model'], model2d, channel ) if self.subtract2d: self._apply_sky_2d(model2d, channel) return cube_models, models2d
def reset_dq_bits(input,bits,extver=None,extname='dq'): """ This function resets bits in the integer array(s) of a FITS file. Parameters ---------- filename : str full filename with path bits : str sum or list of integers corresponding to all the bits to be reset extver : int, optional List of version numbers of the DQ arrays to be corrected [Default Value: None, will do all] extname : str, optional EXTNAME of the DQ arrays in the FITS file [Default Value: 'dq'] Notes ----- The default value of None for the 'extver' parameter specifies that all extensions with EXTNAME matching 'dq' (as specified by the 'extname' parameter) will have their bits reset. Examples -------- 1. The following command will reset the 4096 bits in all the DQ arrays of the file input_file_flt.fits:: reset_dq_bits("input_file_flt.fits", 4096) 2. To reset the 2,32,64 and 4096 bits in the second DQ array, specified as 'dq,2', in the file input_file_flt.fits:: reset_dq_bits("input_file_flt.fits", "2,32,64,4096", extver=2) """ # Interpret bits value bits = bitmask.interpret_bits_value(bits) flist, fcol = parseinput.parseinput(input) for filename in flist: # open input file in write mode to allow updating the DQ array in-place p = fits.open(filename,mode='update') # Identify the DQ array to be updated # If no extver is specified, build a list of all DQ arrays in the file if extver is None: extver = [] for hdu in p: # find only those extensions which match the input extname # using case-insensitive name comparisons for 'extname' if 'extver' in hdu.header and \ hdu.header['extname'].lower() == extname.lower(): extver.append(int(hdu.header['extver'])) else: # Otherwise, insure that input extver values are a list if not isinstance(extver, list): extver = [extver] # for each DQ array identified in the file... for extn in extver: dqarr = p[extname,extn].data dqdtype = dqarr.dtype # reset the desired bits p[extname,extn].data = (dqarr & ~bits).astype(dqdtype) # preserve original dtype log.info('Reset bit values of %s to a value of 0 in %s[%s,%s]' % (bits, filename, extname, extn)) # close the file with the updated DQ array(s) p.close()
def _skymatch(imageList, paramDict, in_memory, clean, logfile): # '_skymatch' converts input imageList and other parameters to # data structures accepted by the "skymatch" package. # It also creates a temporary mask by combining 'static' mask, # DQ image, and user-supplied mask. The combined mask is then # passed to 'skymatch' to be used for excluding "bad" pixels. skyKW = "MDRIZSKY" #header keyword that contains the sky that's been subtracted nimg = len(imageList) if nimg == 0: ml.logentry("Skymatch needs at least one images to perform{0}" \ "sky matching. Nothing to be done.",os.linesep) return # create a list of input file names as provided by the user: user_fnames = [] loaded_fnames = [] filemaskinfos = nimg * [None] for img in imageList: user_fnames.append(img._original_file_name) loaded_fnames.append(img._filename) # parse sky mask catalog file (if any): catfile = paramDict['skymask_cat'] if catfile: #extname = imageList[0].scienceExt #assert(extname is not None and extname != '') catfile = catfile.strip() mfindx = parse_at_file(fname=catfile, default_ext=('SCI', '*'), default_mask_ext=0, clobber=False, fnamesOnly=True, doNotOpenDQ=True, match2Images=user_fnames, im_fmode='update', dq_fmode='readonly', msk_fmode='readonly', logfile=MultiFileLog(console=True), verbose=True) for p in mfindx: filemaskinfos[p[1]] = p[0] # step through the list of input images and create # combined (static + DQ + user supplied, if any) mask, and # create a list of FileExtMaskInfo objects to be passed # to 'skymatch' function. # # This needs to be done in several steps, mostly due to the fact that # the mask catalogs use "original" (e.g., GEIS, WAIVER FITS) file names # while ultimately we want to open the version converted to MEF. Second # reason is that we want to combine user supplied masks with DQ+static # masks provided by astrodrizzle. new_fi = [] sky_bits = bitmask.interpret_bits_value(paramDict['sky_bits']) for i in range(nimg): # extract extension information: extname = imageList[i].scienceExt extver = imageList[i].group if extver is None: extver = imageList[i].getExtensions() assert (extname is not None and extname != '') assert (extver) # create a new FileExtMaskInfo object fi = FileExtMaskInfo(default_ext=(extname, '*'), default_mask_ext=0, clobber=False, doNotOpenDQ=True, fnamesOnly=False, im_fmode='update', dq_fmode='readonly', msk_fmode='readonly') # set image file and extensions: fi.image = loaded_fnames[i] extlist = [(extname, ev) for ev in extver] fi.append_ext(extlist) # set user masks if any (this will open the files for a later use): fi0 = filemaskinfos[i] if fi0 is not None: nmask = len(fi0.mask_images) for m in range(nmask): mask = fi0.mask_images[m] ext = fi0.maskext[m] fi.append_mask(mask, ext) fi.finalize() # combine user masks with static masks: assert (len(extlist) == fi.count ) #TODO: <-- remove after thorough testing masklist = [] mextlist = [] for k in range(fi.count): if fi.mask_images[k].closed: umask = None else: umask = fi.mask_images[k].hdu[fi.maskext[k]].data (mask, mext) = _buildStaticDQUserMask(imageList[i], extlist[k], sky_bits, paramDict['use_static'], fi.mask_images[k], fi.maskext[k], in_memory) masklist.append(mask) mextlist.append(mext) # replace the original user-supplied masks with the # newly computed combined static+DQ+user masks: fi.clear_masks() for k in range(fi.count): if in_memory and mask is not None: # os.stat() on the "original_fname" of the mask will fail # since this is a "virtual" mask. Therefore we need to compute # mask_stat ourselves. We will simply use id(data) for this: mstat = os.stat_result((0, id(mask.hdu)) + 8 * (0, )) fi.append_mask(masklist[k], mextlist[k], mask_stat=mstat) else: fi.append_mask(masklist[k], mextlist[k]) if masklist[k]: masklist[k].release() fi.finalize() new_fi.append(fi) # Run skymatch algorithm: skymatch(new_fi, skymethod=paramDict['skymethod'], skystat=paramDict['skystat'], lower=paramDict['skylower'], upper=paramDict['skyupper'], nclip=paramDict['skyclip'], lsigma=paramDict['skylsigma'], usigma=paramDict['skyusigma'], binwidth=paramDict['skywidth'], skyuser_kwd=skyKW, units_kwd='BUNIT', readonly=not paramDict['skysub'], dq_bits=None, optimize='inmemory' if in_memory else 'balanced', clobber=True, clean=clean, verbose=True, flog=MultiFileLog(console=True, enableBold=False), _taskname4history='AstroDrizzle') # Populate 'subtractedSky' and 'computedSky' of input image objects: for i in range(nimg): assert (not new_fi[i].fnamesOnly and not new_fi[i].image.closed) image = imageList[i] skysubimage = new_fi[i].image.hdu numchips = image._numchips extname = image.scienceExt assert (os.path.samefile(image._filename, skysubimage.filename())) for extver in range(1, numchips + 1, 1): chip = image[extname, extver] if not chip.group_member: continue subtracted_sky = skysubimage[extname, extver].header[skyKW] chip.subtractedSky = subtracted_sky chip.computedSky = subtracted_sky # clean-up: for fi in new_fi: fi.release_all_images()
def destripe_plus(inputfile, suffix='strp', stat='pmode1', maxiter=15, sigrej=2.0, lower=None, upper=None, binwidth=0.3, scimask1=None, scimask2=None, dqbits=None, rpt_clean=0, atol=0.01, cte_correct=True, clobber=False, verbose=True): """Calibrate post-SM4 ACS/WFC exposure(s) and use standalone :ref:`acsdestripe`. This takes a RAW image and generates a FLT file containing its calibrated and destriped counterpart. If CTE correction is performed, FLC will also be present. Parameters ---------- inputfile : str or list of str Input filenames in one of these formats: * a Python list of filenames * a partial filename with wildcards ('\*raw.fits') * filename of an ASN table ('j12345670_asn.fits') * an at-file (``@input``) suffix : str The string to use to add to each input file name to indicate an output product of ``acs_destripe``. This only affects the intermediate output file that will be automatically renamed to ``*blv_tmp.fits`` during the processing. stat : { 'pmode1', 'pmode2', 'mean', 'mode', 'median', 'midpt' } (Default = 'pmode1') Specifies the statistics to be used for computation of the background in image rows: * 'pmode1' - SEXTRACTOR-like mode estimate based on a modified `Pearson's rule <http://en.wikipedia.org/wiki/Nonparametric_skew#Pearson.27s_rule>`_: ``2.5*median-1.5*mean``; * 'pmode2' - mode estimate based on `Pearson's rule <http://en.wikipedia.org/wiki/Nonparametric_skew#Pearson.27s_rule>`_: ``3*median-2*mean``; * 'mean' - the mean of the distribution of the "good" pixels (after clipping, masking, etc.); * 'mode' - the mode of the distribution of the "good" pixels; * 'median' - the median of the distribution of the "good" pixels; * 'midpt' - estimate of the median of the distribution of the "good" pixels based on an algorithm similar to IRAF's `imagestats` task (``CDF(midpt)=1/2``). .. note:: The midpoint and mode are computed in two passes through the image. In the first pass the standard deviation of the pixels is calculated and used with the *binwidth* parameter to compute the resolution of the data histogram. The midpoint is estimated by integrating the histogram and computing by interpolation the data value at which exactly half the pixels are below that data value and half are above it. The mode is computed by locating the maximum of the data histogram and fitting the peak by parabolic interpolation. maxiter : int This parameter controls the maximum number of iterations to perform when computing the statistics used to compute the row-by-row corrections. sigrej : float This parameters sets the sigma level for the rejection applied during each iteration of statistics computations for the row-by-row corrections. lower : float, None (Default = None) Lower limit of usable pixel values for computing the background. This value should be specified in the units of the input image(s). upper : float, None (Default = None) Upper limit of usable pixel values for computing the background. This value should be specified in the units of the input image(s). binwidth : float (Default = 0.1) Histogram's bin width, in sigma units, used to sample the distribution of pixel brightness values in order to compute the background statistics. This parameter is aplicable *only* to *stat* parameter values of `'mode'` or `'midpt'`. clobber : bool Specify whether or not to 'clobber' (delete then replace) previously generated products with the same names. scimask1 : str or list of str Mask images for *calibrated* ``SCI,1``, one for each input file. Pixels with zero values will be masked out, in addition to clipping. scimask2 : str or list of str Mask images for *calibrated* ``SCI,2``, one for each input file. Pixels with zero values will be masked out, in addition to clipping. This is not used for subarrays. dqbits : int, str, None (Default = None) Integer sum of all the DQ bit values from the input image's DQ array that should be considered "good" when building masks for de-striping computations. For example, if pixels in the DQ array can be combinations of 1, 2, 4, and 8 flags and one wants to consider DQ "defects" having flags 2 and 4 as being acceptable for de-striping computations, then `dqbits` should be set to 2+4=6. Then a DQ pixel having values 2,4, or 6 will be considered a good pixel, while a DQ pixel with a value, e.g., 1+2=3, 4+8=12, etc. will be flagged as a "bad" pixel. Alternatively, one can enter a comma- or '+'-separated list of integer bit flags that should be added to obtain the final "good" bits. For example, both ``4,8`` and ``4+8`` are equivalent to setting `dqbits` to 12. | Set `dqbits` to 0 to make *all* non-zero pixels in the DQ mask to be considered "bad" pixels, and the corresponding image pixels not to be used for de-striping computations. | Default value (`None`) will turn off the use of image's DQ array for de-striping computations. | In order to reverse the meaning of the `dqbits` parameter from indicating values of the "good" DQ flags to indicating the "bad" DQ flags, prepend '~' to the string value. For example, in order not to use pixels with DQ flags 4 and 8 for sky computations and to consider as "good" all other pixels (regardless of their DQ flag), set `dqbits` to ``~4+8``, or ``~4,8``. To obtain the same effect with an `int` input value (except for 0), enter -(4+8+1)=-9. Following this convention, a `dqbits` string value of ``'~0'`` would be equivalent to setting ``dqbits=None``. .. note:: DQ masks (if used), *will be* combined with user masks specified in the `scimask1` and `scimask2` parameters (if any). rpt_clean : int An integer indicating how many *additional* times stripe cleaning should be performed on the input image. Default = 0. atol : float, None The threshold for maximum absolute value of bias stripe correction below which repeated cleanings can stop. When `atol` is `None` cleaning will be repeated `rpt_clean` number of times. Default = 0.01 [e]. cte_correct : bool Perform CTE correction. verbose : bool Print informational messages. Default = True. Raises ------ IOError Input file does not exist. ValueError Invalid header values or CALACS version. """ # process input file(s) and if we have multiple input files - recursively # call acs_destripe_plus for each input image: flist = parseinput.parseinput(inputfile)[0] if isinstance(scimask1, str): mlist1 = parseinput.parseinput(scimask1)[0] elif isinstance(scimask1, np.ndarray): mlist1 = [ scimask1.copy() ] elif scimask1 is None: mlist1 = [] elif isinstance(scimask1, list): mlist1 = [] for m in scimask1: if isinstance(m, np.ndarray): mlist1.append(m.copy()) elif isinstance(m, str): mlist1 += parseinput.parseinput(m)[0] else: raise TypeError("'scimask1' must be a list of str or " "numpy.ndarray values.") else: raise TypeError("'scimask1' must be either a str, or a " "numpy.ndarray, or a list of the two type of " "values.") if isinstance(scimask2, str): mlist2 = parseinput.parseinput(scimask2)[0] elif isinstance(scimask2, np.ndarray): mlist2 = [ scimask2.copy() ] elif scimask2 is None: mlist2 = [] elif isinstance(scimask2, list): mlist2 = [] for m in scimask2: if isinstance(m, np.ndarray): mlist2.append(m.copy()) elif isinstance(m, str): mlist2 += parseinput.parseinput(m)[0] else: raise TypeError("'scimask2' must be a list of str or " "numpy.ndarray values.") else: raise TypeError("'scimask2' must be either a str, or a " "numpy.ndarray, or a list of the two type of " "values.") n_input = len(flist) n_mask1 = len(mlist1) n_mask2 = len(mlist2) if n_input == 0: raise ValueError( 'No input file(s) provided or the file(s) do not exist') if n_mask1 == 0: mlist1 = [None] * n_input elif n_mask1 != n_input: raise ValueError('Insufficient masks for [SCI,1]') if n_mask2 == 0: mlist2 = [None] * n_input elif n_mask2 != n_input: raise ValueError('Insufficient masks for [SCI,2]') if n_input > 1: for img, mf1, mf2 in zip(flist, mlist1, mlist2): destripe_plus( inputfile=img, suffix=suffix, stat=stat, lower=lower, upper=upper, binwidth=binwidth, maxiter=maxiter, sigrej=sigrej, scimask1=scimask1, scimask2=scimask2, dqbits=dqbits, cte_correct=cte_correct, clobber=clobber, verbose=verbose ) return inputfile = flist[0] scimask1 = mlist1[0] scimask2 = mlist2[0] # verify that the RAW image exists in cwd cwddir = os.getcwd() if not os.path.exists(os.path.join(cwddir, inputfile)): raise IOError("{0} does not exist.".format(inputfile)) # get image's primary header: header = fits.getheader(inputfile) # verify masks defined (or not) simultaneously: if header['CCDAMP'] == 'ABCD' and \ ((scimask1 is not None and scimask2 is None) or \ (scimask1 is None and scimask2 is not None)): raise ValueError("Both 'scimask1' and 'scimask2' must be specified " "or not specified together.") calacs_str = subprocess.check_output(['calacs.e', '--version']).split()[0] calacs_ver = [int(x) for x in calacs_str.decode().split('.')] if calacs_ver < [8, 3, 1]: raise ValueError('CALACS {0} is incomptible. ' 'Must be 8.3.1 or later.'.format(calacs_str)) # check date for post-SM4 and if 2K subarray or full frame is_sub2K = False ctecorr = header['PCTECORR'] aperture = header['APERTURE'] detector = header['DETECTOR'] date_obs = Time(header['DATE-OBS']) # intermediate filenames blvtmp_name = inputfile.replace('raw', 'blv_tmp') blctmp_name = inputfile.replace('raw', 'blc_tmp') # output filenames tra_name = inputfile.replace('_raw.fits', '.tra') flt_name = inputfile.replace('raw', 'flt') flc_name = inputfile.replace('raw', 'flc') if detector != 'WFC': raise ValueError("{0} is not a WFC image, please check the 'DETECTOR'" " keyword.".format(inputfile)) if date_obs < SM4_DATE: raise ValueError( "{0} is a pre-SM4 image.".format(inputfile)) if header['SUBARRAY'] and cte_correct: if aperture in SUBARRAY_LIST: is_sub2K = True else: LOG.warning('Using non-2K subarray, turning CTE correction off') cte_correct = False # delete files from previous CALACS runs if clobber: for tmpfilename in [blvtmp_name, blctmp_name, flt_name, flc_name, tra_name]: if os.path.exists(tmpfilename): os.remove(tmpfilename) # run ACSCCD on RAW subarray acsccd.acsccd(inputfile) # modify user mask with DQ masks if requested dqbits = bitmask.interpret_bits_value(dqbits) if dqbits is not None: # save 'tra' file in memory to trick the log file # not to save first acs2d log as this is done only # for the purpose of obtaining DQ masks. # WISH: it would have been nice is there was an easy way of obtaining # just the DQ masks as if data were calibrated but without # having to recalibrate them with acs2d. if os.path.isfile(tra_name): fh = open(tra_name) tra_lines = fh.readlines() fh.close() else: tra_lines = None # apply flats, etc. acs2d.acs2d(blvtmp_name, verbose=False, quiet=True) # extract DQ arrays from the FLT image: dq1, dq2 = _read_DQ_arrays(flt_name) if isinstance(scimask1, str): if scimask1.strip() is '': mask1 = None scimask1 = None else: mask1 = fits.getdata(scimask1) elif isinstance(scimask1, np.ndarray): mask1 = scimask1.copy() elif scimask1 is None: mask1 = None else: raise TypeError("'scimask1' must be either a str file name, " "a numpy.ndarray, or None.") scimask1 = acs_destripe._mergeUserMaskAndDQ(dq1, mask1, dqbits) if isinstance(scimask2, str): if scimask2.strip() is '': mask2 = None scimask2 = None else: mask2 = fits.getdata(scimask2) elif isinstance(scimask2, np.ndarray): mask2 = scimask2.copy() elif scimask2 is None: mask2 = None else: raise TypeError("'scimask2' must be either a str file name, " "a numpy.ndarray, or None.") if dq2 is not None: scimask2 = acs_destripe._mergeUserMaskAndDQ(dq2, mask2, dqbits) # reconstruct trailer file: if tra_lines is not None: fh = open(tra_name, mode='w') fh.writelines(tra_lines) fh.close() # delete temporary FLT image: if os.path.isfile(flt_name): os.remove(flt_name) # execute destriping of the subarray (post-SM4 data only) acs_destripe.clean( blvtmp_name, suffix, stat=stat, maxiter=maxiter, sigrej=sigrej, lower=lower, upper=upper, binwidth=binwidth, mask1=scimask1, mask2=scimask2, dqbits=dqbits, rpt_clean=rpt_clean, atol=atol, clobber=clobber, verbose=verbose) blvtmpsfx = 'blv_tmp_{0}'.format(suffix) os.rename(inputfile.replace('raw', blvtmpsfx), blvtmp_name) # update subarray header if is_sub2K and cte_correct: fits.setval(blvtmp_name, 'PCTECORR', value='PERFORM') ctecorr = 'PERFORM' # perform CTE correction on destriped image if cte_correct: if ctecorr == 'PERFORM': acscte.acscte(blvtmp_name) else: LOG.warning( "PCTECORR={0}, cannot run CTE correction".format(ctecorr)) cte_correct = False # run ACS2D to get FLT and FLC images acs2d.acs2d(blvtmp_name) if cte_correct: acs2d.acs2d(blctmp_name) # delete intermediate files os.remove(blvtmp_name) if cte_correct: os.remove(blctmp_name) info_str = 'Done.\nFLT: {0}\n'.format(flt_name) if cte_correct: info_str += 'FLC: {0}\n'.format(flc_name) LOG.info(info_str)