def __init__(self,obs,delFilters=_delFiltsDef,excludefilt=None,noContext=None): """ constructor gets an Observation object (obs) from the pipeline. """ self.modName = string.split(string.split(str(self))[0],'.')[0][1:] self.root = obs.newobspath # root path of the observation dir self.obsName = obs.newobs self.obsPars = obs.newpar self.obsFits = obs.newfits # path to fits files self.messagedir = obs.messagedir # where the module message will go. self.configdir = obs.configdir self.logfile = obs.logfile self.defaultInParFile = os.path.join(obs.pardir,self.modName + '.inpar') self.defaultParamFile = os.path.join(obs.pardir,self.modName + '.param') self.inParFileName = os.path.join(self.obsPars, 'noisy.inpar') self.inpars = pUtil.readInParFile(self.defaultInParFile) # readInParFile returns a dict self.detImName = 'detectionImage.fits' self.detWgtName = 'detectionWeight.fits' self.edgeMaskName = obs.edgeMaskName self.errorList = [] self.inputList = [] self.outputList = {} self.logfile.write(self.modName+": Instantiated detectionImage object.") # excludefilt must be a list of ACS filter names # i.e. --excludefilt=['F502N','F475W'] # in a pipeline context, this form will be passed to the constructor. self.excludeFilters = excludefilt if noContext: checkLists = [obs.sciImageList,obs.weightImageList,obs.flagImageList,obs.rmsImageList] else: checkLists = [obs.sciImageList,obs.contextImageList,obs.weightImageList,obs.flagImageList,obs.rmsImageList] # remove drizzled grism images and other oddities from various lists and further processing if delFilters: self.logfile.write("Checking for input images having filters: "+str(delFilters)) for _obsList in checkLists: for ii in range(len(_obsList)-1,-1,-1): im = _obsList[ii] imfilt1 = fUtil.getFilter1(self.obsFits+'/'+im) imfilt2 = fUtil.getFilter2(self.obsFits+'/'+im) if imfilt1 in delFilters or imfilt2 in delFilters: self.logfile.write(im+" ("+imfilt+") removed from further obs processing.") del _obsList[ii] # now it belongs to the ages for _obsList in checkLists: if len(_obsList) != len(obs.sciImageList): errtxt = "Error: inconsistency in Image Lists after removing unwanted filter images." self.logfile.write(errtxt) raise Exception,errtxt self.sciImageList = obs.sciImageList for i in self.sciImageList: self.inputList.append(i) if len(self.sciImageList) == 0: raise detectionImageError("Image list has no images to make detection image.")
def writeXml(self): """ writeXml calls the xmlUtil catalog functions on a list of source catalogs produced by SExtractor of the observation object and also calls the markupImage function (xmlUtil) on the segmentation images produced by SExtractor. New requirements for the markup now mean the pipeline has to include the filter name in the catalog tag of the xml file. filterName func returns the _useful_ filter name, i.e. the one that is not clear. This is now part of the argument list that goes to the xmlStartCat function. A new requirement has been placed upon this writeXml method to allow certain fields of a catalog file to be excluded from the xml markup. This is done to avoid redundant fields for the same objects being sent to the SDA. See Bugzilla bug #1436. """ self.logfile.write( 'Generating the XML catalog(s) from SExtractor output.') lcount = 0 for catfile in self.catalogList: base = string.split(catfile, '.')[0] # the following is done to just get the filter name out of the # corresponding image. dir, file = os.path.split(catfile) image = os.path.splitext(file)[0] + '.fits' fullfile = os.path.join(self.obsFits, image) if self.excludeList: self.logfile.write( "N.B. The following fields are being excluded from the xml output of " + file + ":") for field in self.excludeList: self.logfile.write(field) xmlCatalogName = base + '.xml' pardict = pUtil.readInParFile(self.inParFileList[lcount]) xmlUtil.xmlStartCat(xmlCatalogName, self.obsName, imgfile=fullfile) try: xmlUtil.xmlStartConfig(xmlCatalogName) except Exception, err: self.errorList.append((self.modName, str(err))) self.logfile.write( 'function xmlStartConfig encountered a problem.') raise Exception(err) try: xmlUtil.xmlPars(pardict, xmlCatalogName) self.logfile.write( 'Input Paramaters prepended to xml catalog file.') except Exception, err: self.errorList.append((self.modName, str(err))) self.logfile.write('function xmlPars encountered a problem.') raise Exception(err)
def __init__(self,obs,Band=None,excludeFilters=None,noContext=None): """ constructor gets an Observation object (obs) from the pipeline. """ self.modName = string.split(string.split(str(self))[0],'.')[0][1:] self.root = obs.newobspath # root path of the observation dir self.obsName = obs.newobs self.obsPars = obs.newpardir self.obsFits = obs.fitsdir # path to fits files self.messagedir = obs.messagedir # where the module message will go. self.configdir = obs.configdir self.logfile = obs.logfile self.defaultInParFile = os.path.join(obs.pardir,self.modName + '_'+ Band + '.inpar') #WZ self.defaultParamFile = os.path.join(obs.pardir,self.modName + '.param') self.inParFileName = os.path.join(self.obsPars, 'noisy.inpar') self.inpars = pUtil.readInParFile(self.defaultInParFile) # readInParFile returns a dict # obs.dethresh =self.inpar['DETECT_THRESH'] # obs.anathresh =self.inpar['ANALYSIS_THRESH'] # self.detImName = 'detectionImage.fits' # self.detWgtName = 'detectionWeight.fits' self.edgeMaskName = obs.edgeMaskName = 'Edgemask.fits' #XX self.errorList = [] self.inputList = [] self.outputList = {} if Band=='nir': #WZ self.detImName = 'detection_nir.fits' self.detWgtName = 'detection_nir_wht.fits' Detector='IR' elif Band=='red': # only choose the last infared filter F160W F140W by XX self.detImName = 'detection_red.fits' self.detWgtName = 'detection_red_wht.fits' Detector='IR' elif Band=='opt': self.detImName = 'detection_opt.fits' self.detWgtName = 'detection_opt_wht.fits' Detector='WFC' elif Band=='uv': self.detImName = 'detection_uv.fits' self.detWgtName = 'detection_uv_wht.fits' Detector='UVIS' else: self.logfile.write("Incorrect band name: "+Band) exit() self.logfile.write(self.modName+": Instantiated detectionImage object.") # excludeFilters must be a list of filter names # i.e. --excludefilt=['F850LP','F225W'] # in a pipeline context, this form will be passed to the constructor. self.excludeFilters = excludeFilters self.det = Detector if noContext: checkLists = [obs.sciImageList,obs.weightImageList,obs.flagImageList,obs.rmsImageList] else: checkLists = [obs.sciImageList,obs.contextImageList,obs.weightImageList,obs.flagImageList,obs.rmsImageList] # remove drizzled grism images and other oddities from various lists and further processing self.logfile.write("Checking for input images not with detector: "+str(Detector)) for _obsList in checkLists: for ii in range(len(_obsList)-1,-1,-1): im = _obsList[ii] imfilt = fUtil.getFilter(self.obsFits+'/'+im) if imfilt == _delFiltsDef: self.logfile.write(im+" ("+imfilt+") removed from further obs processing.") del _obsList[ii] # now it belongs to the ages #pdb.set_trace() #XX for _obsList in checkLists: if len(_obsList) != len(obs.sciImageList): errtxt = "Error: inconsistency in Image Lists after removing unwanted detector images." self.logfile.write(errtxt) raise Exception,errtxt self.sciImageList = obs.sciImageList for i in self.sciImageList: self.inputList.append(i) if len(self.sciImageList) == 0: raise detectionImageError("Image list has no images to make detection image.")
def __init__(self, obs, richField=None, excludeList=None): self.modName = string.split(string.split(str(self))[0], '.')[0][1:] # module name self.root = obs.newobspath # root path of the observation dir self.configdir = obs.configdir self.obsName = obs.newobs self.obsCats = obs.newcats self.obsPars = obs.newpar self.obsFits = obs.newfits # path to fits files self.messagedir = obs.messagedir # where the module message will go. self.obsFitsList = obs.fitslist # list of fits files in newfits self.sciImageList = obs.sciImageList # list of dither-combined filter images self.flagImageList = obs.flagImageList self.rmsImageList = obs.rmsImageList self.logfile = obs.logfile # append log entries to this file self.errorList = [] self.inputList = [] self.outputList = {} self.checkimagesList = [] # used for the markup self.inParFileList = [] # used for the markup self.excludeList = excludeList # used for the markup self.filterCorrections = { } # dictionary of image names/filter extinction corrections self.logfile.write(self.modName + " object instantiated.") print "Making catalogs" #pdb.set_trace() # full path to detection Image #self.detectionImage = os.path.join(self.obsFits,'detectionImage.fits') self.detectionImage = os.path.join(self.obsFits, 'detection_' + obs.band + '.fits') #by xingxing ## Bugzilla bug #2729 - Extinction correction to be done on all magnitudes in filter catalogs ## ## This bug specifies that extinction correction should be done for all magnitudes in all filter ## catalogs. This is done by adjusting the MAG_ZEROPOINT parameter for each filter's parameter ## set (.inpar file). # Now extract the RA and Dec positions (CRVAL1 and CRVAL2 from the header) from the # detection Image and determine E(B-V) at that position. ra = float(fUtil.getKeyVal(self.detectionImage, "CRVAL1")) dec = float(fUtil.getKeyVal(self.detectionImage, "CRVAL2")) # getExtinction function returns a tuple of strings # (extinction_correction, data_quality_flags) self.logfile.write("Determining E(B-V) for detection Image coords:" + str(ra) + ", " + str(dec)) extCorr, dqFlags = extinction.getEBV(ra, dec) # convert the extinction correction, extCorr to a float for use in arithmetic self.logfile.write("E(B-V) determined: " + extCorr) self.eBV = float(extCorr) self.eBV_error = self.eBV * 0.16 # This is the expected uncertainty in E(B-V): 16% # make a list of the data quality flags self.dqList = dqFlags.strip().split()[2:] # Define different input parameters for different detectors # This requirement is described in Buzilla bug #1188 # # Additionally, see Bugzilla bug #2774, user selectable parameter set for rich-field WFC # data. Code is in place to handle an anticipated HRC parameter set, though the spec # for such an HRC parameter set may not be developed. print "Please check the [obs.detector]!! Make sure the right file is loaded! [catalog.py] ___xingxing" #by xingxing #pdb.set_trace() if (string.find(string.upper(obs.detector), 'WFC') > -1 and string.find(string.upper(obs.detector), 'IR') == -1): if richField: self.defaultInParFile = os.path.join( obs.pardir, self.modName + '_wfc_richfield.inpar') self.logfile.write( "Selected rich field WFC input parameter set.") else: self.defaultInParFile = os.path.join( obs.pardir, self.modName + '_wfc_sparsefield.inpar') self.logfile.write( "Selected WFC sparse field input parameter set for SExtractor." ) elif (string.find(string.upper(obs.detector), 'HRC') > -1): if richField: self.defaultInParFile = os.path.join( obs.pardir, self.modName + '_hrc_richfield.inpar') self.logfile.write( "Selected rich field HRC input parameter set.") else: self.defaultInParFile = os.path.join( obs.pardir, self.modName + '_hrc_sparsefield.inpar') self.logfile.write( "Selected sparse field HRC input parameter set for SExtractor." ) elif (string.find(string.upper(obs.detector), 'UVIS') > -1): if richField: self.defaultInParFile = os.path.join( obs.pardir, self.modName + '_uvis_richfield.inpar') self.logfile.write( "Selected rich field UVIS input parameter set.") else: self.defaultInParFile = os.path.join( obs.pardir, self.modName + '_uvis_sparsefield.inpar') self.logfile.write( "Selected UVIS sparse field input parameter set for SExtractor." ) elif (string.find(string.upper(obs.detector), 'IR') > -1): if richField: self.defaultInParFile = os.path.join( obs.pardir, self.modName + '_ir_richfield.inpar') self.logfile.write( "Selected rich field IR input parameter set.") else: self.defaultInParFile = os.path.join( obs.pardir, self.modName + '_ir_sparsefield.inpar') self.logfile.write( "Selected sparse field IR input parameter set for SExtractor." ) else: self.defaultInParFile = os.path.join(obs.pardir, self.modName + '.inpar') self.logfile.write( "Detector is not WFC,HRC or UVIS,IR. Using default input parameter set." ) #output parameters are the same for any detector. self.defaultParamFile = os.path.join(obs.pardir, self.modName + '.param') # default .param file self.inpars = pUtil.readInParFile( self.defaultInParFile) # readInParFile returns a dict self.outpars = pUtil.readParamFile( self.defaultParamFile) # readParamFile returns a list of tuples # Some checking of images before proceeding. if not self.sciImageList: errtxt = "No Science Images listed in attribute sciImageList." self.errorList.append((self.modName, errtxt)) raise AttributeError, errtxt if not self.flagImageList: errtxt = "No Flag Images listed in attribute flagImageList." self.errorList.append((self.modName, )) raise AttributeError, errtxt if len(self.sciImageList) != len(self.flagImageList): errtxt = "Number of Flag Images does not match the number of Science Images." self.errorList.append((self.modName, errtxt)) raise AttributeError, errtxt for im in self.sciImageList: self.inputList.append(os.path.basename(im)) for im in self.flagImageList: self.inputList.append(os.path.basename(im))
def __init__(self, obs, richField=None, excludeList=None): self.modName = string.split(string.split(str(self))[0], '.')[0][1:] # module name self.root = obs.newobspath # root path of the observation dir self.configdir = obs.configdir self.obsName = obs.newobs self.obsCats = obs.catdir self.obsPars = obs.newpardir self.obsFits = obs.fitsdir # path to fits files self.messagedir = obs.messagedir # where the module message will go. self.obsFitsList = obs.fitslist # list of fits files in newfits self.sciImageList = obs.sciImageList # list of dither-combined filter images self.flagImageList = obs.flagImageList self.rmsImageList = obs.rmsImageList self.logfile = obs.logfile # append log entries to this file self.errorList = [] self.inputList = [] self.outputList = {} self.checkimagesList = [] # used for the markup self.inParFileList = [] # used for the markup self.excludeList = excludeList # used for the markup self.filterCorrections = { } # dictionary of image names/filter extinction corrections self.logfile.write(self.modName + " object instantiated.") # full path to detection Image self.detectionImage = os.path.join(self.obsFits, 'detection_' + obs.band + '.fits') # if obs.detector=='WFC': # self.detectionImage = os.path.join(self.obsFits,'detection_opt.fits') # elif obs.detector=='IR': # self.detectionImage = os.path.join(self.obsFits,'detection_nir.fits') # Now extract the RA and Dec positions (CRVAL1 and CRVAL2 from the header) from the # detection Image and determine E(B-V) at that position. ra = float(fUtil.getKeyVal(self.detectionImage, "CRVAL1")) dec = float(fUtil.getKeyVal(self.detectionImage, "CRVAL2")) # getExtinction function returns a tuple of strings # (extinction_correction, data_quality_flags) self.logfile.write("Determining E(B-V) for detection Image coords: " + str(ra) + ", " + str(dec)) #pdb.set_trace() extCorr, dqFlags = extinction.getEBV(ra, dec) # convert the extinction correction, extCorr to a float for use in arithmetic self.logfile.write("E(B-V) determined: " + extCorr) self.eBV = float(extCorr) self.eBV_error = self.eBV * 0.16 # This is the expected uncertainty in E(B-V): 16% # make a list of the data quality flags self.dqList = dqFlags.strip().split()[2:] # Define different input parameters for different detectors # This requirement is described in Buzilla bug #1188 # WZ: This has been changed, as the selection is independent # of detector type... # self.defaultInParFile = os.path.join(obs.pardir,self.modName+'.inpar') if (obs.detector == 'WFC'): self.defaultInParFile = os.path.join(obs.pardir, self.modName + '_opt.inpar') elif (obs.detector == 'IR'): self.defaultInParFile = os.path.join(obs.pardir, self.modName + '_nir.inpar') elif (obs.detector == 'UVIS'): self.defaultInParFile = os.path.join(obs.pardir, self.modName + '_uv.inpar') # self.defaultInParFile = os.path.join(obs.pardir,self.modName+'.inpar') self.logfile.write("Selected input parameter set." + self.defaultInParFile) #output parameters are the same for any detector. self.defaultParamFile = os.path.join(obs.pardir, self.modName + '.param') # default .param file self.inpars = pUtil.readInParFile( self.defaultInParFile) # readInParFile returns a dict self.outpars = pUtil.readParamFile( self.defaultParamFile) # readParamFile returns a list of tuples # Some checking of images before proceeding. if not self.sciImageList: errtxt = "No Science Images listed in attribute sciImageList." self.errorList.append((self.modName, errtxt)) raise AttributeError, errtxt if not self.flagImageList: errtxt = "No Flag Images listed in attribute flagImageList." self.errorList.append((self.modName, )) raise AttributeError, errtxt if len(self.sciImageList) != len(self.flagImageList): errtxt = "Number of Flag Images does not match the number of Science Images." self.errorList.append((self.modName, errtxt)) raise AttributeError, errtxt for im in self.sciImageList: self.inputList.append(os.path.basename(im)) for im in self.flagImageList: self.inputList.append(os.path.basename(im))