def __init__(self,obs,delFilters=_delFiltsDef,excludefilt=None,noContext=None):
""" constructor gets an Observation object (obs) from the pipeline. """
self.modName = string.split(string.split(str(self))[0],'.')[0][1:]
self.root = obs.newobspath # root path of the observation dir
self.obsName = obs.newobs
self.obsPars = obs.newpar
self.obsFits = obs.newfits # path to fits files
self.messagedir = obs.messagedir # where the module message will go.
self.configdir = obs.configdir
self.logfile = obs.logfile
self.defaultInParFile = os.path.join(obs.pardir,self.modName + '.inpar')
self.defaultParamFile = os.path.join(obs.pardir,self.modName + '.param')
self.inParFileName = os.path.join(self.obsPars, 'noisy.inpar')
self.inpars = pUtil.readInParFile(self.defaultInParFile) # readInParFile returns a dict
self.detImName = 'detectionImage.fits'
self.detWgtName = 'detectionWeight.fits'
self.edgeMaskName = obs.edgeMaskName
self.errorList = []
self.inputList = []
self.outputList = {}
self.logfile.write(self.modName+": Instantiated detectionImage object.")
# excludefilt must be a list of ACS filter names
# i.e. --excludefilt=['F502N','F475W']
# in a pipeline context, this form will be passed to the constructor.
self.excludeFilters = excludefilt
if noContext:
checkLists = [obs.sciImageList,obs.weightImageList,obs.flagImageList,obs.rmsImageList]
else:
checkLists = [obs.sciImageList,obs.contextImageList,obs.weightImageList,obs.flagImageList,obs.rmsImageList]
# remove drizzled grism images and other oddities from various lists and further processing
if delFilters:
self.logfile.write("Checking for input images having filters: "+str(delFilters))
for _obsList in checkLists:
for ii in range(len(_obsList)-1,-1,-1):
im = _obsList[ii]
imfilt1 = fUtil.getFilter1(self.obsFits+'/'+im)
imfilt2 = fUtil.getFilter2(self.obsFits+'/'+im)
if imfilt1 in delFilters or imfilt2 in delFilters:
self.logfile.write(im+" ("+imfilt+") removed from further obs processing.")
del _obsList[ii] # now it belongs to the ages
for _obsList in checkLists:
if len(_obsList) != len(obs.sciImageList):
errtxt = "Error: inconsistency in Image Lists after removing unwanted filter images."
self.logfile.write(errtxt)
raise Exception,errtxt
self.sciImageList = obs.sciImageList
for i in self.sciImageList:
self.inputList.append(i)
if len(self.sciImageList) == 0:
raise detectionImageError("Image list has no images to make detection image.")
def writeXml(self):
"""
writeXml calls the xmlUtil catalog functions on a list of source catalogs
produced by SExtractor of the observation object and also calls the
markupImage function (xmlUtil) on the segmentation images produced
by SExtractor. New requirements for the markup now mean the pipeline
has to include the filter name in the catalog tag of the xml file.
filterName func returns the _useful_ filter name, i.e. the one that is
not clear. This is now part of the argument list that goes to the
xmlStartCat function. A new requirement has been placed upon
this writeXml method to allow certain fields of a catalog file
to be excluded from the xml markup. This is done to avoid redundant
fields for the same objects being sent to the SDA. See Bugzilla
bug #1436.
"""
self.logfile.write(
'Generating the XML catalog(s) from SExtractor output.')
lcount = 0
for catfile in self.catalogList:
base = string.split(catfile, '.')[0]
# the following is done to just get the filter name out of the
# corresponding image.
dir, file = os.path.split(catfile)
image = os.path.splitext(file)[0] + '.fits'
fullfile = os.path.join(self.obsFits, image)
if self.excludeList:
self.logfile.write(
"N.B. The following fields are being excluded from the xml output of "
+ file + ":")
for field in self.excludeList:
self.logfile.write(field)
xmlCatalogName = base + '.xml'
pardict = pUtil.readInParFile(self.inParFileList[lcount])
xmlUtil.xmlStartCat(xmlCatalogName, self.obsName, imgfile=fullfile)
try:
xmlUtil.xmlStartConfig(xmlCatalogName)
except Exception, err:
self.errorList.append((self.modName, str(err)))
self.logfile.write(
'function xmlStartConfig encountered a problem.')
raise Exception(err)
try:
xmlUtil.xmlPars(pardict, xmlCatalogName)
self.logfile.write(
'Input Paramaters prepended to xml catalog file.')
except Exception, err:
self.errorList.append((self.modName, str(err)))
self.logfile.write('function xmlPars encountered a problem.')
raise Exception(err)
def __init__(self,obs,Band=None,excludeFilters=None,noContext=None):
""" constructor gets an Observation object (obs) from the pipeline. """
self.modName = string.split(string.split(str(self))[0],'.')[0][1:]
self.root = obs.newobspath # root path of the observation dir
self.obsName = obs.newobs
self.obsPars = obs.newpardir
self.obsFits = obs.fitsdir # path to fits files
self.messagedir = obs.messagedir # where the module message will go.
self.configdir = obs.configdir
self.logfile = obs.logfile
self.defaultInParFile = os.path.join(obs.pardir,self.modName + '_'+ Band + '.inpar') #WZ
self.defaultParamFile = os.path.join(obs.pardir,self.modName + '.param')
self.inParFileName = os.path.join(self.obsPars, 'noisy.inpar')
self.inpars = pUtil.readInParFile(self.defaultInParFile) # readInParFile returns a dict
# obs.dethresh =self.inpar['DETECT_THRESH']
# obs.anathresh =self.inpar['ANALYSIS_THRESH']
# self.detImName = 'detectionImage.fits'
# self.detWgtName = 'detectionWeight.fits'
self.edgeMaskName = obs.edgeMaskName = 'Edgemask.fits' #XX
self.errorList = []
self.inputList = []
self.outputList = {}
if Band=='nir': #WZ
self.detImName = 'detection_nir.fits'
self.detWgtName = 'detection_nir_wht.fits'
Detector='IR'
elif Band=='red': # only choose the last infared filter F160W F140W by XX
self.detImName = 'detection_red.fits'
self.detWgtName = 'detection_red_wht.fits'
Detector='IR'
elif Band=='opt':
self.detImName = 'detection_opt.fits'
self.detWgtName = 'detection_opt_wht.fits'
Detector='WFC'
elif Band=='uv':
self.detImName = 'detection_uv.fits'
self.detWgtName = 'detection_uv_wht.fits'
Detector='UVIS'
else:
self.logfile.write("Incorrect band name: "+Band)
exit()
self.logfile.write(self.modName+": Instantiated detectionImage object.")
# excludeFilters must be a list of filter names
# i.e. --excludefilt=['F850LP','F225W']
# in a pipeline context, this form will be passed to the constructor.
self.excludeFilters = excludeFilters
self.det = Detector
if noContext:
checkLists = [obs.sciImageList,obs.weightImageList,obs.flagImageList,obs.rmsImageList]
else:
checkLists = [obs.sciImageList,obs.contextImageList,obs.weightImageList,obs.flagImageList,obs.rmsImageList]
# remove drizzled grism images and other oddities from various lists and further processing
self.logfile.write("Checking for input images not with detector: "+str(Detector))
for _obsList in checkLists:
for ii in range(len(_obsList)-1,-1,-1):
im = _obsList[ii]
imfilt = fUtil.getFilter(self.obsFits+'/'+im)
if imfilt == _delFiltsDef:
self.logfile.write(im+" ("+imfilt+") removed from further obs processing.")
del _obsList[ii] # now it belongs to the ages
#pdb.set_trace() #XX
for _obsList in checkLists:
if len(_obsList) != len(obs.sciImageList):
errtxt = "Error: inconsistency in Image Lists after removing unwanted detector images."
self.logfile.write(errtxt)
raise Exception,errtxt
self.sciImageList = obs.sciImageList
for i in self.sciImageList:
self.inputList.append(i)
if len(self.sciImageList) == 0:
raise detectionImageError("Image list has no images to make detection image.")
def __init__(self, obs, richField=None, excludeList=None):
self.modName = string.split(string.split(str(self))[0],
'.')[0][1:] # module name
self.root = obs.newobspath # root path of the observation dir
self.configdir = obs.configdir
self.obsName = obs.newobs
self.obsCats = obs.newcats
self.obsPars = obs.newpar
self.obsFits = obs.newfits # path to fits files
self.messagedir = obs.messagedir # where the module message will go.
self.obsFitsList = obs.fitslist # list of fits files in newfits
self.sciImageList = obs.sciImageList # list of dither-combined filter images
self.flagImageList = obs.flagImageList
self.rmsImageList = obs.rmsImageList
self.logfile = obs.logfile # append log entries to this file
self.errorList = []
self.inputList = []
self.outputList = {}
self.checkimagesList = [] # used for the markup
self.inParFileList = [] # used for the markup
self.excludeList = excludeList # used for the markup
self.filterCorrections = {
} # dictionary of image names/filter extinction corrections
self.logfile.write(self.modName + " object instantiated.")
print "Making catalogs"
#pdb.set_trace()
# full path to detection Image
#self.detectionImage = os.path.join(self.obsFits,'detectionImage.fits')
self.detectionImage = os.path.join(self.obsFits, 'detection_' +
obs.band + '.fits') #by xingxing
## Bugzilla bug #2729 - Extinction correction to be done on all magnitudes in filter catalogs ##
## This bug specifies that extinction correction should be done for all magnitudes in all filter
## catalogs. This is done by adjusting the MAG_ZEROPOINT parameter for each filter's parameter
## set (.inpar file).
# Now extract the RA and Dec positions (CRVAL1 and CRVAL2 from the header) from the
# detection Image and determine E(B-V) at that position.
ra = float(fUtil.getKeyVal(self.detectionImage, "CRVAL1"))
dec = float(fUtil.getKeyVal(self.detectionImage, "CRVAL2"))
# getExtinction function returns a tuple of strings
# (extinction_correction, data_quality_flags)
self.logfile.write("Determining E(B-V) for detection Image coords:" +
str(ra) + ", " + str(dec))
extCorr, dqFlags = extinction.getEBV(ra, dec)
# convert the extinction correction, extCorr to a float for use in arithmetic
self.logfile.write("E(B-V) determined: " + extCorr)
self.eBV = float(extCorr)
self.eBV_error = self.eBV * 0.16 # This is the expected uncertainty in E(B-V): 16%
# make a list of the data quality flags
self.dqList = dqFlags.strip().split()[2:]
# Define different input parameters for different detectors
# This requirement is described in Buzilla bug #1188
#
# Additionally, see Bugzilla bug #2774, user selectable parameter set for rich-field WFC
# data. Code is in place to handle an anticipated HRC parameter set, though the spec
# for such an HRC parameter set may not be developed.
print "Please check the [obs.detector]!! Make sure the right file is loaded! [catalog.py] ___xingxing" #by xingxing
#pdb.set_trace()
if (string.find(string.upper(obs.detector), 'WFC') > -1
and string.find(string.upper(obs.detector), 'IR') == -1):
if richField:
self.defaultInParFile = os.path.join(
obs.pardir, self.modName + '_wfc_richfield.inpar')
self.logfile.write(
"Selected rich field WFC input parameter set.")
else:
self.defaultInParFile = os.path.join(
obs.pardir, self.modName + '_wfc_sparsefield.inpar')
self.logfile.write(
"Selected WFC sparse field input parameter set for SExtractor."
)
elif (string.find(string.upper(obs.detector), 'HRC') > -1):
if richField:
self.defaultInParFile = os.path.join(
obs.pardir, self.modName + '_hrc_richfield.inpar')
self.logfile.write(
"Selected rich field HRC input parameter set.")
else:
self.defaultInParFile = os.path.join(
obs.pardir, self.modName + '_hrc_sparsefield.inpar')
self.logfile.write(
"Selected sparse field HRC input parameter set for SExtractor."
)
elif (string.find(string.upper(obs.detector), 'UVIS') > -1):
if richField:
self.defaultInParFile = os.path.join(
obs.pardir, self.modName + '_uvis_richfield.inpar')
self.logfile.write(
"Selected rich field UVIS input parameter set.")
else:
self.defaultInParFile = os.path.join(
obs.pardir, self.modName + '_uvis_sparsefield.inpar')
self.logfile.write(
"Selected UVIS sparse field input parameter set for SExtractor."
)
elif (string.find(string.upper(obs.detector), 'IR') > -1):
if richField:
self.defaultInParFile = os.path.join(
obs.pardir, self.modName + '_ir_richfield.inpar')
self.logfile.write(
"Selected rich field IR input parameter set.")
else:
self.defaultInParFile = os.path.join(
obs.pardir, self.modName + '_ir_sparsefield.inpar')
self.logfile.write(
"Selected sparse field IR input parameter set for SExtractor."
)
else:
self.defaultInParFile = os.path.join(obs.pardir,
self.modName + '.inpar')
self.logfile.write(
"Detector is not WFC,HRC or UVIS,IR. Using default input parameter set."
)
#output parameters are the same for any detector.
self.defaultParamFile = os.path.join(obs.pardir, self.modName +
'.param') # default .param file
self.inpars = pUtil.readInParFile(
self.defaultInParFile) # readInParFile returns a dict
self.outpars = pUtil.readParamFile(
self.defaultParamFile) # readParamFile returns a list of tuples
# Some checking of images before proceeding.
if not self.sciImageList:
errtxt = "No Science Images listed in attribute sciImageList."
self.errorList.append((self.modName, errtxt))
raise AttributeError, errtxt
if not self.flagImageList:
errtxt = "No Flag Images listed in attribute flagImageList."
self.errorList.append((self.modName, ))
raise AttributeError, errtxt
if len(self.sciImageList) != len(self.flagImageList):
errtxt = "Number of Flag Images does not match the number of Science Images."
self.errorList.append((self.modName, errtxt))
raise AttributeError, errtxt
for im in self.sciImageList:
self.inputList.append(os.path.basename(im))
for im in self.flagImageList:
self.inputList.append(os.path.basename(im))
def __init__(self, obs, richField=None, excludeList=None):
self.modName = string.split(string.split(str(self))[0],
'.')[0][1:] # module name
self.root = obs.newobspath # root path of the observation dir
self.configdir = obs.configdir
self.obsName = obs.newobs
self.obsCats = obs.catdir
self.obsPars = obs.newpardir
self.obsFits = obs.fitsdir # path to fits files
self.messagedir = obs.messagedir # where the module message will go.
self.obsFitsList = obs.fitslist # list of fits files in newfits
self.sciImageList = obs.sciImageList # list of dither-combined filter images
self.flagImageList = obs.flagImageList
self.rmsImageList = obs.rmsImageList
self.logfile = obs.logfile # append log entries to this file
self.errorList = []
self.inputList = []
self.outputList = {}
self.checkimagesList = [] # used for the markup
self.inParFileList = [] # used for the markup
self.excludeList = excludeList # used for the markup
self.filterCorrections = {
} # dictionary of image names/filter extinction corrections
self.logfile.write(self.modName + " object instantiated.")
# full path to detection Image
self.detectionImage = os.path.join(self.obsFits,
'detection_' + obs.band + '.fits')
# if obs.detector=='WFC':
# self.detectionImage = os.path.join(self.obsFits,'detection_opt.fits')
# elif obs.detector=='IR':
# self.detectionImage = os.path.join(self.obsFits,'detection_nir.fits')
# Now extract the RA and Dec positions (CRVAL1 and CRVAL2 from the header) from the
# detection Image and determine E(B-V) at that position.
ra = float(fUtil.getKeyVal(self.detectionImage, "CRVAL1"))
dec = float(fUtil.getKeyVal(self.detectionImage, "CRVAL2"))
# getExtinction function returns a tuple of strings
# (extinction_correction, data_quality_flags)
self.logfile.write("Determining E(B-V) for detection Image coords: " +
str(ra) + ", " + str(dec))
#pdb.set_trace()
extCorr, dqFlags = extinction.getEBV(ra, dec)
# convert the extinction correction, extCorr to a float for use in arithmetic
self.logfile.write("E(B-V) determined: " + extCorr)
self.eBV = float(extCorr)
self.eBV_error = self.eBV * 0.16 # This is the expected uncertainty in E(B-V): 16%
# make a list of the data quality flags
self.dqList = dqFlags.strip().split()[2:]
# Define different input parameters for different detectors
# This requirement is described in Buzilla bug #1188
# WZ: This has been changed, as the selection is independent
# of detector type...
# self.defaultInParFile = os.path.join(obs.pardir,self.modName+'.inpar')
if (obs.detector == 'WFC'):
self.defaultInParFile = os.path.join(obs.pardir,
self.modName + '_opt.inpar')
elif (obs.detector == 'IR'):
self.defaultInParFile = os.path.join(obs.pardir,
self.modName + '_nir.inpar')
elif (obs.detector == 'UVIS'):
self.defaultInParFile = os.path.join(obs.pardir,
self.modName + '_uv.inpar')
# self.defaultInParFile = os.path.join(obs.pardir,self.modName+'.inpar')
self.logfile.write("Selected input parameter set." +
self.defaultInParFile)
#output parameters are the same for any detector.
self.defaultParamFile = os.path.join(obs.pardir, self.modName +
'.param') # default .param file
self.inpars = pUtil.readInParFile(
self.defaultInParFile) # readInParFile returns a dict
self.outpars = pUtil.readParamFile(
self.defaultParamFile) # readParamFile returns a list of tuples
# Some checking of images before proceeding.
if not self.sciImageList:
errtxt = "No Science Images listed in attribute sciImageList."
self.errorList.append((self.modName, errtxt))
raise AttributeError, errtxt
if not self.flagImageList:
errtxt = "No Flag Images listed in attribute flagImageList."
self.errorList.append((self.modName, ))
raise AttributeError, errtxt
if len(self.sciImageList) != len(self.flagImageList):
errtxt = "Number of Flag Images does not match the number of Science Images."
self.errorList.append((self.modName, errtxt))
raise AttributeError, errtxt
for im in self.sciImageList:
self.inputList.append(os.path.basename(im))
for im in self.flagImageList:
self.inputList.append(os.path.basename(im))