def buildObs2(self): # This is WZ's test function
""" set up the new observation directory structure. Copy the fits data from
the Ingest area to the new observations dir under DATASETS. Copied files are
determined from the asnDict dictionary. Also, this method sets up an IRAF
environment for the pipeline, removing dependency on the user's environment,
hopefully.
"""
datasets_dir = self.base.getenv('DATASETS')
# pdb.set_trace()
if not os.path.isdir(self.newobspath):
os.mkdir(self.newobspath)
os.mkdir(self.newfits)
self.logfile = logFile(self.newobspath) # Initiate the logfile
self.asnDict = fUtil.makeAsnDict(self.ingestdir)
self.buildObsCallCounter += 1
if self.buildObsCallCounter == 1:
pass
else :
self.fitslist = []
for key in self.asnDict.keys():
self.fitslist.append(key)
for file in self.asnDict[key]:
self.fitslist.append(file)
for i in self.fitslist:
try:
copyfile(os.path.join(self.ingestdir,i),os.path.join(self.newfits,i))
except IOError,err:
self.logfile.write("An IOError has occurred in the copyfile call")
self.logfile.write("IOError:"+str(err))
raise IOError,err
def buildObs(self):
""" set up the new observation directory structure.
"""
datasets_dir = self.base.getenv('DATASETS')
if not os.path.isdir(datasets_dir):
print 'Cannot find the $DATASETS directory....You should run acex programe first.' #xingxing
sys.exit()
if not os.path.isdir(self.newobspath):
os.mkdir(self.newobspath)
os.mkdir(self.fitsdir)
self.logfile = logFile(self.newobspath) # Initiate the logfile
#self.asnDict = fUtil.makeAsnDict(self.ingestdir) #Returns a dictionary of asn table names with an embedded list of associated files.
# buildObsCallCounter tracks the number of times this method is called.
# Previously every call of this method would reset the fitslist attr.
# An effort to allow users to use their own fitslists of images
# for wfp processing of non-wfp filter images. However, fitslist
# will still get zeroed if this method is called more than once.
#self.buildObsCallCounter += 1
#if self.buildObsCallCounter == 1:
# pass
#else :
# self.fitslist = []
self.fitslist = []
#for key in self.asnDict.keys(): # delete by xingxing. will be combined with copy_drz.py later ____________
# self.fitslist.append(key)
# for file in self.asnDict[key]:
# self.fitslist.append(file)
#for i in self.fitslist:
# try:
# copyfile(os.path.join(self.ingestdir,i)f,os.path.join(self.fitsdir,i))
# except IOError,err:
# self.logfile.write("An IOError has occurred in the copyfile call")
# self.logfile.write("IOError:"+str(err))
# raise IOError,err
self.inputList = self.fitslist # inputList is for the mkMsg() method.
self.logfile.write(
'Ingest Data moved to observation FITS dir complete.')
os.mkdir(self.newpardir)
os.mkdir(self.catdir)
os.mkdir(self.prodir) #WZ
os.mkdir(self.reddir) #WZ
os.mkdir(self.aligndir)
# # get any default.* optional input files # delete by xingxing.
# self.defaultlist = glob.glob(self.ingestdir+'/default*')
# for deffile in self.defaultlist:
# copy(deffile,self.fitsdir)
# read the $PIPELINE/configs/login.cl file, adjust the home and userid settings
# and write it to the images dir and make a uparm dir. See Bugzilla bug # 2077
# as to why this is being done.
if not self.ownIraf:
irafLoginLines = open(os.path.join(self.configdir,
"login.cl")).readlines()
newLoginFile = open(os.path.join(self.fitsdir, "login.cl"),
"w")
for line in irafLoginLines:
if string.find(line, "set\thome\t\t=") != -1:
newLoginFile.write("set\thome\t\t= \"" + self.fitsdir +
"/\"\n")
elif string.find(line, "set\tuserid\t\t=") != -1:
newLoginFile.write("set\tuserid\t\t= \"" +
os.environ["USER"] + "\"\n")
else:
newLoginFile.write(line)
newLoginFile.close()
os.mkdir(os.path.join(self.fitsdir, "uparm"))
# set the env var MY_IRAF_HOME to be the images dir of the dataset
os.environ["MY_IRAF_HOME"] = self.fitsdir
#
#pdb.set_trace()
self._setIraf()
if os.path.isfile(os.path.join(self.fitsdir, "default.shifts")):
os.rename(os.path.join(self.fitsdir, "default.shifts"),
os.path.join(self.aligndir, "default.shifts"))
self.logfile.write(
"Warning: buildObs method found a default.shifts file. Moved to align."
)
os.mkdir(self.messagedir)
self.logfile.write('Directory build complete.')
# # AKS - Include convert cts/s -> cts script here # XX needn't
# self.logfile.write('Converting counts/sec to counts.')
# for asn in self.asnDict.keys():
# for file in self.asnDict[asn]:
# convertCts.convert_fits(self, os.path.join(self.fitsdir, file))
#for i in self.fitslist:
# # Skip the association file marked by '_asn.fits'
# if not re.search('_asn', i, re.IGNORECASE):
# convertCts.convert_fits(self, os.path.join(self.fitsdir, i))
#pdb.set_trace()
#self._rotateHdr() #AKS
else:
print "============================================================================by xingxing" #WZ Apr 2013
print " Named Datasets already exists in $DATASETS. Are you sure to continue ?!!!"
print " Skip the step of copy files, please delete other files except the original fits"
print " Skip the *Detection Image : >>>>detection_nir.fits and other files"
# print " Skip the *Detection Catalog : >>>>detection_nir.cat"
# print " Skip the *Filter Catalogs : >>>>a209_f110w_drz.cat"
print "============================================================================by xingxing"
self.logfile = logFile(self.newobspath)
#######pdb.set_trace()
mainList = [] # ['f110w','f390w',...]
asnDict = fUtil.makeAsnDict(
os.path.join(os.environ['INGEST'], self.newobs)) #WZ Nov
for key in asnDict.keys(): # fitslist
filter = string.split(key, '_')[1]
if str.lower(filter[0]) == 'f':
mainList.append(filter)
else:
print "Mistake: ", filter
mainList.sort() #WZ
# print filter
print str(len(mainList)) + ' filters are found:' + str(mainList)
self.datasetsdir2 = os.path.join(self.datasetsdir, self.newobs)
#tmpaplusfiles=os.listdir(self.datasetsdir2) #improved by xingxing
#self.logfile.write("choose directories under "+self.datasetsdir2+"( only f***w !!) ====IMPORTANT====")
#for file in tmpaplusfiles:
# if (len(file)!=5) or (str.lower(file[0]) !='f') or (str.lower(file[-1])!='w'):
# print "__"+file+"__ deleted from the band list."
# else:
# self.aplusfiles.append(file)
# print "__"+file+"__ included ."
#print self.aplusfiles
#for ii in self.aplusfiles :
for ii in mainList:
ii = str.lower(ii)
self.sciImageList.append(self.newobs + '_' + ii + '_' + 'drz.fits')
self.contextImageList.append(self.newobs + '_' + ii + '_' +
'drz_context.fits')
self.weightImageList.append(self.newobs + '_' + ii + '_' +
'drz_weight.fits')
self.flagImageList.append(self.newobs + '_' + ii + '_' +
'FLAG.fits')
self.rmsImageList.append(self.newobs + '_' + ii + '_' + 'RMS.fits')
#print "You can check obs.sciImageList here:"
#print self.sciImageList
self.skip = 1
for file in self.sciImageList:
if not os.path.isfile(os.path.join(self.fitsdir, file)):
self.skip = 0
print "" + file + " has not been copied! "
else:
print "" + file + " already exists! "
if self.skip == 1:
print "All image already exist! We will skip all the copy step! ====IMPORTANT===="
else:
self.logfile.write(
"Copy files over: *RMS* *FLAG* and *drz*fits. This would take a while "
)
for i in range(0, len(mainList)):
copyfile(
os.path.join(self.datasetsdir2, mainList[i], 'Images',
self.sciImageList[i]),
os.path.join(self.fitsdir, self.sciImageList[i]))
copyfile(
os.path.join(self.datasetsdir2, mainList[i], 'Images',
self.contextImageList[i]),
os.path.join(self.fitsdir, self.contextImageList[i]))
copyfile(
os.path.join(self.datasetsdir2, mainList[i], 'Images',
self.weightImageList[i]),
os.path.join(self.fitsdir, self.weightImageList[i]))
copyfile(
os.path.join(self.datasetsdir2, mainList[i], 'Images',
self.flagImageList[i]),
os.path.join(self.fitsdir, self.flagImageList[i]))
copyfile(
os.path.join(self.datasetsdir2, mainList[i], 'Images',
self.rmsImageList[i]),
os.path.join(self.fitsdir, self.rmsImageList[i]))
self.logfile.write(mainList[i] + " band images be copied!")
self.logfile.write("Image copy complete")
return
class DataSet:
""" the class defines a new observation directory structure.
The constructor builds the paths for the new observation
(newobs). This tells anything receiving one of these objects
where all the data is. The buildObs method creates the directory
structure.
"""
# pdb.set_trace()
def __init__(self,newobs,ownIraf=0):
self.modName = string.split(string.split(str(self))[0],'.')[0][1:] # this is the module name
self.newobs = newobs # new observation's name
self.base = path.Env()
self.configdir = self.base.getenv('CONFIGS')
self.pardir = self.base.getenv('PARS')
self.ingestdir = os.path.join(self.base.getenv('INGEST'), newobs) # where it is in the INGEST dir
self.newobspath = os.path.join(self.base.getenv('DATASETS'), newobs) # new observation's path under $PIPELINE/DATASETS
self.newpar = os.path.join(self.newobspath, 'par') # parfile dir
self.newfits = os.path.join(self.newobspath, 'Images') # the dir where all data is sent
self.newcats = os.path.join(self.newobspath, 'Catalogs')
self.newalign = os.path.join(self.newobspath, 'align')
self.messagedir = os.path.join(self.newobspath, 'Messages')
self.root = self.newobspath
self.ownIraf = ownIraf
self.errorList = []
self.inputList = []
self.outputList = {} # outputList is now a dictionary to accomodate
# predecessor images.
# this instance variable is initialised here
# and again in the buildObs method or not depending on the
# value of the method call counter. self.fitslist must be
# reset if buildObs() is called multiple times though if that
# is being done, the caller is doing something extraordinary.
self.fitslist = []
# this counter will track the number of calls of the buildObs method.
self.buildObsCallCounter = 0
def getObsPath(self):
""" return the path of the DataSet's root directory. """
return self.newobspath
def getFitsPath(self):
""" return the path of the DataSet's Images directory. """
return self.newfits
def getCatPath(self):
""" return the path of the DataSet's Catalogs directory. """
return self.newcats
def getParPath(self):
""" return the path of the DataSet's par directory. """
return self.newpar
def buildObs(self):
""" set up the new observation directory structure. Copy the fits data from
the Ingest area to the new observations dir under DATASETS. Copied files are
determined from the asnDict dictionary. Also, this method sets up an IRAF
environment for the pipeline, removing dependency on the user's environment,
hopefully.
"""
datasets_dir = self.base.getenv('DATASETS')
if not os.path.isdir(datasets_dir):
print 'Cannot find the $DATASETS directory....Apsis will try to create it.'
try:
os.makedirs(datasets_dir)
print 'Created $DATASETS directory %s' % datasets_dir
except OSError, error:
print error
sys.exit()
if not os.path.isdir(self.newobspath):
os.mkdir(self.newobspath)
os.mkdir(self.newfits)
self.logfile = logFile(self.newobspath) # Initiate the logfile
self.asnDict = fUtil.makeAsnDict(self.ingestdir)
# buildObsCallCounter tracks the number of times this method is called.
# Previously every call of this method would reset the fitslist attr.
# An effort to allow users to use their own fitslists of images
# for apsis processing of non-apsis filter images. However, fitslist
# will still get zeroed if this method is called more than once.
self.buildObsCallCounter += 1
if self.buildObsCallCounter == 1:
pass
else :
self.fitslist = []
for key in self.asnDict.keys():
self.fitslist.append(key)
for file in self.asnDict[key]:
self.fitslist.append(file)
for i in self.fitslist:
try:
copyfile(os.path.join(self.ingestdir,i),os.path.join(self.newfits,i))
except IOError,err:
self.logfile.write("An IOError has occurred in the copyfile call")
self.logfile.write("IOError:"+str(err))
raise IOError,err
self.inputList = self.fitslist # inputList is for the mkMsg() method.
self.logfile.write('Ingest Data moved to observation FITS dir complete.')
os.mkdir(self.newpar)
os.mkdir(self.newcats)
os.mkdir(self.newalign)
# by XX
self.fobj = pyfits.open(os.path.join(self.ingestdir,self.fitslist[1]))
self.hdr = self.fobj[0].header
self.filter = self.hdr["filter"]
try:
if self.filter in ['F105W','F110W','F125W','F140W','F160W']:
fhp=os.path.join(string.replace(self.pardir,'pars','idctab'),'hotpix.fits') #WZ
if os.path.isfile(fhp):
copyfile(fhp,os.path.join(self.newfits,'hotpix.fits'))
fhp=os.path.join(self.ingestdir,'hotpix2.fits')
if os.path.isfile(fhp):
copyfile(fhp,os.path.join(self.newfits,'hotpix2.fits'))
del fhp
except Exception,err:
print Exception,err
sys.exit()
class DataSet:
""" the class defines a new observation directory structure.
The constructor builds the paths for the new observation
(newobs). This tells anything receiving one of these objects
where all the data is. The buildObs method creates the directory
structure.
"""
def __init__(self, newobs, ownIraf=0):
self.modName = string.split(string.split(str(self))[0],
'.')[0][1:] # this is the module name
self.newobs = newobs # new observation's name
self.base = path.Env()
self.configdir = self.base.getenv('CONFIGS')
self.pardir = self.base.getenv('PARS')
self.ingestdir = os.path.join(self.base.getenv('INGEST'),
newobs) # where it is in the INGEST dir
self.newobspath = os.path.join(
self.base.getenv('DATASETS'),
newobs) # new observation's path under $PIPELINE/DATASETS
self.newpardir = os.path.join(self.newobspath, 'par') # parfile dir
self.fitsdir = os.path.join(self.newobspath,
'Images') # the dir where all data is sent
self.catdir = os.path.join(self.newobspath, 'Catalogs')
self.prodir = os.path.join(self.newobspath, 'Output') #WZ
self.reddir = os.path.join(self.newobspath, 'Red') #WZ
self.aligndir = os.path.join(self.newobspath, 'align')
self.messagedir = os.path.join(self.newobspath, 'Messages')
self.root = self.newobspath
self.ownIraf = ownIraf
self.errorList = []
self.inputList = []
self.outputList = {} # outputList is now a dictionary to accomodate
# predecessor images.
# this instance variable is initialised here
# and again in the buildObs method or not depending on the
# value of the method call counter. self.fitslist must be
# reset if buildObs() is called multiple times though if that
# is being done, the caller is doing something extraordinary.
self.fitslist = []
# this counter will track the number of calls of the buildObs method.
self.buildObsCallCounter = 0
def getObsPath(self):
""" return the path of the DataSet's root directory. """
return self.newobspath
def getFitsPath(self):
""" return the path of the DataSet's Images directory. """
return self.fitsdir
def getCatPath(self):
""" return the path of the DataSet's Catalogs directory. """
return self.catdir
def getProdPath(self):
""" return the path of the DataSet's products directory. """
return self.prodir #WZ
def getProdPath(self):
""" return the path of the DataSet's products directory. """
return self.reddir #WZ
def getParPath(self):
""" return the path of the DataSet's par directory. """
return self.newpardir
def buildObs(self):
""" set up the new observation directory structure. Copy the fits data from
the Ingest area to the new observations dir under DATASETS. Copied files are
determined from the asnDict dictionary. Also, this method sets up an IRAF
environment for the pipeline, removing dependency on the user's environment,
hopefully.
"""
datasets_dir = self.base.getenv('DATASETS')
if not os.path.isdir(datasets_dir):
print 'Cannot find the $DATASETS directory....WFP will try to create it.'
try:
os.makedirs(datasets_dir)
print 'Created $DATASETS directory %s' % datasets_dir
except OSError, error:
print error
sys.exit()
if not os.path.isdir(self.newobspath):
os.mkdir(self.newobspath)
os.mkdir(self.fitsdir)
self.logfile = logFile(self.newobspath) # Initiate the logfile
self.asnDict = fUtil.makeAsnDict(self.ingestdir)
# buildObsCallCounter tracks the number of times this method is called.
# Previously every call of this method would reset the fitslist attr.
# An effort to allow users to use their own fitslists of images
# for wfp processing of non-wfp filter images. However, fitslist
# will still get zeroed if this method is called more than once.
self.buildObsCallCounter += 1
if self.buildObsCallCounter == 1:
pass
else:
self.fitslist = []
for key in self.asnDict.keys():
self.fitslist.append(key)
for file in self.asnDict[key]:
self.fitslist.append(file)
for i in self.fitslist:
try:
copyfile(os.path.join(self.ingestdir, i),
os.path.join(self.fitsdir, i))
except IOError, err:
self.logfile.write(
"An IOError has occurred in the copyfile call")
self.logfile.write("IOError:" + str(err))
raise IOError, err
self.inputList = self.fitslist # inputList is for the mkMsg() method.
self.logfile.write(
'Ingest Data moved to observation FITS dir complete.')
os.mkdir(self.newpardir)
os.mkdir(self.catdir)
os.mkdir(self.prodir) #WZ
os.mkdir(self.reddir) #WZ
os.mkdir(self.aligndir)
# get any default.* optional input files
self.defaultlist = glob.glob(self.ingestdir + '/default*')
for deffile in self.defaultlist:
copy(deffile, self.fitsdir)
# read the $PIPELINE/configs/login.cl file, adjust the home and userid settings
# and write it to the images dir and make a uparm dir. See Bugzilla bug # 2077
# as to why this is being done.
if not self.ownIraf:
irafLoginLines = open(os.path.join(self.configdir,
"login.cl")).readlines()
newLoginFile = open(os.path.join(self.fitsdir, "login.cl"),
"w")
for line in irafLoginLines:
if string.find(line, "set\thome\t\t=") != -1:
newLoginFile.write("set\thome\t\t= \"" + self.fitsdir +
"/\"\n")
elif string.find(line, "set\tuserid\t\t=") != -1:
newLoginFile.write("set\tuserid\t\t= \"" +
os.environ["USER"] + "\"\n")
else:
newLoginFile.write(line)
newLoginFile.close()
os.mkdir(os.path.join(self.fitsdir, "uparm"))
# set the env var MY_IRAF_HOME to be the images dir of the dataset
os.environ["MY_IRAF_HOME"] = self.fitsdir
#
#pdb.set_trace()
self._setIraf()
if os.path.isfile(os.path.join(self.fitsdir, "default.shifts")):
os.rename(os.path.join(self.fitsdir, "default.shifts"),
os.path.join(self.aligndir, "default.shifts"))
self.logfile.write(
"Warning: buildObs method found a default.shifts file. Moved to align."
)
os.mkdir(self.messagedir)
self.logfile.write('Directory build complete.')
# AKS - Include convert cts/s -> cts script here
self.logfile.write('Converting counts/sec to counts.')
for asn in self.asnDict.keys():
for file in self.asnDict[asn]:
convertCts.convert_fits(self,
os.path.join(self.fitsdir, file))
#for i in self.fitslist:
# # Skip the association file marked by '_asn.fits'
# if not re.search('_asn', i, re.IGNORECASE):
# convertCts.convert_fits(self, os.path.join(self.fitsdir, i))
#pdb.set_trace()
self._rotateHdr() #AKS