def archiveGEOSDataToGmiArchive (self, task, archiveDirectory, remoteSystem, \
transferFile, processDirectory):
# construct the destination path
destinationPath = task.destinationPath + "/" + \
task.year + "/" + task.month
gmiAutoObject = GmiAutomationTools ()
commonUtils = CommonUtilities ()
ioRoutines = IoRoutines ()
systemCommands = []
# bbscpCmd = self.constants.BBSCPPATH + "bbscp "
cpCmd = "cp "
newTransferFile = transferFile + "." + str (time())
for resolution in ["2x2.5", "1x1.25"]:
fullArchivePath = archiveDirectory + "/" + resolution \
+ "/" + task.year
systemCommands.append (commonUtils.getCreateRemoteDirCmd \
(remoteSystem, fullArchivePath))
systemCommands.append (cpCmd \
+ destinationPath + "/" \
+ task.filePrefix + "." \
+ task.year + task.month + task.day \
+ "." + resolution + ".nc " \
#+ remoteSystem + ":" + fullArchivePath)
+ fullArchivePath)
jobId = None
try:
ioRoutines.appendLinesToFileAndRename (transferFile, \
systemCommands, \
newTransferFile)
print "Here is the transfer file: ", newTransferFile
jobId = commonUtils.qsubFileAndWait (newTransferFile)
except: raise "Problem submitting transfer job"
# remove the transfer and output file
os.remove (newTransferFile)
if jobId != None:
systemCommand = self.constants.RMPATH + "rm -f " + \
processDirectory + "/*" \
+ jobId[0:6] + "*"
os.system(systemCommand)
return self.constants.NOERROR
def __init__(self):
GmiAutomationTools.constants = GmiAutomationConstants ()
self.tasks = []
self.numberOfEntries = 0
self.ioRoutines = IoRoutines ()
self.comUtilities = CommonUtilities ()
def prepareExecFile (self, sysCommands):
ioRoutines = IoRoutines ()
envFile = self.constants.ENVFILE
# all task files have to include environment information
taskFileLines = []
taskFileLines.append ("#!" + self.constants.BASHPATH)
taskFileLines.append (". " + envFile)
count = 0
self.uniqueID = str(time())
execFile = [] # execution file contains all task scripts
for cmd in sysCommands:
# create a task file each independent task
taskFile = self.constants.TMPPATH + \
'indp_task' + str(count) + '_' + self.uniqueID
taskFileLines.append (cmd)
ioRoutines.writeToFile(taskFileLines, taskFile)
# pop the command in preperation for next task
taskFileLines.pop ()
execFile.append(self.constants.FINAL_TMPPATH + \
'indp_task' + str(count) + '_' + self.uniqueID)
count = count + 1
execFile.append ("")
execFileName = self.constants.FINAL_TMPPATH + \
'run_tasks' + '_' + self.uniqueID
ioRoutines.writeToFile(execFile, execFileName)
# copy task files to FINAL_TMPPATH
systemCommand = 'cp ' + self.constants.TMPPATH + 'indp_task*' + \
self.uniqueID + '* ' + \
self.constants.FINAL_TMPPATH
os.system(systemCommand)
os.system ("chmod 750 " + self.constants.FINAL_TMPPATH + \
"*" + self.uniqueID)
return execFileName
def __init__(self, realTime):
self.comUtilities = CommonUtilities ()
self.autoConstants = GmiAutomationConstants ()
self.gmiAutomationTool = GmiAutomationTools ()
self.ioRoutines = IoRoutines ()
self.netCdfTools = GmiNetCdfFileTools ()
self.realTime = realTime
self.logUtility = GmiLogUtilities ()
self.geos5Fields = GmiGEOS5DasFields ()
self.basePath = ''
def __init__(self):
self.queueFileName = ""
self.nameListName = ""
self.queueJobName = ""
self.runDirectory = ""
self.storageDirectory = ""
self.longDescription = ""
self.segmentDescription = ""
self.modelVersion = ""
self.mailTo = ""
self.queueId = ""
self.base = ""
self.year = ""
self.month = ""
self.archiveSystem = ""
self.scali = None
self.exceptions = GmiExceptions()
self.constants = GmiAutomationConstants()
self.automationObject = GmiAutomationTools()
self.ioObject = IoRoutines()
def __init__(self):
self.queueFileName = ""
self.nameListName = ""
self.queueJobName = ""
self.runDirectory = ""
self.storageDirectory = ""
self.longDescription = ""
self.segmentDescription = ""
self.modelVersion = ""
self.mailTo = ""
self.queueId = ""
self.base = ""
self.year = ""
self.month = ""
self.archiveSystem = ""
self.scali = None
self.exceptions = GmiExceptions ()
self.constants = GmiAutomationConstants ()
self.automationObject = GmiAutomationTools ()
self.ioObject = IoRoutines ()
class GmiAutomationTools:
LINESPERINPUTENTRY = 5.0
LINESPERINPUTENTRYINTEGER = 5
DEFAULTNUMBEROFDAYS = 1
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# Constructor routine.
#---------------------------------------------------------------------------
def __init__(self):
GmiAutomationTools.constants = GmiAutomationConstants ()
self.tasks = []
self.numberOfEntries = 0
self.ioRoutines = IoRoutines ()
self.comUtilities = CommonUtilities ()
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# Destructor routine.
#---------------------------------------------------------------------------
def __del__(self):
pass
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine checks for the existence of the fileName.
#---------------------------------------------------------------------------
def checkForSpecialInputFile (self, fileName):
if len (fileName) == 0:
return GmiAutomationTools.constants.ERROR
if not os.path.exists (fileName):
return GmiAutomationTools.constants.NOSUCHFILE
return GmiAutomationTools.constants.NOERROR
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine reads the input file and returns appropriate error messages.
# The routine uses the class attribute, LINESPERINPUTENTRY, to determine
# if file contains the correct number of lines.
# The file may contains multiple entires with LINESPERINPUTENTRY lines.
#---------------------------------------------------------------------------
def readInputFile (self, fileName):
if self.checkForSpecialInputFile (fileName) != GmiAutomationTools.constants.NOERROR:
return GmiAutomationTools.constants.READERROR
try:
fileObject = open (fileName, 'r')
fileContents = fileObject.read ()
fileObject.close ()
except:
return GmiAutomationTools.constants.READERROR
if len (fileContents) == 0:
return GmiAutomationTools.constants.BADFILENAME
fileLines = fileContents.splitlines()
if len (fileLines) % GmiAutomationTools.LINESPERINPUTENTRY != 0:
return GmiAutomationTools.constants.INCOMPLETEFILE
# there are LINESPERINPUTENTRY per entry
self.numberOfEntries = len (fileLines) / GmiAutomationTools.LINESPERINPUTENTRY
lineCounter = 0
if len (self.tasks) == 0:
entryCounter = 0
else:
entryCounter = len (self.tasks)
while lineCounter < len (fileLines):
self.tasks.append (GmiMetFieldTask ())
self.tasks[entryCounter].numberOfDays = int (fileLines [lineCounter])
lineCounter = lineCounter + 1
self.tasks[entryCounter].firstDay = fileLines [lineCounter]
lineCounter = lineCounter + 1
self.tasks[entryCounter].destinationPath = fileLines [lineCounter]
lineCounter = lineCounter + 1
self.tasks[entryCounter].sourcePath = fileLines [lineCounter]
lineCounter = lineCounter + 1
self.tasks[entryCounter].filePrefix = fileLines [lineCounter]
lineCounter = lineCounter + 1
returnCode = self.tasks[entryCounter].setDate (self.tasks[entryCounter].firstDay)
if returnCode != self.constants.NOERROR:
return self.constants.ERROR
entryCounter = entryCounter + 1
return GmiAutomationTools.constants.NOERROR
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine looks for the input file in the current
# directory and fills the tasks object with the information
# from file or the default information for the current day
#--------------------------------------------------------- ------------------
def getGmiMetFieldTasks (self, inputFileName, destinationPath, defaultForecastSourcePath):
returnCode = self.checkForSpecialInputFile (inputFileName)
if returnCode == GmiAutomationTools.constants.ERROR:
return returnCode
if returnCode != self.constants.NOSUCHFILE:
returnCode = self.readInputFile (inputFileName)
if returnCode != GmiAutomationTools.constants.NOERROR:
return returnCode
else:
returnCode = self.getDefaultForecastTaskConfiguration (destinationPath, \
defaultForecastSourcePath)
if returnCode != GmiAutomationTools.constants.NOERROR:
return returnCode
return GmiAutomationTools.constants.NOERROR
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine fills tasks[0] with the default configuration for a
# task
#---------------------------------------------------------------------------
def getDefaultForecastTaskConfiguration (self, destinationPath, defaultForecastSourcePath):
self.tasks.append (GmiMetFieldTask ())
self.tasks[0].numberOfDays = 1
self.tasks[0].firstDay = self.getCurrentDate ()
self.tasks[0].filePrefix = 'a_flk_04.'
self.tasks[0].setDate (self.tasks[0].firstDay)
self.tasks[0].sourcePath = defaultForecastSourcePath
self.tasks[0].destinationPath = destinationPath
return GmiAutomationTools.constants.NOERROR
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine returns the current date in the format: YYYYMMDD
#---------------------------------------------------------------------------
def getCurrentDate (self):
today = datetime.datetime.date(datetime.datetime.now())
splitToday = re.split('-', str(today))
newToday = splitToday[0] + splitToday[1] + splitToday[2]
return newToday
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine prints the current tasks.
#---------------------------------------------------------------------------
def printTasks (self):
if len (self.tasks) == 0:
print "There are no tasks."
return self.constants.NOERROR
print "There are", len (self.tasks), "task(s)."
print "\n"
taskCounter = 0
while taskCounter < len (self.tasks):
print "Task", taskCounter+1, ":"
print "Number of days to process:", self.tasks[taskCounter].numberOfDays
print "First day:", self.tasks[taskCounter].firstDay
print "Destination path:", self.tasks[taskCounter].destinationPath
print "Source path:", self.tasks[taskCounter].sourcePath
print "File prefix:", self.tasks[taskCounter].filePrefix
print "Year:", self.tasks[taskCounter].year
print "Month:", self.tasks[taskCounter].month
print "Day:", self.tasks[taskCounter].day
print "\n"
taskCounter = taskCounter + 1
return self.constants.NOERROR
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine returns a local file listing from the given file patter.
# The file prefix is expected to include a path if outside the working
# directory. Wildcards are not expected as part of the filePattern!
# DO NOT USE wildcards, such as *!
#---------------------------------------------------------------------------
def getMatchingFiles (self, path, filePattern):
if len (filePattern) == 0:
raise self.constants.INVALIDPATTERN
if not os.path.exists (path):
raise self.constants.NOSUCHPATH
fileList = os.listdir (path)
if filePattern == '*':
return fileList
returnedFileList = []
for file in fileList:
if re.search (filePattern, file):
returnedFileList.append (file)
returnedFileList.sort ()
return returnedFileList
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine creates the directory structure if it does not exist. It
# will exit on error if it is unable to create a directory.
#---------------------------------------------------------------------------
def createDirectoryStructure (self, path):
if len (path) <= 0:
return self.constants.INVALIDINPUT
directories = re.split('/', str(path))
growingPath = ''
for directory in directories:
growingPath = growingPath + directory + '/'
if not os.path.exists (growingPath):
systemCommand = self.constants.MKDIRPATH + 'mkdir ' + growingPath
systemReturnCode = os.system (systemCommand)
if systemReturnCode != 0:
print "Error! returnCode is: ", systemReturnCode, " after attempting to create the directory: ", growingPath, "\n"
return self.constants.BADSYSTEMRETURNCODE
return self.constants.NOERROR
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine adds the specified path to the list of file names given.
#---------------------------------------------------------------------------
def addPathToFileNames (self, path, fileNames):
if len (fileNames) <= 0:
raise self.constants.INVALIDFILENAMES
returnedFileNames = []
for fileName in fileNames:
if len (fileName) == 0:
raise self.constants.INVALIDFILENAMES
newFileName = path + fileName
returnedFileNames.append (newFileName)
return returnedFileNames
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine simply creates a new file from the fileName
#---------------------------------------------------------------------------
def touchNewFile (self, fileName):
fileObject = open (fileName, 'w')
fileObject.close ()
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine will read the given file, determine the last date processed,
# and increment the date and return it.
#---------------------------------------------------------------------------
def getNextDateFromFile (self, fileName, processingDir):
if len (fileName) <= 0:
return self.constants.INVALIDINPUT
if not os.path.exists (fileName):
return self.constants.NOSUCHFILE
# read the last date processed in from file
try:
fileObject = open (fileName, 'r')
fileContents = fileObject.read ()
fileObject.close ()
except:
print "Error! returnCode is: ", returnCode, " after attempting to read ", fileName, "\n"
return self.constants.READERROR
fileLines = fileContents.splitlines()
numberOfEntries = len (fileLines)
lastDateProcessed = fileLines [numberOfEntries -1]
nextDate = self.incrementDate (lastDateProcessed, processingDir)
return nextDate
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine simply returns the next day.
#---------------------------------------------------------------------------
def incrementDate (self, theDate, runPath):
# call external program to increment the date
tempFileName = "out.out" + str(time())
systemCommand = runPath + './incrdate ' + theDate + ' >& ' + tempFileName
systemReturnCode = os.system (systemCommand)
if systemReturnCode != 0:
print "Error! returnCode is: ", systemReturnCode, " after attempting to use the incrdate program!", "\n"
os.remove (tempFileName)
return self.constants.BADSYSTEMRETURNCODE
# read the next day in from file
try:
fileObject = open (tempFileName, 'r')
fileContents = fileObject.read ()
fileObject.close ()
os.remove (tempFileName)
except:
print "Error after reading the next day from file!"
os.remove (tempFileName)
return self.constants.READERROR
fileLines = fileContents.splitlines()
nextDate = fileLines [0]
return nextDate
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine fills tasks[0] with the default configuration for a
# DAS task.
#---------------------------------------------------------------------------
def getDefaultDasTaskConfiguration (self, fileName, processDirectory, \
dasDestinationPath, defaultDasSourcePath):
if len (fileName) <= 0:
return self.constants.INVALIDINPUT
self.tasks.append (GmiMetFieldTask ())
# get the next day in the file
self.tasks[0].firstDay = self.getNextDateFromFile (fileName, processDirectory)
if len (self.tasks[0].firstDay) != GmiMetFieldTask.LENGHTOFEXPECTEDDATESTRING:
print "There was an error getting the NextDateFromFile !\n"
return self.constants.ERROR
self.tasks[0].numberOfDays = 1
self.tasks[0].filePrefix = 'a_llk_04.'
self.tasks[0].setDate (self.tasks[0].firstDay)
self.tasks[0].sourcePath = defaultDasSourcePath
self.tasks[0].destinationPath = dasDestinationPath
return self.constants.NOERROR
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine expects an GmiAutomationTools object with at least one task
# in it's task list. It will create more tasks if the numberOfDays to
# process is greater than one. It will simple append tasks to the list
# and increment each date by one day.
#---------------------------------------------------------------------------
def createAdditionalTasks (self, processingDir):
if len (self.tasks) <= 0:
return self.constants.INVALIDINPUT
newTasks = []
for task in self.tasks:
# make sure the task info is complete
returnCode = task.verifyCompleteness ()
if returnCode != self.constants.NOERROR:
return self.constants.INVALIDINPUT
# add a new task
# only add one, because the current
# task will be the one that represents
# the first day
loopCounter = 1
nextDate = task.firstDay
while loopCounter < task.numberOfDays:
# add a new blank task
newTasks.append (GmiMetFieldTask ())
# set the values of the new task
# to the old one
newTasks [len(newTasks)-1].sourcePath = task.sourcePath
newTasks [len(newTasks)-1].destinationPath = task.destinationPath
newTasks [len(newTasks)-1].filePrefix = task.filePrefix
# make each task just say one day
newTasks [len(newTasks)-1].numberOfDays = 1
nextDate = self.incrementDate (nextDate, processingDir)
# then, change the date
newTasks [len(newTasks)-1].setDate (nextDate)
newTasks [len(newTasks)-1].firstDay = nextDate
newTasks [len(newTasks)-1].sourcePath = task.sourcePath
newTasks [len(newTasks)-1].destinationPath = task.destinationPath
loopCounter = loopCounter + 1
# now that this task has been looked at, set it to 1 day
task.numberOfDays = 1
# append new tasks to tasks list
self.tasks.extend (newTasks)
return self.constants.NOERROR
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine writes all tasks in the self object to the file.
#---------------------------------------------------------------------------
def writeAllTasksToFile (self, fileName):
for task in self.tasks:
returnCode = self.writeTaskToFile (task, fileName)
if returnCode != self.constants.NOERROR:
return returnCode
return self.constants.NOERROR
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This will verify that the task is complete, create a file, or just
# append to it the task information.
#---------------------------------------------------------------------------
def writeTaskToFile (self, task, fileName):
if len (fileName) <= 0:
return self.constants.INVALIDINPUT
# make sure the task info is complete
returnCode = task.verifyCompleteness ()
if returnCode != self.constants.NOERROR:
return self.constants.INVALIDINPUT
if not os.path.exists (fileName):
self.touchNewFile (fileName)
try:
fileObject = open (fileName, 'a')
toWrite = str (task.numberOfDays) + '\n'
fileObject.write (toWrite)
toWrite = str (task.firstDay) + '\n'
fileObject.write (toWrite)
fileObject.write (task.destinationPath + '\n')
fileObject.write (task.sourcePath + '\n')
fileObject.write (task.filePrefix + '\n')
fileObject.close ()
except:
print "There was an error appending to the file: ", fileName, "\n"
return self.constants.WRITEERROR
return self.constants.NOERROR
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine removes the last task from the file.
#---------------------------------------------------------------------------
def removeTaskFromFile (self, fileName, task):
if len (fileName) <= 0:
raise self.constants.INVALIDINPUT
if not os.path.exists (fileName):
raise self.constants.NOSUCHFILE
# make sure the task info is complete
returnCode = task.verifyCompleteness ()
if returnCode != self.constants.NOERROR:
raise self.constants.INVALIDINPUT
try:
fileObject = open (fileName, 'r')
fileContents = fileObject.read ()
fileObject.close
except:
raise self.constants.READERROR
fileLines = fileContents.splitlines()
if len (fileLines) <= 0:
return self.constants.NOERROR
os.remove (fileName)
newFileLines = []
lineCounter = 0
while lineCounter < len (fileLines):
if fileLines [lineCounter] == task.firstDay:
# pop the previous line, which belongs to this task
if len (newFileLines) > 1:
newFileLines.pop (lineCounter-1)
else:
newFileLines = []
# skip the next 3 lines, which belong to this task
lineCounter = lineCounter + 3
else:
newFileLines.append (fileLines [lineCounter])
lineCounter = lineCounter + 1
if len (newFileLines) <= 0:
return self.constants.NOERROR
try:
fileObject = open (fileName, 'w')
for line in newFileLines:
fileObject.write (line + '\n')
fileObject.close
except:
raise self.constants.WRITEERROR
return self.constants.NOERROR
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine removes duplicates from the tasks list.
#---------------------------------------------------------------------------
def removeDuplicateTasks (self):
# no tasks should not be an error
# return because there is nothing to do
if len (self.tasks) <= 1:
return self.constants.NOERROR
newTasks = []
loopCounter = 0
while loopCounter < len (self.tasks):
foundDuplicateTask = False
innerCounter = loopCounter + 1
while innerCounter < len (self.tasks):
if self.tasks[loopCounter].numberOfDays == self.tasks[innerCounter].numberOfDays and \
self.tasks[loopCounter].firstDay == self.tasks[innerCounter].firstDay and \
self.tasks[loopCounter].destinationPath == self.tasks[innerCounter].destinationPath and \
self.tasks[loopCounter].sourcePath == self.tasks[innerCounter].sourcePath and \
self.tasks[loopCounter].filePrefix == self.tasks[innerCounter].filePrefix and \
self.tasks[loopCounter].year == self.tasks[innerCounter].year and \
self.tasks[loopCounter].month == self.tasks[innerCounter].month and \
self.tasks[loopCounter].day == self.tasks[innerCounter].day:
foundDuplicateTask = True
innerCounter = innerCounter + 1
if foundDuplicateTask == False:
newTasks.append(self.tasks[loopCounter])
loopCounter = loopCounter + 1
self.tasks = newTasks
return self.constants.NOERROR
#----------------------------------------------------------------
# Write new tasks to file in case unexpected aborted
# execution occurs.
#----------------------------------------------------------------
def writeNewTasksToFile (self, taskFile, mailTo):
tempFile = taskFile + ".tmp"
if self.writeAllTasksToFile (tempFile) \
!= GmiAutomationTools.constants.NOERROR:
self.ioRoutines.mailMessage ("Problem writing tasks to file " + \
tempFile, \
GmiAutomationTools.constants.ERROR_SUBJECT, \
mailTo)
raise GmiAutomationTools.constants.ERROR
elif os.system (GmiAutomationTools.constants.MVPATH + "/mv " + \
tempFile + " " + taskFile) \
!= GmiAutomationTools.constants.SYS_SUCCESS:
self.ioRoutines.mailMessage ("Problem writing the file " + \
taskFile, \
GmiAutomationTools.constants.ERROR_SUBJECT, mailTo)
raise GmiAutomationTools.constants.ERROR
#----------------------------------------------------------------
# Look for a file with attempted tasks that did complete due
# unexpected problems. Remove duplicate tasks.
#----------------------------------------------------------------
def readAttemptedTasks (self, taskFile, mailTo):
if os.path.exists (taskFile):
returnCode = self.readInputFile (taskFile)
if returnCode != GmiAutomationTools.constants.NOERROR:
self.ioRoutines.mailMessage ("Problem reading " + taskFile \
, GmiAutomationTools.constants.ERROR_SUBJECT, mailTo)
raise (returnCode)
def createTasks (self, configOptions, processDirectory, mailTo, \
realTime, taskFile):
#----------------------------------------------------------------
# Create a single task
#----------------------------------------------------------------
self.tasks = []
self.tasks.append (GmiMetFieldTask())
#----------------------------------------------------------------
# Set up tasks based on the START_DATE and NUM_DAYS
#----------------------------------------------------------------
self.tasks[0].createTaskFromData (configOptions)
self.createAdditionalTasks (processDirectory)
#----------------------------------------------------------------
# Modifications for real-time mode support
#----------------------------------------------------------------
# If additional tasks were created and the mode is REAL_TIME
# there is some conflict; real time is intended to process
# one day at a time following operations
if len(self.tasks) > 1 and realTime[0:4] == "true":
self.ioRoutines.mailMessage ("If REAL_TIME mode is true then " + \
"only one tasks at a time is allowed", \
GmiAutomationTools.constants.ERROR_SUBJECT, \
mailTo)
raise (GmiAutomationTools.constants.ERROR)
if realTime[0:4] == "true":
realTimeDate = self.tasks[0].year + \
self.tasks[0].month + \
self.tasks[0].day
currentDirectory = os.getcwd()
nextRealTimeDate = self.incrementDate (realTimeDate, currentDirectory + "/")
#----------------------------------------------------------------
# End modifications for real-time mode
#----------------------------------------------------------------
try:
self.readAttemptedTasks (taskFile, mailTo)
self.removeDuplicateTasks ()
self.writeNewTasksToFile (taskFile, mailTo)
#self.printTasks ()
except:
raise
sys.exit (0) configFile = optList[0][1] #--------------------------------------------------------------- # objects #--------------------------------------------------------------- netCdfTools = GmiNetCdfFileTools () comUtilities = CommonUtilities () gmiAutomationObject = GmiAutomationTools () autoConstants = GmiAutomationConstants () preStage = GmiPreStage () logUtility = GmiLogUtilities () dasFields = GmiGEOS5DasFields () ioRoutines = IoRoutines () inst2DObject = GmiGEOS5DasInst2D () avg2DObject = GmiGEOS5DasAvg2D () avg3DObject = GmiGEOS5DasAvg3D () dasObjects = [] dasObjects.append (avg2DObject) dasObjects.append (inst2DObject) dasObjects.append (avg3DObject) #--------------------------------------------------------------- # Read input file and set up internal variables #--------------------------------------------------------------- configOptions = netCdfTools.readFileAndReturnFileLines (configFile) processDirectory = comUtilities.returnValue (configOptions, "RUN_DIR") if processDirectory[-1:] != "/":
def makeNameLists (self):
if self.fortranNameList == None:
raise "Set fortranNameList before proceeding"
return
if len (self.nameListPath) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if len (self.nameListFile) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if self.numberOfNameLists <= 0:
raise self.exceptions.BADOBJECTDATA
return
if len (self.startMonth) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if len (self.startYear) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if not os.path.exists (self.nameListPath + "/" + self.nameListFile):
raise self.exceptions.NOSUCHFILE
return
autoTool = GmiAutomationTools ()
pathToNameList = self.nameListPath
namelistfile = self.nameListFile
numNameLists = self.numberOfNameLists
startmonth = self.startMonth
startyear = self.startYear
#---------------------------------------------------------------------------
# Begin Gary Wojcik section
# Modified by Megan Damon 3/20/2008
#---------------------------------------------------------------------------
daysPerMonth=[31,28,31,30,31,30,31,31,30,31,30,31]
monthNames=['jan', 'feb','mar','apr','may','jun','jul',\
'aug','sep','oct','nov', 'dec']
ioObject = IoRoutines ()
fileLines = ioObject.readFile (self.envFile)
for line in fileLines:
if re.search ('MATCH', line):
splitString = line.split("=")
options = splitString[0].split(" ")
option = options[1]
value = splitString[1]
if option == "MATCH_BC_TO_YEAR":
matchBcToYear = bool(1)
if value == "F":
matchBcToYear = bool(0)
elif option == "MATCH_SAD_TO_YEAR":
matchSadToYear = bool(1)
if value == "F":
matchSadToYear = bool(0)
elif option == "MATCH_EMISS_TO_YEAR":
matchEmissToYear = bool(1)
if value == "F":
matchEmissToYear = bool(0)
elif option == "MATCH_AERDUST_TO_YEAR":
matchAerToYear = bool(1)
if value == "F":
matchAerToYear = bool(0)
elif option == "MATCH_LIGHT_TO_YEAR":
matchLightToYear = bool(1)
if value == "F":
matchLightToYear = bool(0)
else:
print "I don't understand this MATCH option and this could be a problem."
elif re.search ('INITIAL', line):
splitString = line.split("=")
options = splitString[0].split(" ")
option = options[1]
value = splitString[1]
if option == "INITIAL_YEAR_FORC_BC":
initialYearForcBc = value
elif option == "INITIAL_YEAR_LIGHT":
initialYearLight = value
else:
print "I don't understand this INITIAL option and this could be a problme"
# read the starting namelist file
infil='%s/%s' % (pathToNameList, namelistfile)
nfil=open(infil,"r")
namelistContent=nfil.readlines()
nfil.close()
if self.fortranNameList == "true":
splitter = "="
endLine = ",\n"
else:
splitter = ":"
endLine = "\n"
# get orginal problem name
# it is assumed the month is in the name
foundIt = -1
for line in namelistContent:
if re.search ('problem_name', line):
origproblemname=line.split(splitter)[1]
origproblemname.strip()
foundIt = 0
if foundIt != 0:
print "Problem finding the problem_name. Check the namelist"
sys.exit (-1)
splitProblemName=origproblemname.split("_")
counter = 0
for word in splitProblemName:
string.join(word.split(), "")
splitProblemName[counter] = word.strip()
splitProblemName[counter] = word.strip("'")
splitProblemName[counter] = word.strip("',\n")
counter = counter + 1
# get orginal problem name
# it is assumed the month is in the name
# baseProblemName=namelistContent[1].split("'")[1]
# splitProblemName=baseProblemName.split("_")
# find the month to start on
for monthNumber in xrange(0,12,1):
if startmonth == monthNames[monthNumber]:
istartmonth=monthNumber
monthId=istartmonth
yearFlag=0
currentYear=startyear
restartYear=currentYear
# will be used to make the namelists file
listOfNamelists = []
# for each namelist
currentYear=str(int(startyear))
for currentNameListCount in xrange(0,numNameLists,1):
# open the file
nfil=open(infil,"r")
# flag for a new year (11->DEC
if istartmonth == 11:
yearFlag = 1
if yearFlag == 1:
restartYear=str(int(currentYear)+1)
yearFlag=0
if monthId == 0:
restartYear=str(int(currentYear)-1)
else:
restartYear=str(int(currentYear))
currentMonth = monthNames[monthId]
metfilelist = '%s%s.list' % (currentMonth, currentYear)
finaldays = daysPerMonth[monthId]
if monthId < 9:
strmonthId='0'+str(monthId+1)
else:
strmonthId=str(monthId+1)
# create namelist variables
if self.oneDay == 1:
endymd = self.endDate
startymd = self.startDate
else:
endymd = currentYear[0:4] + strmonthId + str(daysPerMonth[monthId])
endymd = autoTool.incrementDate (endymd)
if endymd[4:8] == "0229":
newendymd = endymd[0:4]+"0301"
endymd = newendymd
startymd = currentYear[0:4] + strmonthId + '01'
print "endymd: ", endymd
problemName1 = splitProblemName[0].strip()
problemName2 = splitProblemName[1].strip()
restartName = problemName1 + '_' + problemName2 + \
'_' + restartYear + '_'+ \
monthNames[monthId-1]
restartFileName = '%s.rst.nc' % (restartName)
newProblemName = problemName1 + '_' + problemName2 + \
'_' + currentYear + '_' + currentMonth
directoryPath = '%s/' % (pathToNameList)
dflag = os.path.exists(directoryPath)
if dflag == 0:
os.makedirs(directoryPath)
listOfNamelists.append (newProblemName+".in")
newProblemNameFullPath = '%s%s.in' % (directoryPath,newProblemName)
ofil = open(newProblemNameFullPath,"w")
if self.fortranNameList == "true":
newProblemName = "'" + newProblemName + "'"
metfilelist = "'" + metfilelist + "'"
print "newProblemName: ", newProblemName
# create new file
for inline in nfil.readlines():
# find the variable name / keyWord
splitLine = inline.split()
# skip if there are no items
if len(splitLine) == 0:
continue
keyWord = splitLine[0]
splitKeyWord = re.split(splitter, keyWord)
keyWord = splitKeyWord[0]
if len(splitLine) == 2:
value = splitLine[1]
if keyWord == 'problem_name':
print "newProblemName: ", newProblemName
ofil.write('problem_name' + splitter + newProblemName + endLine)
elif keyWord == 'BEG_YY':
print "BEG_YY: ", currentYear[0:4]
ofil.write('BEG_YY' + splitter + currentYear[0:4] + endLine)
elif keyWord == 'BEG_MM':
print "BEG_MM: ", str(int(strmonthId))
ofil.write('BEG_MM' + splitter + str(int(strmonthId)) + endLine)
elif keyWord == 'BEG_DD':
print "BEG_DD: 0"
ofil.write('BEG_DD' + splitter + ' 1' + endLine)
elif keyWord == 'BEG_H':
print "BEG_H: 0"
ofil.write('BEG_H' + splitter + ' 0' + endLine)
elif keyWord == 'BEG_M':
print "BEG_M: 0"
ofil.write('BEG_M' + splitter + ' 0' + endLine)
elif keyWord == 'BEG_S':
print "BEG_S: 0"
ofil.write('BEG_S' + splitter + ' 0' + endLine)
elif keyWord == "END_YY":
ofil.write('END_YY' + splitter + endymd[0:4] + endLine)
elif keyWord == "END_MM":
ofil.write('END_MM' + splitter + str(int(endymd[4:6])) + endLine)
elif keyWord == "END_DD":
ofil.write('END_DD' + splitter + str(int(endymd[6:8])) + endLine)
elif keyWord == 'END_H':
ofil.write('END_H' + splitter + ' 0' + endLine)
elif keyWord == 'END_M':
ofil.write('END_M' + splitter + ' 0' + endLine)
elif keyWord == 'END_S':
ofil.write('END_S' + splitter + ' 0' + endLine)
elif keyWord == 'met_filnam_list':
ofil.write('met_filnam_list' + splitter + metfilelist + endLine)
elif keyWord == 'restart_infile_name' and \
self.replaceRestartFile != 0:
print "changing restart!"
ofil.write('restart_infile_name' + splitter + restartFileName + endLine)
elif keyWord == 'forc_bc_start_num' and \
matchBcToYear == True:
forcBcStartNum = int(startymd[0:4]) - int(initialYearForcBc)
if forcBcStartNum < 0:
print "forcBcStartNum cannot be less than 0!"
sys.exit (-1)
print "forcBcStartNum: ", forcBcStartNum
ofil.write('forc_bc_start_num' + splitter + str(forcBcStartNum) + endLine)
#ofil.write('forc_bc_start_num' + splitter + str(int(startymd[0:4]) - 1949) + endLine)
elif keyWord == 'lightYearDim' and \
matchLightToYear == True:
lightYearStartNum = int(startymd[0:4]) - int(initialYearLight)
if lightYearStartNum < 0:
print "lightYearStartNum cannot be less than 0!"
sys.exit (-1)
print "lightYearStartNum: ", lightYearStartNum
ofil.write('lightYearDim' + splitter + str(lightYearStartNum) + endLine)
#ofil.write('lightYearDim' + splitter + str(int(startymd[0:4]) - 1989) + endLine)
elif keyWord == 'lbssad_infile_name' and \
matchSadToYear == True:
print "changing lbssad"
splitLbssad = re.split("_", value)
newLbssad = splitLbssad[0] + "_" + splitLbssad[1] + "_" + splitLbssad[2] + "_" + startymd[0:4] + ".nc"
ofil.write('lbssad_infile_name' + splitter + newLbssad + endLine)
elif keyWord == 'emiss_infile_name' and \
matchEmissToYear == True:
splitEmiss = re.split("_", value)
newEmiss = splitEmiss[0] + "_" + startymd[0:4] + "_" + splitEmiss[2]+ "_" + splitEmiss[3] + "_" + splitEmiss[4]
print "newEmiss: ", newEmiss
ofil.write('emiss_infile_name' + splitter + newEmiss + endLine)
elif keyWord == 'AerDust_infile_name' and \
matchAerToYear == True:
splitAer = re.split("_", value)
newAer = splitAer[0] + "_" + splitAer[1] + "_" + splitAer[2] + "_" + startymd[0:4] + "_" + splitAer[4]
print "newAer: ", newAer
ofil.write('AerDust_infile_name' + splitter + newAer + endLine)
elif keyWord == 'begGmiDate':
ofil.write('begGmiDate' + splitter + currentYear + strmonthId \
+ '01' + endLine)
elif keyWord == 'begGmiTime':
ofil.write('begGmiTime' + splitter + '000000' + endLine)
elif keyWord == 'endGmiDate':
ofil.write('endGmiDate' + splitter + str(int(endymd[0:8])) + endLine)
elif keyWord == 'endGmiTime':
ofil.write('endGmiTime' + splitter + '000000' + endLine)
elif keyWord == 'gmi_sec':
ofil.write('gmi_sec' + splitter + self.gmiSeconds + endLine)
elif keyWord == 'rd_restart' and \
self.replaceRestartFile != 0:
ofil.write('rd_restart' + splitter + 'T' + endLine)
else:
data=inline.replace('\r','') #Removes ^M character from strings written
ofil.write(data)
intYear = int (currentYear)
monthId = monthId + 1
if monthId == 12:
currentYear = str (intYear + 1)
monthId = 0
ioObject.writeToFile (listOfNamelists,pathToNameList+"/namelists.list")
if int(currentYear)% 4 == 0 and monthId == 2:
print "leap year!"
daysInMonth = 29
else:
daysInMonth = int(daysPerMonth[monthId-1])
gmiSeconds = int(self.gmiSeconds) + (daysInMonth * 24 * 3600)
# write new gmiSeconds to file
ioObject.writeToFile (["export gmi_sec=" + str(gmiSeconds)], self.envFile)
for nameList in listOfNamelists:
print nameList
sys.stdout.flush()
sys.exit(0)
import getopt
import sys
import os
import time
import commands
import subprocess
from IoRoutines import IoRoutines
NUM_ARGS = 2
NUM_SPLIT_ARGS = 5
ioRoutines = IoRoutines()
# Get options from command line
optList, argList = getopt.getopt(sys.argv[1:], 'f:e:')
if len(optList) != NUM_ARGS:
usage()
stdoutFileName = optList[0][1]
expEnvFile = optList[1][1]
try:
expEnvLines = ioRoutines.readFile(expEnvFile)
except:
print "Check the file: ", expEnvFile
sys.exit(-1)
def makeNameLists(self):
if self.fortranNameList == None:
raise "Set fortranNameList before proceeding"
return
if len(self.nameListPath) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if len(self.nameListFile) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if self.numberOfNameLists <= 0:
raise self.exceptions.BADOBJECTDATA
return
if len(self.startMonth) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if len(self.startYear) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if not os.path.exists(self.nameListPath + "/" + self.nameListFile):
raise self.exceptions.NOSUCHFILE
return
autoTool = GmiAutomationTools()
pathToNameList = self.nameListPath
namelistfile = self.nameListFile
numNameLists = self.numberOfNameLists
startmonth = self.startMonth
startyear = self.startYear
#---------------------------------------------------------------------------
# Begin Gary Wojcik section
# Modified by Megan Damon 3/20/2008
#---------------------------------------------------------------------------
daysPerMonth = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
monthNames=['jan', 'feb','mar','apr','may','jun','jul',\
'aug','sep','oct','nov', 'dec']
ioObject = IoRoutines()
fileLines = ioObject.readFile(self.envFile)
for line in fileLines:
if re.search('MATCH', line):
splitString = line.split("=")
options = splitString[0].split(" ")
option = options[1]
value = splitString[1]
if option == "MATCH_BC_TO_YEAR":
matchBcToYear = bool(1)
if value == "F":
matchBcToYear = bool(0)
elif option == "MATCH_SAD_TO_YEAR":
matchSadToYear = bool(1)
if value == "F":
matchSadToYear = bool(0)
elif option == "MATCH_EMISS_TO_YEAR":
matchEmissToYear = bool(1)
if value == "F":
matchEmissToYear = bool(0)
elif option == "MATCH_AERDUST_TO_YEAR":
matchAerToYear = bool(1)
if value == "F":
matchAerToYear = bool(0)
elif option == "MATCH_LIGHT_TO_YEAR":
matchLightToYear = bool(1)
if value == "F":
matchLightToYear = bool(0)
else:
print "I don't understand this MATCH option and this could be a problem."
elif re.search('INITIAL', line):
splitString = line.split("=")
options = splitString[0].split(" ")
option = options[1]
value = splitString[1]
if option == "INITIAL_YEAR_FORC_BC":
initialYearForcBc = value
elif option == "INITIAL_YEAR_LIGHT":
initialYearLight = value
else:
print "I don't understand this INITIAL option and this could be a problme"
# read the starting namelist file
infil = '%s/%s' % (pathToNameList, namelistfile)
nfil = open(infil, "r")
namelistContent = nfil.readlines()
nfil.close()
if self.fortranNameList == "true":
splitter = "="
endLine = ",\n"
else:
splitter = ":"
endLine = "\n"
# get orginal problem name
# it is assumed the month is in the name
foundIt = -1
for line in namelistContent:
if re.search('problem_name', line):
origproblemname = line.split(splitter)[1]
origproblemname.strip()
foundIt = 0
if foundIt != 0:
print "Problem finding the problem_name. Check the namelist"
sys.exit(-1)
splitProblemName = origproblemname.split("_")
counter = 0
for word in splitProblemName:
string.join(word.split(), "")
splitProblemName[counter] = word.strip()
splitProblemName[counter] = word.strip("'")
splitProblemName[counter] = word.strip("',\n")
counter = counter + 1
# get orginal problem name
# it is assumed the month is in the name
# baseProblemName=namelistContent[1].split("'")[1]
# splitProblemName=baseProblemName.split("_")
# find the month to start on
for monthNumber in xrange(0, 12, 1):
if startmonth == monthNames[monthNumber]:
istartmonth = monthNumber
monthId = istartmonth
yearFlag = 0
currentYear = startyear
restartYear = currentYear
# will be used to make the namelists file
listOfNamelists = []
# for each namelist
currentYear = str(int(startyear))
for currentNameListCount in xrange(0, numNameLists, 1):
# open the file
nfil = open(infil, "r")
# flag for a new year (11->DEC
if istartmonth == 11:
yearFlag = 1
if yearFlag == 1:
restartYear = str(int(currentYear) + 1)
yearFlag = 0
if monthId == 0:
restartYear = str(int(currentYear) - 1)
else:
restartYear = str(int(currentYear))
currentMonth = monthNames[monthId]
metfilelist = '%s%s.list' % (currentMonth, currentYear)
finaldays = daysPerMonth[monthId]
if monthId < 9:
strmonthId = '0' + str(monthId + 1)
else:
strmonthId = str(monthId + 1)
# create namelist variables
if self.oneDay == 1:
endymd = self.endDate
startymd = self.startDate
else:
endymd = currentYear[0:4] + strmonthId + str(
daysPerMonth[monthId])
endymd = autoTool.incrementDate(endymd)
if endymd[4:8] == "0229":
newendymd = endymd[0:4] + "0301"
endymd = newendymd
startymd = currentYear[0:4] + strmonthId + '01'
print "endymd: ", endymd
problemName1 = splitProblemName[0].strip()
problemName2 = splitProblemName[1].strip()
restartName = problemName1 + '_' + problemName2 + \
'_' + restartYear + '_'+ \
monthNames[monthId-1]
restartFileName = '%s.rst.nc' % (restartName)
newProblemName = problemName1 + '_' + problemName2 + \
'_' + currentYear + '_' + currentMonth
directoryPath = '%s/' % (pathToNameList)
dflag = os.path.exists(directoryPath)
if dflag == 0:
os.makedirs(directoryPath)
listOfNamelists.append(newProblemName + ".in")
newProblemNameFullPath = '%s%s.in' % (directoryPath,
newProblemName)
ofil = open(newProblemNameFullPath, "w")
if self.fortranNameList == "true":
newProblemName = "'" + newProblemName + "'"
metfilelist = "'" + metfilelist + "'"
print "newProblemName: ", newProblemName
# create new file
for inline in nfil.readlines():
# find the variable name / keyWord
splitLine = inline.split()
# skip if there are no items
if len(splitLine) == 0:
continue
keyWord = splitLine[0]
splitKeyWord = re.split(splitter, keyWord)
keyWord = splitKeyWord[0]
if len(splitLine) == 2:
value = splitLine[1]
if keyWord == 'problem_name':
print "newProblemName: ", newProblemName
ofil.write('problem_name' + splitter + newProblemName +
endLine)
elif keyWord == 'BEG_YY':
print "BEG_YY: ", currentYear[0:4]
ofil.write('BEG_YY' + splitter + currentYear[0:4] +
endLine)
elif keyWord == 'BEG_MM':
print "BEG_MM: ", str(int(strmonthId))
ofil.write('BEG_MM' + splitter + str(int(strmonthId)) +
endLine)
elif keyWord == 'BEG_DD':
print "BEG_DD: 0"
ofil.write('BEG_DD' + splitter + ' 1' + endLine)
elif keyWord == 'BEG_H':
print "BEG_H: 0"
ofil.write('BEG_H' + splitter + ' 0' + endLine)
elif keyWord == 'BEG_M':
print "BEG_M: 0"
ofil.write('BEG_M' + splitter + ' 0' + endLine)
elif keyWord == 'BEG_S':
print "BEG_S: 0"
ofil.write('BEG_S' + splitter + ' 0' + endLine)
elif keyWord == "END_YY":
ofil.write('END_YY' + splitter + endymd[0:4] + endLine)
elif keyWord == "END_MM":
ofil.write('END_MM' + splitter + str(int(endymd[4:6])) +
endLine)
elif keyWord == "END_DD":
ofil.write('END_DD' + splitter + str(int(endymd[6:8])) +
endLine)
elif keyWord == 'END_H':
ofil.write('END_H' + splitter + ' 0' + endLine)
elif keyWord == 'END_M':
ofil.write('END_M' + splitter + ' 0' + endLine)
elif keyWord == 'END_S':
ofil.write('END_S' + splitter + ' 0' + endLine)
elif keyWord == 'met_filnam_list':
ofil.write('met_filnam_list' + splitter + metfilelist +
endLine)
elif keyWord == 'restart_infile_name' and \
self.replaceRestartFile != 0:
print "changing restart!"
ofil.write('restart_infile_name' + splitter +
restartFileName + endLine)
elif keyWord == 'forc_bc_start_num' and \
matchBcToYear == True:
forcBcStartNum = int(
startymd[0:4]) - int(initialYearForcBc)
if forcBcStartNum < 0:
print "forcBcStartNum cannot be less than 0!"
sys.exit(-1)
print "forcBcStartNum: ", forcBcStartNum
ofil.write('forc_bc_start_num' + splitter +
str(forcBcStartNum) + endLine)
#ofil.write('forc_bc_start_num' + splitter + str(int(startymd[0:4]) - 1949) + endLine)
elif keyWord == 'lightYearDim' and \
matchLightToYear == True:
lightYearStartNum = int(
startymd[0:4]) - int(initialYearLight)
if lightYearStartNum < 0:
print "lightYearStartNum cannot be less than 0!"
sys.exit(-1)
print "lightYearStartNum: ", lightYearStartNum
ofil.write('lightYearDim' + splitter +
str(lightYearStartNum) + endLine)
#ofil.write('lightYearDim' + splitter + str(int(startymd[0:4]) - 1989) + endLine)
elif keyWord == 'lbssad_infile_name' and \
matchSadToYear == True:
print "changing lbssad"
splitLbssad = re.split("_", value)
newLbssad = splitLbssad[0] + "_" + splitLbssad[
1] + "_" + splitLbssad[2] + "_" + startymd[0:4] + ".nc"
ofil.write('lbssad_infile_name' + splitter + newLbssad +
endLine)
elif keyWord == 'emiss_infile_name' and \
matchEmissToYear == True:
splitEmiss = re.split("_", value)
newEmiss = splitEmiss[0] + "_" + startymd[
0:4] + "_" + splitEmiss[2] + "_" + splitEmiss[
3] + "_" + splitEmiss[4]
print "newEmiss: ", newEmiss
ofil.write('emiss_infile_name' + splitter + newEmiss +
endLine)
#/discover/nobackup/projects/gmi/gmidata2/input/chemistry/aerodust/aerodust_MERRA2_1x1.25x72_20190101.nc
elif keyWord == 'AerDust_infile_name' and \
matchAerToYear == True:
print "Modified"
splitAer = re.split("_", value)
newAer = splitAer[0] + "_" + splitAer[1] + "_" + splitAer[
2] + "_" + startymd[0:4] + "0101.nc"
print "newAer: ", newAer
ofil.write('AerDust_infile_name' + splitter + newAer +
endLine)
elif keyWord == 'begGmiDate':
ofil.write('begGmiDate' + splitter + currentYear + strmonthId \
+ '01' + endLine)
elif keyWord == 'begGmiTime':
ofil.write('begGmiTime' + splitter + '000000' + endLine)
elif keyWord == 'endGmiDate':
ofil.write('endGmiDate' + splitter +
str(int(endymd[0:8])) + endLine)
elif keyWord == 'endGmiTime':
ofil.write('endGmiTime' + splitter + '000000' + endLine)
elif keyWord == 'gmi_sec':
ofil.write('gmi_sec' + splitter + self.gmiSeconds +
endLine)
elif keyWord == 'rd_restart' and \
self.replaceRestartFile != 0:
ofil.write('rd_restart' + splitter + 'T' + endLine)
else:
data = inline.replace(
'\r', '') #Removes ^M character from strings written
ofil.write(data)
intYear = int(currentYear)
monthId = monthId + 1
if monthId == 12:
currentYear = str(intYear + 1)
monthId = 0
ioObject.writeToFile(listOfNamelists,
pathToNameList + "/namelists.list")
if int(currentYear) % 4 == 0 and monthId == 2:
print "leap year!"
daysInMonth = 29
else:
daysInMonth = int(daysPerMonth[monthId - 1])
gmiSeconds = int(self.gmiSeconds) + (daysInMonth * 24 * 3600)
# write new gmiSeconds to file
ioObject.writeToFile(["export gmi_sec=" + str(gmiSeconds)],
self.envFile)
for nameList in listOfNamelists:
print nameList
class GmiDasTasks:
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# Constructor routine.
#---------------------------------------------------------------------------
def __init__(self, realTime):
self.comUtilities = CommonUtilities ()
self.autoConstants = GmiAutomationConstants ()
self.gmiAutomationTool = GmiAutomationTools ()
self.ioRoutines = IoRoutines ()
self.netCdfTools = GmiNetCdfFileTools ()
self.realTime = realTime
self.logUtility = GmiLogUtilities ()
self.geos5Fields = GmiGEOS5DasFields ()
self.basePath = ''
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# Destructor routine.
#---------------------------------------------------------------------------
def __del__(self):
pass
def processTask (self, task, dasObjects, destPath, processDirectory, \
archiveDirectory, remoteSystem, levsFile, constFieldsPath, \
mailTo, logFile, \
configFile, taskFile, transferFile):
try:
#---------------------------------------------------------------------------
#Only needed in debugging cases when removing steps below
#---------------------------------------------------------------------------
for dasObject in dasObjects:
dasObject.SOURCESTYLE = "GMI"
#---------------------------------------------------------------------------
self.basePath = task.destinationPath + "/" + \
task.year + "/" + task.month + \
"/" + task.filePrefix + "."
print "Creating necessary directories ..."
self.createDirectories (task, remoteSystem, archiveDirectory, transferFile, \
processDirectory)
print "Done creating necessary directories ..."
# By now, the data has been staged
task.sourcePath = task.destinationPath
print "Checking for DAS source files ..."
self.checkSourceFiles (task, dasObjects, mailTo)
print "Done checking for DAS source files ..."
print "Distributed processing ..."
self.doProcessing (task, dasObjects, destPath)
print "Done with distributed processing ..."
print "Doing optical depth calculation"
self.doOpticalDepth (task, dasObjects)
print "Done calculating optical depth"
# NEED TO MAKE SURE that OPTDEPTH gets carried along
print "Merging across DAS types ..."
self.mergeAcrossDataTypes (task, dasObjects, mailTo, levsFile)
print "Done merging across DAS types ..."
print "Adding lat,lev,and time file"
self.add1DVars (task)
print "Done adding lat,lev, and time"
print "Remapping the lon coordinate"
self.remapData (task, mailTo, levsFile)
print "Done remapping the lon coordinate"
print "rename Fields..."
self.renameFields (task)
print "Done renaming Fields..."
print "add longtitude array to all files"
self.addLongitudeToArray (task)
print "done adding longitude array to all files"
print "Checking output files ..."
self.checkFiles (task)
print "Done checking output files ..."
print "Archiving the finished met fields and logging ..."
self.archiveFilesAndLog (task, dasObjects, archiveDirectory, \
remoteSystem, mailTo, logFile, \
transferFile, processDirectory)
print "Done archiving the finished met fields and logging ..."
self.removeInterFiles (task, dasObjects)
print "Done removing intermmediate files ..."
print "Removing processed files ..."
self.removeFiles (task)
print "Done removing processed files ..."
print "Updating the config file..."
self.updateConfigFile (task, configFile)
print "Done updating the config file..."
print "Removing task from list..."
self.removeTaskFromAttempts (task, taskFile, mailTo)
print "Done removing task from list..."
print "Removing processing prefixes and fields"
self.removeExtraPrefixesAndFields (dasObjects)
print "Done removing processing prefixes and fields"
print "Mailing success message..."
self.ioRoutines.mailMessage ("Succesful completion of the task: " + \
task.year + " " + task.month + " " + \
task.day, "MERRA-2 Success", mailTo)
print "Done mailing success message..."
except Exception, error:
print "GmiDasTasks error: ", error
self.ioRoutines.mailMessage ("GmiDasTasks error: "+ str(error), \
self.autoConstants.ERROR_SUBJECT, \
mailTo)
class GmiProduction:
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# Constructor routine.
#---------------------------------------------------------------------------
def __init__(self):
self.queueFileName = ""
self.nameListName = ""
self.queueJobName = ""
self.runDirectory = ""
self.storageDirectory = ""
self.longDescription = ""
self.segmentDescription = ""
self.modelVersion = ""
self.mailTo = ""
self.queueId = ""
self.base = ""
self.year = ""
self.month = ""
self.archiveSystem = ""
self.scali = None
self.exceptions = GmiExceptions ()
self.constants = GmiAutomationConstants ()
self.automationObject = GmiAutomationTools ()
self.ioObject = IoRoutines ()
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# Destructor routine.
#---------------------------------------------------------------------------
def __del__(self):
pass
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine will modify the file name specified by queueFileName.
# The job name and the namelist file in the mpirun
# command will be changed.
#---------------------------------------------------------------------------
def modifyQueueFile (self, defaultQueueFile, numProcessors, numProcessorsPerNode, \
chargeCode, chemicalMechanism, destinationDirectory, wallTime, \
useNlInMpi, lastJobId):
print "numProcessorsPerNode: ", numProcessorsPerNode
print "useNlInMpi: ", useNlInMpi
if len (self.nameListName) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if len (self.runDirectory) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if numProcessors <= 0:
raise self.exceptions.ERROR
if not os.path.exists (defaultQueueFile):
raise self.exceptions.NOSUCHFILE
return
extension = self.nameListName[len(self.nameListName)-len('.in'):len(self.nameListName)]
if extension != ".in":
raise self.exceptions.BADOBJECTDATA
return
self.base = self.nameListName[0:len(self.nameListName)-len(extension)]
print "self.base: ", self.base
# create the new queue file
if len (self.base) > self.constants.MAXPBSJOBNAMELENGTH:
self.queueJobName = self.base[5:self.constants.MAXPBSJOBNAMELENGTH+5]
else:
self.queueJobName = self.base
if self.nameListName != "transfer.in":
self.queueFileName = destinationDirectory + "/" + self.base + '.qsub'
else:
self.queueFileName = destinationDirectory + "/transfer.qsub"
print self.queueFileName
systemCommand = self.constants.CPPATH + 'cp ' + defaultQueueFile + ' ' + self.queueFileName
systemReturnCode = os.system (systemCommand)
if systemReturnCode != 0:
print "\nThere was an error copying the file ", defaultQueueFile, " to ", self.queueFileName
sys.exit (1)
print "reading the file: ", self.queueFileName
# read the new file
fileLines = self.ioObject.readFile (self.queueFileName)
print "files lines from file: ", fileLines
print "numProcessors, ", numProcessors
print "numProcessorsPerNode, ", numProcessorsPerNode
numberOfNodes=math.ceil(float(numProcessors)/float(numProcessorsPerNode))
print "numberOfNodes = ", numberOfNodes
# change the new file
lineCounter = 0
for line in fileLines:
if re.search ("#SBATCH -A", line):
fileLines [lineCounter] = "#SBATCH -A " + chargeCode
elif line[0:7] == "#PBS -N":
fileLines [lineCounter] = "#PBS -N " + self.queueJobName
elif line[0:7] == "#PBS -q":
fileLines [lineCounter] = "#PBS -q " + self.queueName
elif re.search ("mpirun", line):
fileLines [lineCounter] = line + " " + \
" -np " + str(numProcessors) + \
" $GEMHOME/gmi.x "
if useNlInMpi == "T":
fileLines [lineCounter] = fileLines [lineCounter] + \
" -d " + self.nameListName
fileLines [lineCounter] = fileLines [lineCounter] + \
"| tee stdout.log"
elif re.search ("MOCK", line):
fileLines [lineCounter] = " $GEMHOME/./gmi.x " + \
" -d " + self.nameListName + "| tee stdout.log"
# elif re.search ("#PBS -W group_list", line):
# fileLines [lineCounter] = "#PBS -W group_list=" + chargeCode
# elif re.search ("#PBS -W depend", line):
elif re.search ("#SBATCH --depend", line):
print "found depend line"
if lastJobId != None:
print "job id is not none"
fileLines [lineCounter] = "#SBATCH --dependency=afterany:" + lastJobId
else:
print "IN ELSE"
fileLines [lineCounter] = ""
elif re.search ("CHEMCASE", line):
fileLines [lineCounter] = "setenv CHEMCASE " + chemicalMechanism
elif re.search ("setenv workDir", line):
fileLines [lineCounter] = "setenv workDir " + self.runDirectory
elif re.search ("setenv GEMHOME", line):
fileLines [lineCounter] = "setenv GEMHOME " + self.runDirectory
elif re.search ("select=", line):
print "replacing select line"
fileLines [lineCounter] = "#PBS -l select=" + str (int(numberOfNodes)) + ":ncpus=" + numProcessorsPerNode + ":mpiprocs=" + numProcessorsPerNode
if self.scali != None:
if self.scali == "true":
fileLines [lineCounter] = fileLines [lineCounter] + ":scali=true"
elif re.search ("walltime", line):
fileLines [lineCounter] = "#PBS -l walltime=" + wallTime
elif re.search ("cp namelist.list gmiResourceFile.rc", line):
fileLines [lineCounter] = "cp " + self.nameListName + " gmiResourceFile.rc"
elif re.search ("bbscp", line):
if re.search ("bbscp year/diagnostics", line):
fileLines [lineCounter] = "bbscp " + self.year + "/diagnostics/*" + \
self.year + "*" + self.month + "* " + self.archivesystem + ":" \
+ self.storageDirectory
else:
filesLines [lineCounter] = ""
lineCounter = lineCounter + 1
# write the file
self.ioObject.touchNewFile (self.queueFileName)
self.ioObject.writeToFile (fileLines, self.queueFileName)
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routines submits the job using the queueFileName in the object.
# The job id is both returned and set in the object as queueId.
#---------------------------------------------------------------------------
def submitJobToQueue (self):
if len (self.queueFileName) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if not os.path.exists (self.queueFileName):
raise self.exceptions.NOSUCHFILE
return
self.queueId = commands.getoutput (self.constants.QSUBPATH + "qsub " + self.queueFileName)
return self.queueId
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine returns the status of a queue job either JOBINQUEUE or
# JOBNOTINQUEUE.
#---------------------------------------------------------------------------
def isJobInQueue (self):
if len (self.queueId) <= 0:
raise self.exceptions.BADOBJECTDATA
return
qstatCommand = self.constants.QSTATPATH + "qstat -p | " + self.constants.GREPPATH + "grep " + self.queueId + " | " + self.constants.AWKPATH +"awk \'{print $1}\' "
status = commands.getoutput (qstatCommand)
if status == '':
return self.constants.JOBNOTINQUEUE
if status == self.queueId:
return self.constants.JOBINQUEUE
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine checks for specific files that are expected from a GMI run.
#---------------------------------------------------------------------------
def checkForValidRun (self):
if len (self.nameListName) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if len (self.runDirectory) <= 0:
raise self.exceptions.BADOBJECTDATA
return
extension = self.nameListName[len(self.nameListName)-len('.in'):len(self.nameListName)]
if extension != ".in":
raise self.exceptions.BADOBJECTDATA
return
fileList = self.automationObject.getMatchingFiles (self.runDirectory, ".asc")
if len (fileList) <= 0:
return self.constants.NOTVALIDRUN
# read the standard output file
splitString = string.split (self.queueId, ".")
queueNumber = splitString [0]
if queueNumber == "":
return self.constants.NOTVALIDRUN
fileList = self.automationObject.getMatchingFiles (self.runDirectory, queueNumber)
if len (fileList) <= 0:
return self.constants.NOTVALIDRUN
standardOutFile = fileList [0]
fileLines = self.ioObject.readFile (self.runDirectory + "/" + standardOutFile)
if len (fileLines) <= 0:
return self.constants.NOTVALIDRUN
# now search the file for the following string:
searchString = "Successful completion of the run"
foundIt = 0
for line in fileLines:
if re.search (searchString, line):
foundIt = 1
break
if foundIt == 0:
return self.constants.NOTVALIDRUN
return self.constants.VALIDRUN
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine transfers data to a "completed" dieectory under a run
# directory and catalogs them by job segment.
#---------------------------------------------------------------------------
def copyOutputDataToTempArchive (self, remoteSystem):
print "top of copyOutputData"
if len (self.nameListName) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if len (self.runDirectory) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if len (remoteSystem) == 0:
if not os.path.exists (self.runDirectory):
print "The runDirectory is : ", self.runDirectory
raise self.exceptions.NOSUCHPATH
return
status = [] # contains status return codes and information
if len (self.base) <= 0:
self.base = self.nameListName[0:len(self.nameListName)-len('.in')]
# "completed/YYYY" directory
self.storageDirectory = self.runDirectory + "/completed" + "/" + self.year
if len (remoteSystem) == 0:
self.automationObject.createDirectoryStructure (self.storageDirectory)
else:
systemCommand = "ssh " + remoteSystem + " mkdir -p " + \
self.storageDirectory
print systemCommand
os.system (systemCommand)
# "run_info/PET"
runInfoDirectory = self.storageDirectory + "/run_info"
petDirectory = runInfoDirectory + "/PET"
if len (remoteSystem) == 0:
self.automationObject.createDirectoryStructure (petDirectory)
else:
status.append (os.system ("ssh " + remoteSystem + " mkdir -p " + \
petDirectory))
# move the PET files
systemCommand = "mv -f " + self.runDirectory + "/PET* " + petDirectory
print systemCommand
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
# move all .nc files to the storageDirectory for now
systemCommand = "mv -f " + self.runDirectory + "/*.nc " + self.storageDirectory
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
# move the restart file
if len (remoteSystem) == 0:
status.append (self.moveFileToDirectory (self.storageDirectory + "/" + \
self.base + ".rst.nc", #
runInfoDirectory))
else:
status.append (os.system ("ssh " + remoteSystem + " mv -f " + \
self.storageDirectory + "/" + self.base \
+ ".rst.nc " + runInfoDirectory))
# now copy it back for the next segment
systemCommand = "cp " + runInfoDirectory + "/*" + self.base + "*.rst.nc " + \
self.runDirectory
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
# move the standard error/output file from the run directory
# and rename it to $base.out in the run_info directory
# get the numerical part of the queueId
# POSSIBLE BUG for non discover NCCS systems
splitString = string.split (self.queueId, ".")
queueNumber = splitString [0]
systemCommand = "mv -f " + self.runDirectory + "/*.e" + queueNumber + " " + \
runInfoDirectory + "/" + self.base + ".out"
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
# ascii file
systemCommand = "mv -f " + self.runDirectory + "/*" + self.base + "*.asc " + runInfoDirectory
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
# qsub file
systemCommand = "mv -f " + self.runDirectory + "/*" + self.base + "*.qsub " + runInfoDirectory
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
# log directory
systemCommand = "mv -f " + self.runDirectory + "/*" + "esm_log_*" + self.base + " " + \
runInfoDirectory
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
# timing file
systemCommand = "mv -f " + self.runDirectory + "/*" + "esm_timing.*" + self.base + " " + \
runInfoDirectory
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
# ftiming file
systemCommand = "mv -f " + self.runDirectory + "/ftiming* " + runInfoDirectory
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
# namelist file
systemCommand = "mv -f " + self.runDirectory + "/" + self.nameListName + " " + runInfoDirectory
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
# metfields file
metFileDirectory = self.runDirectory + "/completed/" + "/metfile_lists/"
if len (remoteSystem) == 0:
self.automationObject.createDirectoryStructure (metFileDirectory)
else:
status.append (os.system ("ssh " + remoteSystem + " mkdir -p " + \
metFileDirectory))
systemCommand = "cp " + self.runDirectory + "/*" + self.month + self.year + \
"*.list " + metFileDirectory
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
# diagostics files
diagnosticsDirectory = self.storageDirectory + "/diagnostics"
if len (remoteSystem) == 0:
self.automationObject.createDirectoryStructure (diagnosticsDirectory)
else:
status.append (os.system ("ssh " + remoteSystem + " mkdir -p " + \
diagnosticsDirectory))
# flux file
systemCommand = "mv -f " + self.storageDirectory + "/*" + self.base + "*_flux.nc " + \
diagnosticsDirectory
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
# tend file
systemCommand = "mv -f " + self.storageDirectory + "/*" + self.base + "*.tend.nc " + \
diagnosticsDirectory
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
# qk file
systemCommand = "mv -f " + self.storageDirectory + "/*" + self.base + "*.qk.nc " \
+ diagnosticsDirectory
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
# qj file
systemCommand = "mv " + self.storageDirectory + "/*" + self.base + "*.qj.nc " + \
diagnosticsDirectory
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
# qqjk file
systemCommand = "mv -f " + self.storageDirectory + "/*" + self.base + "*.qqjk.nc " + \
diagnosticsDirectory
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
# station output
stationDirectory = self.storageDirectory + "/stations/" + self.month + "/"
if len (remoteSystem) == 0:
self.automationObject.createDirectoryStructure (stationDirectory)
else:
status.append (os.system ("ssh " + remoteSystem + " mkdir -p " + \
stationDirectory))
systemCommand = "mv -f " + self.storageDirectory + "/*" + self.base + \
"*.profile.nc " + stationDirectory
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
# summary folders
systemCommand = "mv -f " + self.runDirectory + "/completed/gmic_* " + runInfoDirectory
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
systemCommand = "mv -f " + self.runDirectory + "/stdout.log " + runInfoDirectory
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
systemCommand = "cp " + self.runDirectory + "/gmi.x " + runInfoDirectory
if len (remoteSystem) == 0:
status.append (os.system (systemCommand))
else:
status.append (os.system ("ssh " + remoteSystem + " " + systemCommand))
return status
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine simple copies the file to the directory specified.
#---------------------------------------------------------------------------
def moveFileToDirectory (self, file, directory):
status = self.constants.NOERROR
if not os.path.exists (file):
status = self.constants.INCOMPLETEDATASET
else:
output = commands.getoutput (self.constants.MVPATH + "mv " + file + " " + directory)
if output != "":
status = self.constants.WRITEERROR
print "here-last"
return status
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine takes information from the GmiProduction object and writes
# a summary to the "completed" directory.
# Precondition: nameListName defined, runDirectory, what about qsubFile?
#---------------------------------------------------------------------------
def writeSummary (self) :
if len (self.nameListName) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if len (self.runDirectory) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if not os.path.exists (self.runDirectory):
raise self.exceptions.NOSUCHPATH
return
if len (self.base) <= 0:
self.base = self.nameListName[0:len(self.nameListName)-len('.in')]
if not os.path.exists (self.runDirectory + "/completed/`" + self.base):
self.automationObject.createDirectoryStructure (self.runDirectory + "/completed/" + self.base)
summaryFile = self.runDirectory + "/completed/" + self.base + "/summary.txt"
self.ioObject.touchNewFile (summaryFile)
fileLines = []
if len (self.longDescription) <= 0:
fileLines.append ("Long description: none")
else:
fileLines.append ("Long description: " + self.longDescription)
if len (self.segmentDescription) <= 0:
fileLines.append ("Segment description: " + self.base)
else:
fileLines.append ("Segment description: " + self.segmentDescription)
if len (self.modelVersion) <= 0:
fileLines.append ("Model version: none")
else:
fileLines.append ("Model version: " + self.modelVersion)
if len (self.queueFileName) <= 0:
fileLines.append ("Qsub file: none")
else:
fileLines.append ("Qsub file: " + self.queueFileName)
fileLines.append ("Run directory: " + self.runDirectory)
if len (self.mailTo) <= 0:
fileLines.append ("Mail to: none")
else:
fileLines.apend ("Mail to: " + self.mailTo)
if len (self.queueId) <= 0:
fileLines.append ("Queue Id: none")
else:
fileLines.append ("Queue Id: " + self.queueId)
fileLines.append ("Base: " + self.base)
self.ioObject.writeToFile (fileLines, summaryFile)
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
#
# This routine reads the namelist file and fills in the year and month
# variables in the object.
#---------------------------------------------------------------------------
def fillInYearAndMonth (self):
if len (self.runDirectory) <= 0 or len (self.nameListName) <= 0:
raise self.exceptions.BADOBJECTDATA
return
# read the new file
fileLines = self.ioObject.readFile (self.runDirectory + "/" + self.nameListName)
if len (fileLines) <= 0:
raise self.exceptions.ERROR
for line in fileLines:
if re.search ("begGmiDate", line):
# get the part after the =
splitString = string.split(line, "=")
# get rid of comma and white spce
date = string.strip (splitString[1])
splitString = string.split (date, ",")
ymd = splitString [0]
if len(ymd) != 8:
print "length of ymd is not 8\n";
raise self.exceptions.ERROR
self.year = ymd [0:4]
# get the month (first 3 letters in the month)
when = datetime.date (string.atoi (self.year), #
string.atoi (date[4:6]), string.atoi (date[6:8]))
self.month = string.lower (when.strftime("%b"))
class GmiProduction:
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# Constructor routine.
#---------------------------------------------------------------------------
def __init__(self):
self.queueFileName = ""
self.nameListName = ""
self.queueJobName = ""
self.runDirectory = ""
self.storageDirectory = ""
self.longDescription = ""
self.segmentDescription = ""
self.modelVersion = ""
self.mailTo = ""
self.queueId = ""
self.base = ""
self.year = ""
self.month = ""
self.archiveSystem = ""
self.scali = None
self.exceptions = GmiExceptions()
self.constants = GmiAutomationConstants()
self.automationObject = GmiAutomationTools()
self.ioObject = IoRoutines()
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# Destructor routine.
#---------------------------------------------------------------------------
def __del__(self):
pass
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine will modify the file name specified by queueFileName.
# The job name and the namelist file in the mpirun
# command will be changed.
#---------------------------------------------------------------------------
def modifyQueueFile (self, defaultQueueFile, numProcessors, numProcessorsPerNode, \
chargeCode, chemicalMechanism, destinationDirectory, wallTime, \
useNlInMpi, lastJobId):
print "numProcessorsPerNode: ", numProcessorsPerNode
print "useNlInMpi: ", useNlInMpi
if len(self.nameListName) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if len(self.runDirectory) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if numProcessors <= 0:
raise self.exceptions.ERROR
if not os.path.exists(defaultQueueFile):
raise self.exceptions.NOSUCHFILE
return
extension = self.nameListName[len(self.nameListName) -
len('.in'):len(self.nameListName)]
if extension != ".in":
raise self.exceptions.BADOBJECTDATA
return
self.base = self.nameListName[0:len(self.nameListName) -
len(extension)]
print "self.base: ", self.base
# create the new queue file
if len(self.base) > self.constants.MAXPBSJOBNAMELENGTH:
self.queueJobName = self.base[5:self.constants.
MAXPBSJOBNAMELENGTH + 5]
else:
self.queueJobName = self.base
if self.nameListName != "transfer.in":
self.queueFileName = destinationDirectory + "/" + self.base + '.qsub'
else:
self.queueFileName = destinationDirectory + "/transfer.qsub"
print self.queueFileName
systemCommand = self.constants.CPPATH + 'cp ' + defaultQueueFile + ' ' + self.queueFileName
systemReturnCode = os.system(systemCommand)
if systemReturnCode != 0:
print "\nThere was an error copying the file ", defaultQueueFile, " to ", self.queueFileName
sys.exit(1)
print "reading the file: ", self.queueFileName
# read the new file
fileLines = self.ioObject.readFile(self.queueFileName)
print "files lines from file: ", fileLines
print "numProcessors, ", numProcessors
print "numProcessorsPerNode, ", numProcessorsPerNode
numberOfNodes = math.ceil(
float(numProcessors) / float(numProcessorsPerNode))
print "numberOfNodes = ", numberOfNodes
# change the new file
lineCounter = 0
for line in fileLines:
if re.search("#SBATCH -A", line):
fileLines[lineCounter] = "#SBATCH -A " + chargeCode
elif line[0:7] == "#PBS -N":
fileLines[lineCounter] = "#PBS -N " + self.queueJobName
elif line[0:7] == "#PBS -q":
fileLines[lineCounter] = "#PBS -q " + self.queueName
elif re.search("mpirun", line):
fileLines [lineCounter] = line + " " + \
" -np " + str(numProcessors) + \
" $GEMHOME/gmi.x "
if useNlInMpi == "T":
fileLines [lineCounter] = fileLines [lineCounter] + \
" -d " + self.nameListName
fileLines [lineCounter] = fileLines [lineCounter] + \
"| tee stdout.log"
elif re.search("MOCK", line):
fileLines [lineCounter] = " $GEMHOME/./gmi.x " + \
" -d " + self.nameListName + "| tee stdout.log"
# elif re.search ("#PBS -W group_list", line):
# fileLines [lineCounter] = "#PBS -W group_list=" + chargeCode
# elif re.search ("#PBS -W depend", line):
elif re.search("#SBATCH --depend", line):
print "found depend line"
if lastJobId != None:
print "job id is not none"
fileLines[
lineCounter] = "#SBATCH --dependency=afterany:" + lastJobId
else:
print "IN ELSE"
fileLines[lineCounter] = ""
elif re.search("CHEMCASE", line):
fileLines[lineCounter] = "setenv CHEMCASE " + chemicalMechanism
elif re.search("setenv workDir", line):
fileLines[lineCounter] = "setenv workDir " + self.runDirectory
elif re.search("setenv GEMHOME", line):
fileLines[lineCounter] = "setenv GEMHOME " + self.runDirectory
elif re.search("select=", line):
print "replacing select line"
fileLines[lineCounter] = "#PBS -l select=" + str(
int(numberOfNodes)
) + ":ncpus=" + numProcessorsPerNode + ":mpiprocs=" + numProcessorsPerNode
if self.scali != None:
if self.scali == "true":
fileLines[lineCounter] = fileLines[
lineCounter] + ":scali=true"
elif re.search("walltime", line):
fileLines[lineCounter] = "#PBS -l walltime=" + wallTime
elif re.search("cp namelist.list gmiResourceFile.rc", line):
fileLines[
lineCounter] = "cp " + self.nameListName + " gmiResourceFile.rc"
elif re.search("bbscp", line):
if re.search("bbscp year/diagnostics", line):
fileLines [lineCounter] = "bbscp " + self.year + "/diagnostics/*" + \
self.year + "*" + self.month + "* " + self.archivesystem + ":" \
+ self.storageDirectory
else:
filesLines[lineCounter] = ""
lineCounter = lineCounter + 1
# write the file
self.ioObject.touchNewFile(self.queueFileName)
self.ioObject.writeToFile(fileLines, self.queueFileName)
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routines submits the job using the queueFileName in the object.
# The job id is both returned and set in the object as queueId.
#---------------------------------------------------------------------------
def submitJobToQueue(self):
if len(self.queueFileName) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if not os.path.exists(self.queueFileName):
raise self.exceptions.NOSUCHFILE
return
self.queueId = commands.getoutput(self.constants.QSUBPATH + "qsub " +
self.queueFileName)
return self.queueId
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine returns the status of a queue job either JOBINQUEUE or
# JOBNOTINQUEUE.
#---------------------------------------------------------------------------
def isJobInQueue(self):
if len(self.queueId) <= 0:
raise self.exceptions.BADOBJECTDATA
return
qstatCommand = self.constants.QSTATPATH + "qstat -p | " + self.constants.GREPPATH + "grep " + self.queueId + " | " + self.constants.AWKPATH + "awk \'{print $1}\' "
status = commands.getoutput(qstatCommand)
if status == '':
return self.constants.JOBNOTINQUEUE
if status == self.queueId:
return self.constants.JOBINQUEUE
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine checks for specific files that are expected from a GMI run.
#---------------------------------------------------------------------------
def checkForValidRun(self):
if len(self.nameListName) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if len(self.runDirectory) <= 0:
raise self.exceptions.BADOBJECTDATA
return
extension = self.nameListName[len(self.nameListName) -
len('.in'):len(self.nameListName)]
if extension != ".in":
raise self.exceptions.BADOBJECTDATA
return
fileList = self.automationObject.getMatchingFiles(
self.runDirectory, ".asc")
if len(fileList) <= 0:
return self.constants.NOTVALIDRUN
# read the standard output file
splitString = string.split(self.queueId, ".")
queueNumber = splitString[0]
if queueNumber == "":
return self.constants.NOTVALIDRUN
fileList = self.automationObject.getMatchingFiles(
self.runDirectory, queueNumber)
if len(fileList) <= 0:
return self.constants.NOTVALIDRUN
standardOutFile = fileList[0]
fileLines = self.ioObject.readFile(self.runDirectory + "/" +
standardOutFile)
if len(fileLines) <= 0:
return self.constants.NOTVALIDRUN
# now search the file for the following string:
searchString = "Successful completion of the run"
foundIt = 0
for line in fileLines:
if re.search(searchString, line):
foundIt = 1
break
if foundIt == 0:
return self.constants.NOTVALIDRUN
return self.constants.VALIDRUN
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine transfers data to a "completed" dieectory under a run
# directory and catalogs them by job segment.
#---------------------------------------------------------------------------
def copyOutputDataToTempArchive(self, remoteSystem):
print "top of copyOutputData"
if len(self.nameListName) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if len(self.runDirectory) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if len(remoteSystem) == 0:
if not os.path.exists(self.runDirectory):
print "The runDirectory is : ", self.runDirectory
raise self.exceptions.NOSUCHPATH
return
status = [] # contains status return codes and information
if len(self.base) <= 0:
self.base = self.nameListName[0:len(self.nameListName) -
len('.in')]
# "completed/YYYY" directory
self.storageDirectory = self.runDirectory + "/completed" + "/" + self.year
if len(remoteSystem) == 0:
self.automationObject.createDirectoryStructure(
self.storageDirectory)
else:
systemCommand = "ssh " + remoteSystem + " mkdir -p " + \
self.storageDirectory
print systemCommand
os.system(systemCommand)
# "run_info/PET"
runInfoDirectory = self.storageDirectory + "/run_info"
petDirectory = runInfoDirectory + "/PET"
if len(remoteSystem) == 0:
self.automationObject.createDirectoryStructure(petDirectory)
else:
status.append (os.system ("ssh " + remoteSystem + " mkdir -p " + \
petDirectory))
# move the PET files
systemCommand = "mv -f " + self.runDirectory + "/PET* " + petDirectory
print systemCommand
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
# move all .nc files to the storageDirectory for now
systemCommand = "mv -f " + self.runDirectory + "/*.nc " + self.storageDirectory
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
# move the restart file
if len(remoteSystem) == 0:
status.append (self.moveFileToDirectory (self.storageDirectory + "/" + \
self.base + ".rst.nc", #
runInfoDirectory))
else:
status.append (os.system ("ssh " + remoteSystem + " mv -f " + \
self.storageDirectory + "/" + self.base \
+ ".rst.nc " + runInfoDirectory))
# now copy it back for the next segment
systemCommand = "cp " + runInfoDirectory + "/*" + self.base + "*.rst.nc " + \
self.runDirectory
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
# move the standard error/output file from the run directory
# and rename it to $base.out in the run_info directory
# get the numerical part of the queueId
# POSSIBLE BUG for non discover NCCS systems
splitString = string.split(self.queueId, ".")
queueNumber = splitString[0]
systemCommand = "mv -f " + self.runDirectory + "/*.e" + queueNumber + " " + \
runInfoDirectory + "/" + self.base + ".out"
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
# ascii file
systemCommand = "mv -f " + self.runDirectory + "/*" + self.base + "*.asc " + runInfoDirectory
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
# qsub file
systemCommand = "mv -f " + self.runDirectory + "/*" + self.base + "*.qsub " + runInfoDirectory
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
# log directory
systemCommand = "mv -f " + self.runDirectory + "/*" + "esm_log_*" + self.base + " " + \
runInfoDirectory
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
# timing file
systemCommand = "mv -f " + self.runDirectory + "/*" + "esm_timing.*" + self.base + " " + \
runInfoDirectory
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
# ftiming file
systemCommand = "mv -f " + self.runDirectory + "/ftiming* " + runInfoDirectory
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
# namelist file
systemCommand = "mv -f " + self.runDirectory + "/" + self.nameListName + " " + runInfoDirectory
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
# metfields file
metFileDirectory = self.runDirectory + "/completed/" + "/metfile_lists/"
if len(remoteSystem) == 0:
self.automationObject.createDirectoryStructure(metFileDirectory)
else:
status.append (os.system ("ssh " + remoteSystem + " mkdir -p " + \
metFileDirectory))
systemCommand = "cp " + self.runDirectory + "/*" + self.month + self.year + \
"*.list " + metFileDirectory
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
# diagostics files
diagnosticsDirectory = self.storageDirectory + "/diagnostics"
if len(remoteSystem) == 0:
self.automationObject.createDirectoryStructure(
diagnosticsDirectory)
else:
status.append (os.system ("ssh " + remoteSystem + " mkdir -p " + \
diagnosticsDirectory))
# flux file
systemCommand = "mv -f " + self.storageDirectory + "/*" + self.base + "*_flux.nc " + \
diagnosticsDirectory
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
# tend file
systemCommand = "mv -f " + self.storageDirectory + "/*" + self.base + "*.tend.nc " + \
diagnosticsDirectory
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
# qk file
systemCommand = "mv -f " + self.storageDirectory + "/*" + self.base + "*.qk.nc " \
+ diagnosticsDirectory
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
# qj file
systemCommand = "mv " + self.storageDirectory + "/*" + self.base + "*.qj.nc " + \
diagnosticsDirectory
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
# qqjk file
systemCommand = "mv -f " + self.storageDirectory + "/*" + self.base + "*.qqjk.nc " + \
diagnosticsDirectory
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
# station output
stationDirectory = self.storageDirectory + "/stations/" + self.month + "/"
if len(remoteSystem) == 0:
self.automationObject.createDirectoryStructure(stationDirectory)
else:
status.append (os.system ("ssh " + remoteSystem + " mkdir -p " + \
stationDirectory))
systemCommand = "mv -f " + self.storageDirectory + "/*" + self.base + \
"*.profile.nc " + stationDirectory
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
# summary folders
systemCommand = "mv -f " + self.runDirectory + "/completed/gmic_* " + runInfoDirectory
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
systemCommand = "mv -f " + self.runDirectory + "/stdout.log " + runInfoDirectory
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
systemCommand = "cp " + self.runDirectory + "/gmi.x " + runInfoDirectory
if len(remoteSystem) == 0:
status.append(os.system(systemCommand))
else:
status.append(
os.system("ssh " + remoteSystem + " " + systemCommand))
return status
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine simple copies the file to the directory specified.
#---------------------------------------------------------------------------
def moveFileToDirectory(self, file, directory):
status = self.constants.NOERROR
if not os.path.exists(file):
status = self.constants.INCOMPLETEDATASET
else:
output = commands.getoutput(self.constants.MVPATH + "mv " + file +
" " + directory)
if output != "":
status = self.constants.WRITEERROR
print "here-last"
return status
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
# This routine takes information from the GmiProduction object and writes
# a summary to the "completed" directory.
# Precondition: nameListName defined, runDirectory, what about qsubFile?
#---------------------------------------------------------------------------
def writeSummary(self):
if len(self.nameListName) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if len(self.runDirectory) <= 0:
raise self.exceptions.BADOBJECTDATA
return
if not os.path.exists(self.runDirectory):
raise self.exceptions.NOSUCHPATH
return
if len(self.base) <= 0:
self.base = self.nameListName[0:len(self.nameListName) -
len('.in')]
if not os.path.exists(self.runDirectory + "/completed/`" + self.base):
self.automationObject.createDirectoryStructure(self.runDirectory +
"/completed/" +
self.base)
summaryFile = self.runDirectory + "/completed/" + self.base + "/summary.txt"
self.ioObject.touchNewFile(summaryFile)
fileLines = []
if len(self.longDescription) <= 0:
fileLines.append("Long description: none")
else:
fileLines.append("Long description: " + self.longDescription)
if len(self.segmentDescription) <= 0:
fileLines.append("Segment description: " + self.base)
else:
fileLines.append("Segment description: " + self.segmentDescription)
if len(self.modelVersion) <= 0:
fileLines.append("Model version: none")
else:
fileLines.append("Model version: " + self.modelVersion)
if len(self.queueFileName) <= 0:
fileLines.append("Qsub file: none")
else:
fileLines.append("Qsub file: " + self.queueFileName)
fileLines.append("Run directory: " + self.runDirectory)
if len(self.mailTo) <= 0:
fileLines.append("Mail to: none")
else:
fileLines.apend("Mail to: " + self.mailTo)
if len(self.queueId) <= 0:
fileLines.append("Queue Id: none")
else:
fileLines.append("Queue Id: " + self.queueId)
fileLines.append("Base: " + self.base)
self.ioObject.writeToFile(fileLines, summaryFile)
#---------------------------------------------------------------------------
# AUTHORS: Megan Damon NASA GSFC / NGIT / TASC
#
# DESCRIPTION:
#
# This routine reads the namelist file and fills in the year and month
# variables in the object.
#---------------------------------------------------------------------------
def fillInYearAndMonth(self):
if len(self.runDirectory) <= 0 or len(self.nameListName) <= 0:
raise self.exceptions.BADOBJECTDATA
return
# read the new file
fileLines = self.ioObject.readFile(self.runDirectory + "/" +
self.nameListName)
if len(fileLines) <= 0:
raise self.exceptions.ERROR
for line in fileLines:
if re.search("begGmiDate", line):
# get the part after the =
splitString = string.split(line, "=")
# get rid of comma and white spce
date = string.strip(splitString[1])
splitString = string.split(date, ",")
ymd = splitString[0]
if len(ymd) != 8:
print "length of ymd is not 8\n"
raise self.exceptions.ERROR
self.year = ymd[0:4]
# get the month (first 3 letters in the month)
when = datetime.date(
string.atoi(self.year), #
string.atoi(date[4:6]),
string.atoi(date[6:8]))
self.month = string.lower(when.strftime("%b"))
import getopt
import sys
import os
import time
import commands
import subprocess
from IoRoutines import IoRoutines
NUM_ARGS=2
NUM_SPLIT_ARGS=5
ioRoutines = IoRoutines()
# Get options from command line
optList, argList = getopt.getopt(sys.argv[1:],'f:e:')
if len (optList) != NUM_ARGS:
usage ()
stdoutFileName = optList[0][1]
expEnvFile = optList[1][1]
try:
expEnvLines = ioRoutines.readFile(expEnvFile)
except:
print "Check the file: ", expEnvFile
sys.exit(-1)