def makeDirAndCopy(dir, jo, logger):
# jo of the form <package>/<options>.py
localPackageDir = os.path.join(dir, os.path.dirname(jo))
optionsFile = os.path.basename(jo)
mkdirIfNew(localPackageDir)
fpJO = fpJOMaker(jo)
logger.debug('JobOption Copy: \n%s\n to %s' %
(fpJO, localPackageDir))
shutil.copy(fpJO, localPackageDir)
def getJobDumpLocation(self, minder):
relPath = minder.runPath.split(minder.workDirs['packages'])[1]
if relPath.startswith('/'): relPath = relPath[1:]
try:
descDumpLoc = join(self.pathToDump, relPath)
except:
descDumpLoc = None
if descDumpLoc and not exists(descDumpLoc):
mkdirIfNew(descDumpLoc)
return descDumpLoc
def createLogDirectories(parentDir):
if not os.path.exists(parentDir) or not os.path.isabs(parentDir):
print 'Unable to setup the logger because the path: ' + str(parentDir)
print 'either does not exist, or is not absolute.'
sys.exit()
try:
mkdirIfNew(os.path.join(parentDir,'logs'))
except Exception, e:
print 'Unable to create the path: ' + str(os.path.join(parentDir,'logs'))
print str(e)
print exc2string(sys.exc_info())
sys.exit()
def checkPath(self, path):
if not path.exists():
m = 'Path: %s does not exist. Path creation ' % str(path)
self.logger.error(m)
try:
mkdirIfNew(path)
m += 'succeeded'
except:
m += 'failed'
self.logger.error(m)
# for now, convert back to the old string format so we dont
# have to ptopogate RTTpath objects to other modules
return str(path)
def checkPaths(config):
pathsToMake = ['workBasePath', 'resultsBasePath']
[mkdirIfNew(config[p]) for p in pathsToMake]
pathsThatMustExistAndBeAbs = [
'workBasePath', 'resultsBasePath', 'jobGroupConfig', 'dataSetCatalog',
'libToolsLoc', 'dCubeCfgFile', 'pathToVetoFile'
]
badPaths = []
badPaths.extend([
config[xmlTagName] for xmlTagName in pathsThatMustExistAndBeAbs
if checkPathsCondition(config, xmlTagName)
])
badPaths.extend([
aPath for aPath in config.get('confiles', {}).values()
if checkPathsCondition2(aPath)
])
msg = ''
if badPaths: msg = 'Bad paths: %s' % str(badPaths)
return msg
def makeDir(self, parentDir):
try:
# create base directories for work directory and results
# directory trees
if os.path.exists(parentDir):
self.logger.debug('Emptying base directory %s' % parentDir)
emptyDirMkDir(parentDir)
else:
self.logger.debug('Creating base directory %s' % parentDir)
mkdirIfNew(parentDir)
# to permit signal in Tester to know if directories have been made
self.status = True
msg = self.logger.debug('Base directory %s ready' % parentDir)
except Exception, e:
msg = 'Error handling base directory %s\nException%s\nTraceback\n%s' % (
parentDir, str(e), exc2string2())
self.logger.error(msg)
def createLogDirectories(parentDir):
if not os.path.exists(parentDir) or not os.path.isabs(parentDir):
print 'Unable to setup the logger because the path: ' + str(parentDir)
print 'either does not exist, or is not absolute.'
sys.exit()
logsDirName = 'logfiles_' + now()
fullLogsDirName = os.path.join(parentDir, logsDirName)
try:
if os.path.exists(fullLogsDirName):
rmdirMkdir(fullLogsDirName)
else:
mkdirIfNew(fullLogsDirName)
except Exception, e:
print 'Unable to create path: ' + str(fullLogsDirName)
print str(e)
print exc2string2()
sys.exit()
def __init__(self, RTTSrcDir, RTTShareDir, RTTCfgFile, RTTLogDir, useDB,
name):
#self.RTTSrcDir = os.environ['RTTSrcDir']
#self.RTTLibDir = os.environ['RTTShareDir']
#self.topLevelRTTconfFileLocation = os.environ['RTTCfgFile']
#self.logDir = os.environ['RTTLogDir']
#self.useDB = os.environ.has_key('RTTFactoryUseDB')
self.RTTSrcDir = RTTSrcDir
self.RTTLibDir = RTTShareDir
self.topLevelRTTconfFileLocation = RTTCfgFile
self.logDir = os.path.normpath(
os.path.join(RTTLogDir, 'logfiles_' + now()))
self.useDB = useDB
self.name = name
mkdirIfNew(self.logDir)
self.logger = makeLocalLog(self.logDir, 'RTTRunnerError',
'RTTRunnerWarning', 'RTTRunnerInfo',
'RTTRunnerDebug', name + '_log')
def setupRunDir(self, jMinder):
"""
Creates run directory (obtained from the descriptor)
Uses JobGroupKits to establish what is needed to be copied to it.
Uses ScriptWriter to write a run script to it.
This version does not write a requirements file into the run
directory.
"""
self.logger.debug("DirectoryMakerProject: entering setupRunDir")
self.logger.debug("setupRunDir processing: %s" %(jMinder.identifiedName))
runPath = jMinder.runPath
def writeScript(fn, script):
ofn = os.path.join(runPath, fn)
of = open(ofn, 'w')
of.write(script)
of.close()
changePerm(ofn, 0777)
dicts = jMinder.scripts.values()
[writeScript(d['scriptName'], d['script']) for d in dicts]
#######################################################################
# handle aux files (= files needed in the run dir to run the job)
#######################################################################
# if the aux file is a sub directory of the Install directory, make
# the corresponding directory in the work directory
dict = {}
for path in jMinder.auxFiles:
base, fn = os.path.split(path)
if base: dict[base] = fn
dirs = dict.keys()
dirs = [os.path.join(runPath, d) for d in dirs]
dirs = [mkdirIfNew(d) for d in dirs]
auxFileDirectory = jMinder.shareArea
# explicitly remove the package XML file, we add it back later
auxFiles = [a for a in jMinder.auxFiles if a.strip() != os.path.basename(jMinder.confFile).strip()]
# copy the files from the install area (sub) dir to the run (sub) dir.
# for f in jMinder.auxFiles:
for f in auxFiles:
src = join(auxFileDirectory, f)
dest = join(runPath, f)
if not os.path.exists(src):
self.logger.error("Non-existent aux file: %s , skipping" % src )
continue
# shutil.copy(src, dest)
os.symlink(src, dest)
changePerm(dest, 0777)
# now add the package XML file
confFileName = os.path.basename(jMinder.confFile).strip()
src = jMinder.confFile.strip()
dst = join(runPath, confFileName)
os.symlink(src, dst)
changePerm(dst, 0777)
# write out a python script to run the RTT test wrappers
self.writeTestRunnerScript(jMinder.shareArea, jMinder.testDBPath, jMinder.runPath)
# Output the RTT argdict pickle file
pickleDict = jMinder.rttArgDictPickle
cPickle.dump(pickleDict['what'], open(pickleDict['where'], 'wb'))
# Now perform tasks specific to the different descriptors
self.descSetUpRunDirFn()
# make a note of the files in the run directory
# at this point - in case a rerun is needed
of = open(os.path.join(runPath, 'listDir.txt'), 'w')
[of.write(p+'\n') for p in os.listdir(runPath)]
of.close()
self.logger.debug("returning from setupRunDir()")
def createDumpLocation(self):
mkdirIfNew(self.pathToDump)
try:
mkdirIfNew(os.path.join(parentDir,'logs'))
except Exception, e:
print 'Unable to create the path: ' + str(os.path.join(parentDir,'logs'))
print str(e)
print exc2string(sys.exc_info())
sys.exit()
logsDirName = 'logs/logfiles_'+now()
fullLogsDirName = os.path.join(parentDir, logsDirName)
try:
if os.path.exists(fullLogsDirName):
rmdirMkdir(fullLogsDirName)
else:
mkdirIfNew(fullLogsDirName)
except Exception, e:
print 'Unable to create path: ' + str(fullLogsDirName)
print str(e)
print exc2string(sys.exc_info())
sys.exit()
return fullLogsDirName
# ----------------------------------------------------------------
def createFileHandler(name,level,formatter):
theHandler = logging.FileHandler(name)
theHandler.setFormatter(formatter)
theHandler.setLevel(level)
