def _getDbDir(self):
dbDict = {HpcScriptBase.GUILLIMIN:
'/lb/project/bws-221-aa/software/tools/annovar_dbs/',
HpcScriptBase.CNG: '/RunsGA/Analyses/software/src/annovar/',
HpcScriptBase.CNG_MAC:
'/RunsGA/Analyses/software/src/annovar/'}
return dbDict[_guessLocation()]
def __init__(self,
cmd,
nbCpus=1,
memory=4,
jobName=None,
targetDir=None,
jobIdList=None,
machineToUseList=None,
machineToExcludeList=None,
onSubmitAction=False,
walltime=None,
queue=None,
email=None,
scriptName=None,
optionList=None):
if _guessLocation() == HpcScriptBase.GUILLIMIN:
memory = 2
self.cmd = cmd
nodeNb = 1
if isinstance(nbCpus, types.TupleType):
nodeNb, nbCpus = nbCpus
self.nbCpus = nbCpus
self.nodeNb = nodeNb
self.memory = memory
self.jobName = jobName
self.targetDir = targetDir
if jobIdList and not isinstance(jobIdList, types.ListType):
jobIdList = [jobIdList]
self.jobIdList = jobIdList
self.machineToUseList = machineToUseList
self.machineToExcludeList = machineToExcludeList
self.onSubmitAction = onSubmitAction
self.walltime = walltime
self.queue = queue
self.email = email
self.scriptName = scriptName
self.optionList = optionList
def _getSubmitParamListFromCmd(self, fileName, expectedFormat,
outputFilePattern, cmd, dumpFileName=None,
fileToJobIdDict=None, cmdList=None, i=None):
if fileToJobIdDict is None:
fileToJobIdDict = {}
baseFile = onSubmitAction = walltime = queue = email = None
machineToUseList = self._machineList
machineToExcludeList = self._machineToExcludeList
nbCpus = nodeNb = 1
memory = 4
if _guessLocation() == HpcScriptBase.GUILLIMIN:
memory = 2
jobName = otherExpectedFormatList = None
if isinstance(expectedFormat, types.TupleType):
expectedFormat, baseExt = expectedFormat
baseFile = self.__getFileWithExtensionInDir(
os.path.dirname(fileName), baseExt)
elif isinstance(expectedFormat, types.ListType):
otherExpectedFormatList = expectedFormat[1:]
expectedFormat = expectedFormat[0]
print 'F', fileName, expectedFormat # , myStr(cmd)
# print cmdList
if fileName is not None and not \
isinstance(fileName, types.ListType) and \
expectedFormat != Utilities.getFileExtension(fileName):
fileName = self.__getFileWithExtensionInDir(
os.path.dirname(fileName), expectedFormat)
outputFile = outputFilePattern
if '%s' in outputFilePattern:
outputFile = outputFilePattern % fileName
doneFile = outputFile + '_done'
if os.path.isfile(doneFile):
print 'Already processed file: %s' % fileName
if isinstance(cmd, Cmd) and cmd.onSubmitAction:
self._onSubmitAction(None, fileName, outputFile, cmd)
return outputFile
cluster = guessHpc()
logSuffix = ''
if not jobName and not isinstance(cmd, types.StringType):
jobName = cmd.jobName
if cluster.TASK_ID_VAR and \
((jobName and '[' in jobName) or
(not isinstance(cmd, types.StringType) and
cmd.scriptName and '[' in cmd.scriptName)):
logSuffix = '_${%s}' % cluster.TASK_ID_VAR
errorFile = outputFile + '%s.err' % logSuffix
outFile = outputFile + '%s.out' % logSuffix
cmdJobIdList = []
scriptName = optionList = None
if not isinstance(cmd, types.StringType):
nbCpus = cmd.nbCpus
jobName = cmd.jobName
memory = cmd.memory
if cmd.targetDir:
errorFile = os.path.join(
cmd.targetDir, os.path.basename(errorFile))
outFile = os.path.join(
cmd.targetDir, os.path.basename(outFile))
if cmd.jobIdList:
cmdJobIdList = cmd.jobIdList
if cmd.machineToUseList:
machineToUseList = cmd.machineToUseList
if cmd.machineToExcludeList:
machineToExcludeList = cmd.machineToExcludeList
if cmd.walltime:
walltime = cmd.walltime
if cmd.queue:
queue = cmd.queue
if cmd.email:
email = cmd.email
if cmd.scriptName:
scriptName = cmd.scriptName
if cmd.optionList:
optionList = cmd.optionList
nodeNb = cmd.nodeNb
onSubmitAction = cmd.onSubmitAction
cmd = cmd.cmd
if jobName and '[' in jobName and not scriptName:
# scriptName = FileNameGetter(outputFile).get('_cmd.sh')
scriptName = outputFile + '_cmd.sh'
cmd += ' 1> %s 2> %s' % (outFile, errorFile)
errorFile = outFile = None
print 'New cmd = [%s]' % cmd
cmd = self._replaceInputAndOutputInStr(cmd, fileName, outputFile)
if baseFile:
cmd = cmd.replace('[base]', baseFile)
if cmd[-1] == ';':
cmd = cmd[:-1]
if logSuffix:
doneFile = doneFile.replace('_done', '%s_done' % logSuffix)
cmd = cmd + ' && touch %s' % doneFile
# print cmd
# if isinstance(cluster, HpcBase):
jobId = None
if isinstance(fileName, types.StringType):
jobId = fileToJobIdDict.get(fileName)
elif fileName is not None:
jobId = [fileToJobIdDict.get(
currentFileName) for currentFileName in fileName if
currentFileName in fileToJobIdDict]
if not jobId:
jobId = []
if not isinstance(jobId, types.ListType):
jobId = [jobId]
if otherExpectedFormatList:
jobId += [fileToJobIdDict.get(otherExpectedFormat)
for otherExpectedFormat in otherExpectedFormatList if
otherExpectedFormat in fileToJobIdDict]
jobId += cmdJobIdList
if jobId:
otherExpectedFormatStr = ''
if otherExpectedFormatList:
otherExpectedFormatStr += ' && ' + \
' && '.join(
['[ -f %s_done ]' % otherExpectedFormat for
otherExpectedFormat in otherExpectedFormatList])
if expectedFormat:
cmd = 'if [ -f %s_done ]%s; then %s; fi' % (
expectedFormat, otherExpectedFormatStr, cmd)
if self._recursive:
if not dumpFileName:
dumpFileName = self.__createCmdListDumpFileAndGetName(
fileName, cmdList, i)
cmd += ' && python %s -p recursiveCmd -f %s -i %d' % (
os.path.abspath(__file__), dumpFileName, i)
return cmd, fileName, outputFile, onSubmitAction, walltime, queue, \
machineToUseList, machineToExcludeList, nbCpus, nodeNb, memory,\
jobName, expectedFormat, otherExpectedFormatList, errorFile,\
outFile, jobId, dumpFileName, email, scriptName, optionList