def analyzeDirectory(tracker, directory, shellDir):
# create the line handle from directory
directoryComp = directory.split(os.sep)
if 'individualMatfiles' in directoryComp:
lineEffector = directoryComp[-4]
expDateTime = directoryComp[-3]
protocol = directoryComp[-1]
lineName = "%s_%s_%s" % (lineEffector, expDateTime, protocol)
else:
lineEffector = directoryComp[-2]
protocol = directoryComp[-1]
lineName = "%s_%s" % (lineEffector, protocol)
# if directoryComp[len(directoryComp)-2] == 'individualMatfiles':
# lineName = directoryComp[len(directoryComp)-4] + '_' + directoryComp[len(directoryComp)-3] + '_' + directoryComp[len(directoryComp)-1]
# else:
# lineName = directoryComp[len(directoryComp)-2] + '_' + directoryComp[len(directoryComp)-1]
cmdLineName = lineName
normLineHandle = functions.sanitizeForFileName(lineName)
normLineHandle = string.replace(normLineHandle,',','_')
shfile = os.path.dirname(shellDir) + os.sep + settings.batchScriptDirProjectsSuffix + 'navigationalAnalysis/matlabNavAnalysis_' + tracker + '_' + normLineHandle + '.sh'
expParameters = "'%s' 'runningOnCluster' '1' " % (cmdLineName)
#functions.createQsubFile(shfile, settings.mcrLocation, settings.matlabScriptAnalysis[tracker], directory, expParameters)
functions.createQsubFile(shfile, '/misc/local/matlab-2019a/', settings.matlabScriptAnalysis[tracker], directory, expParameters)
qsubAdditional = ""
if tracker == 't8':
#qsubAdditional = ' -pe batch 3 '
qsubAdditional = ' -n 3 '
elif tracker == 't1':
#qsubAdditional = ' -pe batch 1 '
qsubAdditional = ' -n 1 '
elif tracker in settings.SPATIAL_TRACKERS:
#qsubAdditional = ' -pe batch 1 '
qsubAdditional = ' -n 1 '
jobName = 'nav_' + tracker + '_' + normLineHandle ;
qsubOut = shfile[:-3] + '.log'
shfile = string.replace(shfile,',','_')
jobName = string.replace(jobName,',','_')
qsubAdditional += settings.supplementaryQsubParams[tracker]
#qsubCommand = 'qsub -N ' + re.escape(jobName) + qsubAdditional + ' -j y -b y -o ' + re.escape(qsubOut) + ' -cwd ' + re.escape(shfile)
qsubCommand = 'bsub -J ' + re.escape(jobName) + qsubAdditional + ' -o ' + re.escape(qsubOut) + ' ' + re.escape(shfile)
os.system("chmod 755 " + re.escape(shfile))
#qsubCommand = re.escape(shfile)
functions.addQsubToQueue(qsubCommand)
return
not filename.endswith('values.bin') and
not filename.endswith('led1.bin') and
not filename.endswith('led2.bin')) : # we only care about bin files
btdfile = 'btdfiles/btd_' + filename[:-4] + ".mat"
os.path.normpath(filename)
lineName, filenameExtension = os.path.splitext(filename)
if os.path.isfile(dirpath + os.sep + btdfile):
print 'Skipping ' + lineName
else:
#run that line
print 'Running ' + lineName
lineHandle = functions.sanitizeForFileName(lineName)
shfile = os.path.dirname(fullProjectDirPath) + os.sep + settings.batchScriptDirProjectsSuffix + 'totalAnalysis/' + lineHandle + '.sh'
#scratch location
mcrCacheRoot = settings.mcrCacheRootBase + lineHandle
expParameters = lineName
functions.createQsubFile(shfile, settings.mcrLocation, settings.matlabScriptComplete[tracker], dirpath, None)
#qsubAdditional = ' -pe batch 8 -l h_rt=2:00:00' + settings.supplementaryQsubParams[tracker]
qsubAdditional = ' -n 8 -W 120' + settings.supplementaryQsubParams[tracker]
jobName = 'Mat_' + tracker + '_' + lineHandle[16:-3] ;
qsubOut = shfile[:-3] + '.log'
njob = njob + 1;
#qsubCommand = 'qsub -N ' + jobName + qsubAdditional + ' -j y -b y -o ' + qsubOut + ' -cwd ' + shfile
qsubCommand = 'bsub -J ' + jobName + qsubAdditional + ' -o ' + qsubOut + ' ' + shfile
os.system("chmod 755 " + shfile)
#qsubCommand = shfile
functions.addQsubToQueue(qsubCommand)
functions.runAllQsubsToCompletion()
else:
print '\r\n' + '------ ' + 'Processing ' + lineName + '------ '
pathComponents = dirpath.split(os.sep)
dirLength = len(pathComponents)
# we find where the "Extracted-files" folder is and then assume the tracker is on the next subdir structure. Not super robust..
tracker = functions.getTrackerName(pathComponents)
lineHandle = functions.sanitizeForFileName(
lineName)
shfile = os.path.dirname(
fullProjectDirPath
) + os.sep + settings.batchScriptDirProjectsSuffix + 'toMatFiles/' + lineHandle + '.sh'
mcrCacheRoot = settings.mcrCacheRootBase + lineHandle
expParameters = lineName
functions.createQsubFile(
shfile, settings.mcrLocation,
settings.matlabScriptFileName[tracker],
dirpath, expParameters)
if (os.path.exists(dirpath + os.sep + filename)):
sumBinFileSizes += os.path.getsize(dirpath +
os.sep +
filename)
numBatchSlots = max(
2, min(16, int(sumBinFileSizes * 1e-9)))
controlsPat = re.compile('FCF_|ywr')
if tracker == 't8':
#qsubAdditional = ' -pe batch 16'
qsubAdditional = ' -n 16'
else:
#qsubAdditional = ' -pe batch ' + str(numBatchSlots)
qsubAdditional = ' -n ' + str(numBatchSlots)
#qsubAdditional = qsubAdditional + ' -l h_rt=2:00:00' + settings.supplementaryQsubParams[tracker]