def submitJobs(self, jobsPerNode=100, mock=False):
import os
import time
if True:
print("writing file for", self.jobName)
# write it to a file
fn = settings.pj(self.commandsDir, self.jobName)
with open(fn, "w") as fh:
for bashLine in self.bashLines:
fh.write(bashLine + "\n")
else:
# just execute
for bashLine in self.bashLines:
os.system(bashLine)
# print(bashLine)
return
self.createJobDirectory(mock)
self.writeJobPickle(mock, jobsPerNode)
arrayRunnerFileName = self.jobName + ".arrayRunner.sh"
if not mock:
file = open(arrayRunnerFileName, 'w')
file.write(pythonInterpreterAbsolutePath + "searchWrapper.py -d " +
self.directoryName)
file.close()
jobRange = " 1-" + str(self.lastJobIndex) + " "
#stdOutDir = "/u/project/mweinste/mweinste/dsNickFury3/schedulerOutput "
stdOutDir = "/dev/null "
#stdErrOut = stdOutDir
stdErrOut = os.getcwd(
) + os.sep + self.directoryName + "/schedulerOutput/ "
command = "qsub -cwd -V -N " + self.jobName + " -l h_data=8G,time=23:59:00 -M [email protected] -m a -e " + stdErrOut + "-o " + stdOutDir + "-t " + jobRange + arrayRunnerFileName
#command = "qsub -cwd -V -N " + self.jobName + " -l h_data=2G,time=23:59:00 -M [email protected] -m a -e " + os.getcwd() + os.sep + self.directoryName + "/schedulerOutput/ -o " + os.getcwd() + os.sep + self.directoryName + "/schedulerOutput/ " + "-t " + jobRange + arrayRunnerFileName
#command = "qsub -cwd -V -N " + self.jobName + " -l h_data=2G,time=23:59:00 -e " + os.getcwd() + os.sep + self.directoryName + "/schedulerOutput/ -o " + os.getcwd() + os.sep + self.directoryName + "/schedulerOutput/ " + "-t " + jobRange + arrayRunnerFileName
if not mock:
success = False
failed = 0
while not success and failed <= 20:
print("Submitting: " + command)
result = os.system(command)
success = result == 0
if not success:
time.sleep(30)
failed += 1
else:
print("MOCK SUBMIT: " + command)
for line in self.bashLines:
print(line)
def createResultsDirectory(self, mock):
# MSR-only
import os
if not mock:
self.resultsDir = settings.results_dir
self.commandsDir = settings.pj(settings.commands_dir,
args.chromosome)
try:
os.mkdir(self.resultsDir)
except FileExistsError as e:
pass
self.geneResultsDir = os.path.join(self.resultsDir,
args.chromosome, self.jobName)
try:
os.makedirs(self.geneResultsDir)
except FileExistsError as e:
pass
try:
os.makedirs(self.commandsDir)
except FileExistsError as e:
pass
def from_temp(name):
filename = settings.pj(settings.tmpdir, name + ".pkl")
with open(filename, "rb") as f:
return pickle.load(f)
def to_temp(name, object):
filename = settings.pj(settings.tmpdir, name + ".pkl")
with open(filename, "wb") as f:
pickle.dump(object, f)