startedOK = True
break
if os.path.exists(tmpDir + "/mpiOK"):
if int(os.popen('lamnodes | wc').readline().split()[0]) > MIN_LAM_NODES:
## debug
os.system('echo "' + str(int(os.popen('lamnodes | wc').readline().split()[0])) + '" > ' + tmpDir + '/MIN_LAM_NODES_CHECK')
startedOK = True
break
try:
lamkill = os.system('lamhalt; lamwipe')
except:
None
try:
counterApplications.add_to_MPIErrorLog(application, tmpDir,
socket.gethostname())
except:
None
# if not os.path.exists('/http/mpi.log/' + application + 'ErrorLog'):
# os.system('touch /http/mpi.log/' + application + 'ErrorLog')
# outlog = open('/http/mpi.log/' + application + 'ErrorLog', mode = 'a')
# outlog.write('MPI fails on ' + time.ctime(time.time()) +
# ' Directory: ' + tmpDir + '\n')
# outlog.close()
if not startedOK:
## Logging
try:
## launch actual R or multtest process
if(test_type == "Cox"):
issue_echo(' about to launch CoxCommand', tmpDir)
CoxCommand(tmpDir, R_pomelo_bin)
else:
issue_echo(' about to launch multestCommand', tmpDir)
multestCommand(tmpDir, num_permut, test_type)
time.sleep(TIME_BETWEEN_CHECKS + random.uniform(0.1, 3))
collectZombies()
if os.path.exists(tmpDir + "/MemoryERROR"):
issue_echo(' MEMORY ERROR ', tmpDir)
counterApplications.add_to_MPIErrorLog('PomeloII-' + test_type,
tmpDir, socket.gethostname(),
message = 'MPI MEMORY ERROR')
mpi_crash_log(tmpDir, "MPI MEMORY ERROR")
cleanups(tmpDir, newDir)
writeErrorMessage(tmpDir)
break
if os.path.exists(tmpDir + "/RterminatedOK"):
issue_echo(' startedOK : RterminatedOK exists', tmpDir)
startedOK = True
break
if os.path.exists(tmpDir + "/mpiOK"):
issue_echo(' startedOK : mpiOK exists', tmpDir)
startedOK = True
break
