startedOK = True break if os.path.exists(tmpDir + "/mpiOK"): if int(os.popen('lamnodes | wc').readline().split()[0]) > MIN_LAM_NODES: ## debug os.system('echo "' + str(int(os.popen('lamnodes | wc').readline().split()[0])) + '" > ' + tmpDir + '/MIN_LAM_NODES_CHECK') startedOK = True break try: lamkill = os.system('lamhalt; lamwipe') except: None try: counterApplications.add_to_MPIErrorLog(application, tmpDir, socket.gethostname()) except: None # if not os.path.exists('/http/mpi.log/' + application + 'ErrorLog'): # os.system('touch /http/mpi.log/' + application + 'ErrorLog') # outlog = open('/http/mpi.log/' + application + 'ErrorLog', mode = 'a') # outlog.write('MPI fails on ' + time.ctime(time.time()) + # ' Directory: ' + tmpDir + '\n') # outlog.close() if not startedOK: ## Logging try:
## launch actual R or multtest process if(test_type == "Cox"): issue_echo(' about to launch CoxCommand', tmpDir) CoxCommand(tmpDir, R_pomelo_bin) else: issue_echo(' about to launch multestCommand', tmpDir) multestCommand(tmpDir, num_permut, test_type) time.sleep(TIME_BETWEEN_CHECKS + random.uniform(0.1, 3)) collectZombies() if os.path.exists(tmpDir + "/MemoryERROR"): issue_echo(' MEMORY ERROR ', tmpDir) counterApplications.add_to_MPIErrorLog('PomeloII-' + test_type, tmpDir, socket.gethostname(), message = 'MPI MEMORY ERROR') mpi_crash_log(tmpDir, "MPI MEMORY ERROR") cleanups(tmpDir, newDir) writeErrorMessage(tmpDir) break if os.path.exists(tmpDir + "/RterminatedOK"): issue_echo(' startedOK : RterminatedOK exists', tmpDir) startedOK = True break if os.path.exists(tmpDir + "/mpiOK"): issue_echo(' startedOK : mpiOK exists', tmpDir) startedOK = True break