def calc_jobs_number(): cores = 1 try: if ostype == "Darwin": txt = gop('sysctl -n hw.physicalcpu') else: txt = gop('grep "processor\W:" /proc/cpuinfo | wc -l') cores = int(txt) except: pass return str(cores * 2)
def calc_jobs_number(): cores = 1 try: if isinstance(self, OSMac): txt = gop('sysctl -n hw.physicalcpu') else: txt = gop('grep "processor\W:" /proc/cpuinfo | wc -l') cores = int(txt) except: pass return str(cores * 2)
def job_sow(): workingdir = os.getcwd() hi = 0 paramvals = set_params() while len(paramvals) > 0: ##look if host has max jobs from me oput = gop( 'ssh -o NumberOfPasswordPrompts=0 -f ' + hosts[hi] + ' "hostname; top -b -n1 | grep matnjm | grep -c python"').split() if len(oput) == 2 and int(oput[1]) < maxjobs: p1, p2 = paramvals.pop(0) #print p1,p2 host = oput[0] runfilename = 'ParamRuns/runfile_%.3f_%.3f' % (p1, p2) gen_script(runfilename, par1, p1, par2, p2, host) time.sleep(0.1) print p1, p2, host os.system('ssh -o NumberOfPasswordPrompts=0 -f ' + host + ' "cd ' + workingdir + '; nice -n 10 python ' + runfilename + '" &') #print 'running' hi = (hi + 1) % len(hosts) ##cycle through hosts if len(new_jobs) > 0: os.chdir('..') os.system('python ' + new_jobs + ' &')
def job_sow(): workingdir=os.getcwd() hi=0 paramvals=set_params() while len(paramvals)>0: ##look if host has max jobs from me oput=gop('ssh -o NumberOfPasswordPrompts=0 -f '+hosts[hi]+' "hostname; top -b -n1 | grep matnjm | grep -c python"').split() if len(oput)==2 and int(oput[1])<maxjobs: p1,p2=paramvals.pop(0) #print p1,p2 host=oput[0] runfilename='ParamRuns/runfile_%.3f_%.3f'%(p1,p2) gen_script(runfilename,par1,p1,par2,p2,host) time.sleep(0.1) print p1, p2, host os.system('ssh -o NumberOfPasswordPrompts=0 -f '+host+' "cd '+workingdir+'; nice -n 10 python '+runfilename+'" &') #print 'running' hi=(hi+1)%len(hosts) ##cycle through hosts if len(new_jobs)>0: os.chdir('..') os.system('python '+new_jobs+' &')
os.system('ssh '+host[h]+' top -b -n1 | grep matnjm | grep -c python > '+host[h]+'.stat &') #os.system('echo "'+str(time.time())+'" > time'+str(exnum)+'_start.txt') os.mkdir('FinishedResults'+str(exnum)) for beta in sp.arange(betamax,betamin,-betastep): gmax=1.-beta for gamma in sp.arange(gammamin,gmax,gammastep): success=0 while success==0: if h1==(H-1): h1=0 else: h1=h for h in range(h1,H): try: #os.system('ssh '+host[h]+' top -b -n1 | grep matnjm | grep -c python > '+host[h]+'.stat &') #time.sleep(1.0) hoststate[h]=int(gop("cat "+host[h]+".stat")) print "["+str(h+1)+"] "+host[h]+" \t:", hoststate[h] time.sleep(0.1) if hoststate[h] >= 1: print "skipping "+host[h] time.sleep(0.1) else: #while os.path.isfile(host[h]+".lock"): # continue #open(host[h]+".lock", 'w').close() #os.system('ssh '+host[h]+' top -b -n1 | grep matnjm | grep -c python > '+host[h]+'.stat &') #time.sleep(1.0) hoststate[h]=int(gop("cat "+host[h]+".stat")) os.system("echo %d > %s.stat"%(hoststate[h]+1,host[h])) #os.remove(host[h]+".lock")
host.append("linuxbox1") host.append("linuxbox2") host.append("linuxbox3") host.append("linuxbox4") H = len(host) if len(sys.argv) == 1: from commands import getoutput as gop def frac(text): return round(float(text[2]) / float(text[1]), 2) for h in range(H): print "[" + str(h + 1) + "] " + host[h] + " \t:", load_txt = gop("ssh " + host[h] + " \"uptime\" ").split()[-1] if load_txt == "host": # in "No route to host" print "\tno connection" continue load = float(load_txt) memo = gop("ssh " + host[h] + " \"free -m | grep Mem \" ").split() swap = gop("ssh " + host[h] + " \"free -m | grep Swap\" ").split() core = int( gop("ssh " + host[h] + " \"cat /proc/cpuinfo|grep processor|wc -l\" ")) loco = round(load / core, 2) print "\tmemo", frac(memo), "\tswap", frac(swap), "\tload/core", loco, if loco >= 0.80: print "!!!\n"
import sys from terran import * Timeout = int(sys.argv[2]) KeyUnit = sys.argv[1] #Timeout = 330 simulator = terran(Timeout) from commands import getoutput as gop output = gop('cat ./terranbuilds/%s%s.dat | grep -m 1 "|"' % (KeyUnit, Timeout)).split('\n')[0].split('|') #if output[0] == '': #output = gop('cat Marauder%s.dat | grep -m 1 "-"' % Timeout).split('-') Q = [] i = 0 for item in output: i += 1 Q.append((i, Item(globals()[item], simulator))) P = pipeline(Q) simulator.evaluate(P)
'.stat &') #os.system('echo "'+str(time.time())+'" > time'+str(exnum)+'_start.txt') os.mkdir('FinishedResults' + str(exnum)) for beta in sp.arange(betamax, betamin, -betastep): gmax = 1. - beta for gamma in sp.arange(gammamin, gmax, gammastep): success = 0 while success == 0: if h1 == (H - 1): h1 = 0 else: h1 = h for h in range(h1, H): try: #os.system('ssh '+host[h]+' top -b -n1 | grep matnjm | grep -c python > '+host[h]+'.stat &') #time.sleep(1.0) hoststate[h] = int(gop("cat " + host[h] + ".stat")) print "[" + str( h + 1) + "] " + host[h] + " \t:", hoststate[h] time.sleep(0.1) if hoststate[h] >= 1: print "skipping " + host[h] time.sleep(0.1) else: #while os.path.isfile(host[h]+".lock"): # continue #open(host[h]+".lock", 'w').close() #os.system('ssh '+host[h]+' top -b -n1 | grep matnjm | grep -c python > '+host[h]+'.stat &') #time.sleep(1.0) hoststate[h] = int(gop("cat " + host[h] + ".stat")) os.system("echo %d > %s.stat" % (hoststate[h] + 1, host[h]))
host.append("linuxbox2") host.append("linuxbox3") host.append("linuxbox4") H = len(host) if len(sys.argv) == 1 : from commands import getoutput as gop def frac(text): return round( float(text[2])/float(text[1]), 2 ) for h in range(H): print "["+str(h+1)+"] "+host[h]+" \t:", load_txt = gop("ssh "+host[h]+" \"uptime\" ").split()[-1] if load_txt=="host" : # in "No route to host" print "\tno connection" continue load = float( load_txt ) memo = gop("ssh "+host[h]+" \"free -m | grep Mem \" ").split() swap = gop("ssh "+host[h]+" \"free -m | grep Swap\" ").split() core = int( gop("ssh "+host[h]+" \"cat /proc/cpuinfo|grep processor|wc -l\" ") ) loco = round( load/core, 2 ) print "\tmemo", frac(memo), "\tswap", frac(swap), "\tload/core", loco, if loco>=0.80: print "!!!\n" os.system("ssh "+host[h]+" \"ps aux | head -n 1\" ") os.system("ssh "+host[h]+" \"ps aux | sort -rnk +3 | head -n 5\" ")