def externalRun(settingsFile=None):
plugName = sys.argv[1]
name = plugName[:-3]
if not settingsFile:
if len(sys.argv) < 2:
raise "Must supply settings file"
settingsFile = sys.argv[2]
s = settings.runSettings(settingsFile)
solSettings = s.probConf
plug = __import__(plugName)
log = logger.logger(name, 0)
j = 0
labels = []
for l in s.probConf:
k = ""
for key in l['settings']:
k += key + "=" + str(l['settings'][key]) + ", "
labels.append(k)
for d in solSettings:
log = plug.run(int(sys.argv[3]), log, d, name, j)
j += 1
log.log()
log.plot(labels)
import copy
def ktourn(pop,k):
best = None
for i in xrange(k):
obj = random.choice(pop)
if not best or obj>best:
best = obj
return best
s = None
if len(sys.argv)>1:
s = settings.runSettings(sys.argv[1])
else:
s = settings.runSettings()
s.seed =time.time()
random.seed(s.seed)
mu = 100
k = 8
pop = []
i = 0
while i<mu:
x = bbsa.bbsa(copy.deepcopy(s))
if not x.evalExist() or not x.lastExist():
print "\nFailed\n"
continue
print x.toDict()
def runNSGA(settingsFile = None):
s = None
if not settingsFile:
if len(sys.argv)<2:
raise "Must supply settings file"
settingsFile = sys.argv[1]
s = settings.runSettings(settingsFile,sys.argv[2])
if s.hyperSettings['seed']:
s.seed = s.hyperSettings['seed']
else:
s.seed =time.time()
random.seed(s.seed)
mu = s.hyperSettings['mu']
childK = s.hyperSettings['childK']
lamb = s.hyperSettings['lambda']
maxEvals = s.hyperSettings['evaluations']
mutateRate = s.hyperSettings['mutateRate']
mateRate = s.hyperSettings['mateRate']
cur = mu
pop = []
i = 0
while i<mu:
x = bbsa.bbsa(copy.deepcopy(s))
if not x.evalExist() or not x.lastExist():
continue
x.evaluate()
if x.valid():
pop.append(x)
i+=1
else:
pop.sort()
for p in pop:
print p.fitness
fronts = pareto.pareto(pop)
while cur<maxEvals:
c = 0
childs = []
while c<lamb:
choice = random.choice([0,1,2])
rate = random.random()
if c+1!=lamb and rate<mateRate:
mom = fronts.tournSelect(childK)
dad = fronts.tournSelect(childK)
x,y = mom.mate(dad)
if x.evalExist() and x.lastExist():
x.evaluate()
if x.aveEval>0:
childs.append(x)
c+=1
if y.evalExist() and y.lastExist():
y.evaluate()
if y.aveEval>0:
childs.append(y)
c+=1
elif rate<mateRate+mutateRate:
x=fronts.tournSelect(childK).mutate()
if x.evalExist() and x.lastExist():
x.evaluate()
if x.aveEval>0:
childs.append(x)
c+=1
else:
x = fronts.tournSelect(childK).altMutate()
if x.evalExist():
x.evaluate()
if x.aveEval>0:
childs.append(x)
c+=1
pop = fronts.getPop()
pop.extend(childs)
fronts = pareto.pareto(pop)
fronts.keepMu(mu)
cur+=lamb
su = 0.0
ave = 0.0
for i in xrange(mu):
su+=fronts.pop[i].fitness
ave = su/mu
i = 0
print cur, ave, len(fronts.fronts.keys())
fronts.fronts[0].sort()
for ind in fronts.fronts[0]:
print"\t",i,ind.fitness,",",ind.aveEval,",",ind.time,",",ind.distance,"\t",ind.name
i+=1
ind.makeGraph()
ind.plot()
ind.logger.log()
ind.makeProg()
i = 0
fronts.fronts[0].sort()
for ind in fronts.fronts[0]:
print"\t",i,ind.fitness,",",ind.aveEval,",",ind.time,",",ind.distance,"\t",ind.name
i+=1
ind.directory += "finalFront/"
ind.logger.name = ind.name
ind.makeGraph()
ind.plot()
ind.logger.log()
ind.makeProg()
def runHydra(settingsFile = None):
s = None
if not settingsFile:
if len(sys.argv)<2:
raise "Must supply settings file"
settingsFile = sys.argv[1]
s = settings.runSettings(settingsFile)
t = s.hyperSettings['type']
mpi = s.hyperSettings['mpi']
procs = s.hyperSettings['procs']
runName = s.hyperSettings['runName']
runNum = s.hyperSettings['runNumber']
if len(sys.argv)>2:
runNum = int(sys.argv[2])
runDir = 'experiments/'+runName+"/"+str(runNum)+"/"
#Setup file locations
#check if experiment dir exists
lsProc = Popen(['ls','experiments/'],stdout=PIPE,stderr=PIPE)
stdout,stderr = lsProc.communicate()
if lsProc.returncode!=0:
mkdirProc = Popen(['mkdir','experiments/'],stdout=PIPE,stderr=PIPE)
#check runName dir exists
lsProc = Popen(['ls','experiments/'+runName],stdout=PIPE,stderr=PIPE)
stdout,stderr = lsProc.communicate()
if lsProc.returncode!=0:
mkdirProc = Popen(['mkdir','experiments/'+runName],stdout=PIPE,stderr=PIPE)
#check if run dir exists
lsProc = Popen(['ls','experiments/'+runName+"/"+str(runNum)],stdout=PIPE,stderr=PIPE)
stdout,stderr = lsProc.communicate()
if lsProc.returncode!=0:
lsProc = Popen(['mkdir','experiments/'+runName+"/"+str(runNum)],stdout=PIPE,stderr=PIPE)
else:
print "Run Number directory exists. Run may not perform properly"
choice = raw_input("Continue?(y/n): ")
if choice=='y' or choice=='Y' or choice =='yes' or choice =='Yes':
print "Continuing"
elif choice=='n' or choice=='N' or choice=='no' or choice =='No':
print "Quitting"
return
else:
print "Not valid Option"
lsProc = Popen(['ls','experiments/'+runName+"/"+str(runNum)+"/"+settingsFile],stdout=PIPE,stderr=PIPE)
stdout,stderr = lsProc.communicate()
if lsProc.returncode!=0:
cpProc = Popen(['cp',settingsFile,runDir],stdout=PIPE,stderr=PIPE)
cpProc.communicate()
for r in s.probConf:
if 'name' in r['settings']:
cpProc = Popen(['cp',r['settings']['name'],runDir],stdout=PIPE,stderr=PIPE)
cpProc.communicate()
if cpProc.returncode!=0:
print "Cannot find file",r['settings']['name']
return
settingsFile = settingsFile.split("/")[-1]
#hostfile stuff
multiComp = False
hsf = ""
f = ""
if s.hyperSettings['hosts']:
multiComp =True
for host in s.hyperSettings['hosts']:
if 'user' in host:
hsf+=host['user']
else:
hsf+="sitaware"
hsf+="@"+host['name']
if 'slots' in host:
hsf+=" slots="+host['slots']
hsf+="\n"
fl =open(runDir+'host_file','w')
fl.write(hsf)
fl.close()
print hsf
if mpi:
#do MPI
if t=='SGA':
#SGA
print "SGA"
if multiComp:
runProc = Popen(['mpiexec','--hostfile','host_file','-wd','~/EvoBBSA','-n',str(procs),'python','mpi-SGA.py',settingsFile,runDir])
else:
runProc = Popen(['mpiexec','-n',str(procs),'python','mpi-SGA.py',settingsFile,runDir])
stdout,stderr = runProc.communicate()
elif t=='nsga':
#NSGA
print "nsga"
mkProc = Popen(['mkdir',runDir+'/finalFront'],stdout=PIPE,stderr=PIPE)
stdout,stderr = mkProc.communicate()
if multiComp:
runProc = Popen(['mpiexec','--hostfile','host_file','-wd','~/EvoBBSA','-n',str(procs),'python','mpi-nsga.py',settingsFile,runDir])
else:
runProc = Popen(['mpiexec','-n',str(procs),'python','mpi-nsga.py',settingsFile,runDir])
stdout,stderr = runProc.communicate()
else:
raise "Not valid Evolution Type"
else:
#don't do MPI
if t=='SGA':
#SGA
print "SGA"
runProc = Popen(['python','SGA.py',settingsFile,runDir])
stdout,stderr = runProc.communicate()
elif t=='nsga':
#NSGA
print "nsga"
mkProc = Popen(['mkdir',runDir+'finalFront'],stdout=PIPE,stderr=PIPE)
stdout,stderr = mkProc.communicate()
runProc = Popen(['python','nsga.py',settingsFile,runDir])
stdout,stderr = runProc.communicate()
else:
raise "Not valid Evolution Type"
def runHydra(settingsFile=None):
s = None
if not settingsFile:
if len(sys.argv) < 2:
raise "Must supply settings file"
settingsFile = sys.argv[1]
s = settings.runSettings(settingsFile)
t = s.hyperSettings['type']
mpi = s.hyperSettings['mpi']
procs = s.hyperSettings['procs']
runName = s.hyperSettings['runName']
runNum = s.hyperSettings['runNumber']
if len(sys.argv) > 2:
runNum = int(sys.argv[2])
runDir = 'experiments/' + runName + "/" + str(runNum) + "/"
#Setup file locations
#check if experiment dir exists
lsProc = Popen(['ls', 'experiments/'], stdout=PIPE, stderr=PIPE)
stdout, stderr = lsProc.communicate()
if lsProc.returncode != 0:
mkdirProc = Popen(['mkdir', 'experiments/'], stdout=PIPE, stderr=PIPE)
#check runName dir exists
lsProc = Popen(['ls', 'experiments/' + runName], stdout=PIPE, stderr=PIPE)
stdout, stderr = lsProc.communicate()
if lsProc.returncode != 0:
mkdirProc = Popen(['mkdir', 'experiments/' + runName],
stdout=PIPE,
stderr=PIPE)
#check if run dir exists
lsProc = Popen(['ls', 'experiments/' + runName + "/" + str(runNum)],
stdout=PIPE,
stderr=PIPE)
stdout, stderr = lsProc.communicate()
if lsProc.returncode != 0:
lsProc = Popen(['mkdir', 'experiments/' + runName + "/" + str(runNum)],
stdout=PIPE,
stderr=PIPE)
else:
print "Run Number directory exists. Run may not perform properly"
choice = raw_input("Continue?(y/n): ")
if choice == 'y' or choice == 'Y' or choice == 'yes' or choice == 'Yes':
print "Continuing"
elif choice == 'n' or choice == 'N' or choice == 'no' or choice == 'No':
print "Quitting"
return
else:
print "Not valid Option"
lsProc = Popen([
'ls', 'experiments/' + runName + "/" + str(runNum) + "/" + settingsFile
],
stdout=PIPE,
stderr=PIPE)
stdout, stderr = lsProc.communicate()
if lsProc.returncode != 0:
cpProc = Popen(['cp', settingsFile, runDir], stdout=PIPE, stderr=PIPE)
cpProc.communicate()
for r in s.probConf:
if 'name' in r['settings']:
cpProc = Popen(['cp', r['settings']['name'], runDir],
stdout=PIPE,
stderr=PIPE)
cpProc.communicate()
if cpProc.returncode != 0:
print "Cannot find file", r['settings']['name']
return
settingsFile = settingsFile.split("/")[-1]
#hostfile stuff
multiComp = False
hsf = ""
f = ""
if s.hyperSettings['hosts']:
multiComp = True
for host in s.hyperSettings['hosts']:
if 'user' in host:
hsf += host['user']
else:
hsf += "sitaware"
hsf += "@" + host['name']
if 'slots' in host:
hsf += " slots=" + host['slots']
hsf += "\n"
fl = open(runDir + 'host_file', 'w')
fl.write(hsf)
fl.close()
print hsf
if mpi:
#do MPI
if t == 'SGA':
#SGA
print "SGA"
if multiComp:
runProc = Popen([
'mpiexec', '--hostfile', 'host_file', '-wd', '~/EvoBBSA',
'-n',
str(procs), 'python', 'mpi-SGA.py', settingsFile, runDir
])
else:
runProc = Popen([
'mpiexec', '-n',
str(procs), 'python', 'mpi-SGA.py', settingsFile, runDir
])
stdout, stderr = runProc.communicate()
elif t == 'nsga':
#NSGA
print "nsga"
mkProc = Popen(['mkdir', runDir + '/finalFront'],
stdout=PIPE,
stderr=PIPE)
stdout, stderr = mkProc.communicate()
if multiComp:
runProc = Popen([
'mpiexec', '--hostfile', 'host_file', '-wd', '~/EvoBBSA',
'-n',
str(procs), 'python', 'mpi-nsga.py', settingsFile, runDir
])
else:
runProc = Popen([
'mpiexec', '-n',
str(procs), 'python', 'mpi-nsga.py', settingsFile, runDir
])
stdout, stderr = runProc.communicate()
else:
raise "Not valid Evolution Type"
else:
#don't do MPI
if t == 'SGA':
#SGA
print "SGA"
runProc = Popen(['python', 'SGA.py', settingsFile, runDir])
stdout, stderr = runProc.communicate()
elif t == 'nsga':
#NSGA
print "nsga"
mkProc = Popen(['mkdir', runDir + 'finalFront'],
stdout=PIPE,
stderr=PIPE)
stdout, stderr = mkProc.communicate()
runProc = Popen(['python', 'nsga.py', settingsFile, runDir])
stdout, stderr = runProc.communicate()
else:
raise "Not valid Evolution Type"
import parseTree import settings s = settings.runSettings() x = parseTree.parseTree(s) x.createER() print x.toDict() print x.evaluate() #print x.eval2() x.makeProg() x.report() x.makeGraph()
def runSGA(settingsFile = None):
s = None
if not settingsFile:
if len(sys.argv)<2:
raise "Must supply settings file"
settingsFile = sys.argv[1]
s = settings.runSettings(settingsFile,sys.argv[2])
if s.hyperSettings['seed']:
s.seed = s.hyperSettings['seed']
else:
s.seed =time.time()
random.seed(s.seed)
mu = s.hyperSettings['mu']
childK = s.hyperSettings['childK']
lamb = s.hyperSettings['lambda']
maxEvals = s.hyperSettings['evaluations']
mutateRate = s.hyperSettings['mutateRate']
mateRate = s.hyperSettings['mateRate']
cur = mu
pop = []
i = 0
while i<mu:
x = bbsa.bbsa(copy.deepcopy(s))
if not x.evalExist() or not x.lastExist():
continue
x.evaluate()
if x.valid():
pop.append(x)
i+=1
pop.sort()
for p in pop:
print p.fitness
while cur<maxEvals:
c = 0
childs = []
while c<lamb:
choice = random.choice([0,1,2])
rate = random.random()
if c+1!=lamb and rate<mateRate:
mom = ktourn(pop,childK)
dad = ktourn(pop,childK)
x,y = mom.mate(dad)
if x.evalExist() and x.lastExist():
x.evaluate()
childs.append(x)
c+=1
if y.evalExist() and y.lastExist():
y.evaluate()
childs.append(y)
c+=1
elif rate<mateRate+mutateRate:
x=ktourn(pop,childK).mutate()
if x.evalExist() and x.lastExist():
x.evaluate()
childs.append(x)
c+=1
else:
x = ktourn(pop,childK).altMutate()
if x.evalExist():
x.evaluate()
childs.append(x)
c+=1
pop.extend(childs)
pop.sort()
pop.reverse()
pop = pop[:mu]
cur+=lamb
su = 0.0
ave = 0.0
for i in xrange(len(pop)):
su+=pop[i].fitness
ave = su/len(pop)
print cur, ave,pop[0].fitness,pop[0].aveEval,pop[0].aveOps
pop[0].makeGraph()
pop[0].plot()
pop[0].logger.log()
pop[0].makeProg()
f.close()
print
print pop[0].fitness,pop[0].aveEval
print pop[0].toDict()
f = open(str(pop[0].name)+"-Dumb.py","w")
pop[0].makeProg()
f.close()
def runMpiNSGA(settingsFile=None):
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
if rank != 0:
processManager.childProc()
else:
proc = processManager.processManager()
s = None
if not settingsFile:
if len(sys.argv) < 2:
raise "Must supply settings file"
settingsFile = sys.argv[1]
s = settings.runSettings(settingsFile, sys.argv[2])
if s.hyperSettings['seed']:
s.seed = s.hyperSettings['seed']
else:
s.seed = time.time()
random.seed(s.seed)
mu = s.hyperSettings['mu']
childK = s.hyperSettings['childK']
lamb = s.hyperSettings['lambda']
maxEvals = s.hyperSettings['evaluations']
mutateRate = s.hyperSettings['mutateRate']
mateRate = s.hyperSettings['mateRate']
cur = mu
pop = []
i = 0
while i < mu:
x = bbsa.bbsa(copy.deepcopy(s))
if not x.evalExist() or not x.lastExist():
continue
proc.add(x)
i += 1
#print "start"
proc.start()
#print "wait"
proc.wait(mu)
#print "not wait"
pop = proc.getPop()
pop.sort()
for p in pop:
print p.fitness
while cur < maxEvals:
c = 0
childs = []
while c < lamb:
choice = random.choice([0, 1, 2])
rate = random.random()
if c + 1 != lamb and rate < .3:
mom = ktourn(pop, childK)
dad = ktourn(pop, childK)
x, y = mom.mate(dad)
if x.evalExist() and x.lastExist():
proc.add(x)
c += 1
if y.evalExist() and y.lastExist():
proc.add(y)
c += 1
elif rate < .6:
x = ktourn(pop, childK).mutate()
if x.evalExist() and x.lastExist():
proc.add(x)
c += 1
else:
x = ktourn(pop, childK).altMutate()
if x.evalExist():
proc.add(x)
c += 1
#print "start"
proc.start()
#print "wait"
proc.wait(lamb)
#print "no wait"
childs = proc.getPop()
pop.extend(childs)
pop.sort()
pop.reverse()
pop = pop[:mu]
cur += lamb
su = 0.0
ave = 0.0
for i in xrange(len(pop)):
su += pop[i].fitness
ave = su / len(pop)
print cur, ave, pop[0].fitness, pop[0].aveEval, pop[0].aveOps
pop[0].makeGraph()
pop[0].plot()
pop[0].logger.log()
pop[0].makeProg()
print
print pop[0].fitness, pop[0].aveEval
print pop[0].toDict()
pop[0].makeProg()
proc.kill()
import bbsa
import settings
import fitness
#x = fitness.kmeansClassify({'name':'temp.txt','k':2})
#print x.evaluate([1 for i in xrange(256)])
s = settings.runSettings()
x = bbsa.bbsa(s)
x.evaluate()
x.makeGraph()
x.plot()
print x.toDict()
y = bbsa.bbsa(s)
y.load(x.toDict())
print y.toDict()
def extHydra(settingsFile = None):
s = None
plugNameDir = sys.argv[1]
plugName = sys.argv[1].split("/")[-1]
if not settingsFile:
if len(sys.argv)<2:
raise "Must supply settings file"
settingsFile = sys.argv[2]
s = settings.runSettings(settingsFile)
t = s.hyperSettings['type']
mpi = s.hyperSettings['mpi']
procs = s.hyperSettings['procs']
runName = s.hyperSettings['runName']
runNum = s.hyperSettings['runNumber']
if len(sys.argv)>3:
runNum = int(sys.argv[3])
runDir = 'externalRuns/'+runName+"/"+str(runNum)
#Setup file locations
#check if experiment dir exists
lsProc = Popen(['ls','externalRuns/'],stdout=PIPE,stderr=PIPE)
stdout,stderr = lsProc.communicate()
if lsProc.returncode!=0:
mkdirProc = Popen(['mkdir','externalRuns/'],stdout=PIPE,stderr=PIPE)
#check runName dir exists
lsProc = Popen(['ls','externalRuns/'+runName],stdout=PIPE,stderr=PIPE)
stdout,stderr = lsProc.communicate()
if lsProc.returncode!=0:
mkdirProc = Popen(['mkdir','externalRuns/'+runName],stdout=PIPE,stderr=PIPE)
#check if run dir exists
lsProc = Popen(['ls','externalRuns/'+runName+"/"+str(runNum)],stdout=PIPE,stderr=PIPE)
stdout,stderr = lsProc.communicate()
if lsProc.returncode!=0:
lsProc = Popen(['mkdir','externalRuns/'+runName+"/"+str(runNum)],stdout=PIPE,stderr=PIPE)
else:
print "Run Number directory exists. Run may not perform properly"
choice = raw_input("Continue?(y/n): ")
if choice=='y' or choice=='Y' or choice =='yes' or choice =='Yes':
print "Continuing"
elif choice=='n' or choice=='N' or choice=='no' or choice =='No':
print "Quitting"
return
else:
print "Not valid Option"
cpProc = Popen(['cp',settingsFile,runDir],stdout=PIPE,stderr=PIPE)
cpProc.communicate()
cpProc = Popen(['cp',plugNameDir,'.'],stdout=PIPE,stderr=PIPE)
cpProc.communicate()
for r in s.probConf:
if 'name' in r['settings']:
cpProc = Popen(['cp',r['settings']['name'],runDir],stdout=PIPE,stderr=PIPE)
cpProc.communicate()
if cpProc.returncode!=0:
print "Cannot find file",r['settings']['name']
return
settingsFile = settingsFile.split("/")[-1]
loc = plugNameDir.replace('/','.')
print loc
loc = 'externalRuns'+loc[11:]
print loc
if plugName[-2:] == 'py':
plugName = plugName[:-3]
runProc = Popen(['python','../../../run.py',plugName,settingsFile,"30"],cwd=runDir)
runProc.communicate()
cpProc = Popen(['mv',plugNameDir,runDir],stdout=PIPE,stderr=PIPE)
cpProc.communicate()
def runMpiNSGA(settingsFile = None):
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
if rank!=0:
processManager.childProc()
else:
proc = processManager.processManager()
s = None
if not settingsFile:
if len(sys.argv)<2:
raise "Must supply settings file"
settingsFile = sys.argv[1]
s = settings.runSettings(settingsFile,sys.argv[2])
if s.hyperSettings['seed']:
s.seed = s.hyperSettings['seed']
else:
s.seed =time.time()
random.seed(s.seed)
mu = s.hyperSettings['mu']
childK = s.hyperSettings['childK']
lamb = s.hyperSettings['lambda']
maxEvals = s.hyperSettings['evaluations']
mutateRate = s.hyperSettings['mutateRate']
mateRate = s.hyperSettings['mateRate']
cur = mu
pop = []
i = 0
while i<mu:
x = bbsa.bbsa(copy.deepcopy(s))
if not x.evalExist() or not x.lastExist():
continue
proc.add(x)
i+=1
#print "start"
proc.start()
#print "wait"
proc.wait(mu)
#print "not wait"
pop = proc.getPop()
pop.sort()
for p in pop:
print p.fitness
while cur<maxEvals:
c = 0
childs = []
while c<lamb:
choice = random.choice([0,1,2])
rate = random.random()
if c+1!=lamb and rate<.3:
mom = ktourn(pop,childK)
dad = ktourn(pop,childK)
x,y = mom.mate(dad)
if x.evalExist() and x.lastExist():
proc.add(x)
c+=1
if y.evalExist() and y.lastExist():
proc.add(y)
c+=1
elif rate<.6:
x=ktourn(pop,childK).mutate()
if x.evalExist() and x.lastExist():
proc.add(x)
c+=1
else:
x = ktourn(pop,childK).altMutate()
if x.evalExist():
proc.add(x)
c+=1
#print "start"
proc.start()
#print "wait"
proc.wait(lamb)
#print "no wait"
childs = proc.getPop()
pop.extend(childs)
pop.sort()
pop.reverse()
pop = pop[:mu]
cur+=lamb
su = 0.0
ave = 0.0
for i in xrange(len(pop)):
su+=pop[i].fitness
ave = su/len(pop)
print cur, ave,pop[0].fitness,pop[0].aveEval,pop[0].aveOps
pop[0].makeGraph()
pop[0].plot()
pop[0].logger.log()
pop[0].makeProg()
print
print pop[0].fitness,pop[0].aveEval
print pop[0].toDict()
pop[0].makeProg()
proc.kill()