def readTriModel(params,runDirs,doPickle=True):
lumModelGrid=read.modelGrid(runDirs[0],params[0])
vphModelGrid=read.modelGrid(runDirs[1],params[1])
IGEModelGrid=read.modelGrid(runDirs[2],params[2])
if doPickle:
pickle.dump(lumModelGrid,file('lmg.pkl','w'))
pickle.dump(vphModelGrid,file('vmg.pkl','w'))
pickle.dump(IGEModelGrid,file('img.pkl','w'))
return lumModelGrid,vphModelGrid,IGEModelGrid
def initLum(doPickle=True):
runDirs,params=launcherSteps.initLaunchLum()
lumModelGrid=read.modelGrid(runDirs,params)
basePath=os.path.split(runDirs[0])[0]
if doPickle:
pickle.dump(lumModelGrid,file(os.path.join(basePath,'lum0.pkl'),'w'))
return lumModelGrid,dalekSteps.getNextLumInterval(lumModelGrid)
def getInitLumParams(runDirs,params,doPickle=True):
##### FOR DEBUG PURPOSES ONLY SHOULD NOT BE USE LATER ON
#params=pickle.load(file('params.pkl'))
#runDirs=pickle.load(file('rundirs.pkl'))
#runDirs=[item.replace(remoteBasePath,localBasePath) for item in runDirs]
##### FOR DEBUG PURPOSES ONLY SHOULD NOT BE USE LATER ON
modelGrid=read.modelGrid(runDirs,params)
if doPickle:
pickle.dump(modelGrid,file('mg.pkl','w'))
def runLumCycle(doPickle=True,maxIter=None,meritThresh=0.01,samples=10,fitHist=None):
if fitHist==None: fitHist=param.fitHistory()
lumModelGrid,interval=initLum()
#fitHist.addHistItem([interval],[lumModelGrid],[None])
#print interval
i=1
while True:
runDirs,params=launcherSteps.launchLum(np.linspace(interval['interval'][0],interval['interval'][1],samples))
lumModelGrid=read.modelGrid(runDirs,params)
basePath=os.path.split(runDirs[0])[0]
if doPickle:
pickle.dump(lumModelGrid,file(os.path.join(basePath,'lum%d.pkl'%i),'w'))
i+=1
interval=dalekSteps.getNextLumInterval(lumModelGrid)
#fitHist.addHistItem([interval],[lumModelGrid],[None])
print "Current interval:",interval
if maxIter!=None:
if i>maxIter: break
#fitHist.write2pickle('lumCycle.pkl')
if interval['merit']<meritThresh: break
return interval
def evolve2(generations=200,populationSize=150,savePath='.',continueGA=False,mutationRate=0.2,mutationScale=0.05):
#Initializing random seed
random.seed(seed)
evolveHeader='#gen specno fitness %s\n'%' '.join(['lum','vph']+selElements)
evolveHeaderFMT='%d %d %s '+' '.join(['%s']*len(['lum','vph']+selElements))
evolveDBPath=os.path.join(savePath,'evolution.dat')
file(evolveDBPath,'w').write(evolveHeader)
generationGapNo=int(populationSize*generationGapFraction)
subPopulationNo=int(populationSize*generationGapFraction*subPopulationFraction)
gws=elauncher.gateways()
basePath=config.getAutoDir()
#Smoothing UV
origSpec=config.getOrigSpec(preProcess=True)
try:
os.remove('break_after_loop')
except:
pass
try:
os.remove('debug_after_loop')
except:
pass
#continue GA if stuff there if not make new one
if continueGA:
x=[]
y=[]
y2=[]
yerr=[]
for fname in np.sort(glob(os.path.join(savePath,'generation*.pkl'))):
x.append(int(re.search('\d+',os.path.basename(fname)).group()))
fitness=np.loadtxt(fname.replace('.pkl','.dat'))
y.append(np.mean(fitness))
y2.append(np.max(fitness))
yerr.append(np.std(fitness))
plotSet.genPlotGenVSFitness(x,y,y2,yerr,outName='gen_vs_fitness_log.png')
plotSet.genPlotGenVSFitness(x,y,y2,yerr,logPlot=False,outName='gen_vs_fitness_linear.png')
curGenerationModel=cPickle.load(file(os.path.join(savePath,'generation%04d.pkl'%np.max(x))))
firstGeneration=np.max(x)+1
else:
x=[]#generation Number
y=[]#fitness medium
y2=[]#fitness max
yerr=[]#fitness std
curGenerationSet=createRandomParamSet(populationSize)
firstGeneration=0
gws.checkAvailability()
print gws.availability
curGenerationModel,tmp=elauncher.cloudLaunch(curGenerationSet.paramGrid,gws.getAvailGateWays(),origSpec=origSpec)
keepChildren=param.multiParam()
keepChildren.grid=np.array([])
for i in range(firstGeneration,generations):
curTime=time.time()
#Skipping for continueGA
if not continueGA:
curGenerationModel.grid=np.concatenate((keepChildren.grid,curGenerationModel.grid))
curGenerationModel._initSpecs()
if i%5 == 0:
fname="generation%04d.pkl"%i
print "Saving current generation to %s"%fname
cPickle.dump(curGenerationModel,file(os.path.join(savePath,fname),'w'))
print "Saved"
else:
continueGA=False
fitness=curGenerationModel['fitness']
fitnessIDX=np.argsort(fitness)[::-1]
#TEST CASE FOR VARYING OPEN PARAMETERS ________ TEST _____ TEST
if i == 1:
openGAParameters=['lum','vph']
#TEST CASE FOR VARYING OPEN PARAMETERS ________ TEST _____ TEST
#Saving the fitness, making plots
np.savetxt(os.path.join(savePath,'generation%04d.dat'%i),fitness)
x.append(i)
y.append(np.mean(fitness))
y2.append(np.max(fitness))
yerr.append(np.std(fitness))
#Saving to evolution DB
evolveDB=zip([i]*populationSize,range(populationSize),
*[curGenerationModel[item] for item in ['fitness','lum','vph']+selElements])
np.savetxt(file(evolveDBPath,'a'),evolveDB,fmt=evolveHeaderFMT)
#Plotting the generation
plotSet.genPlotSingleGeneration(curGenerationModel,
fitness,
generation=i,
no=10,
pdfName=os.path.join(savePath,'generation%04d.pdf'%i))
plotSet.genPlotStatus(curGenerationModel,
pdfName=os.path.join(savePath,'gen_status%04d.pdf'%i))
plotSet.genPlotHistogram(curGenerationModel,
fitness=fitness,
pdfName=os.path.join(savePath,'gen_histogram%04d.pdf'%i))
plotSet.genPlotGenVSFitness(x,y,y2,yerr,
outName=os.path.join(savePath,'gen_vs_fitness_log.png'))
plotSet.genPlotGenVSFitness(x,y,y2,yerr,
logPlot=False,
outName=os.path.join(savePath,'gen_vs_fitness_linear.png'))
#Checking for break conditions
if os.path.exists('break_after_loop') or os.path.exists(os.path.join(savePath,'break_after_loop')): break
if os.path.exists('debug_after_loop'):
try:
os.remove('debug_after_loop')
except:
pass
pdb.set_trace()
#made plots
#start selective breeding
#First the children that are kept for the subpopulation are put into a seperate variable
if mode=='subpopulation':
if populationSize==subPopulationNo:
keepChildren=param.multiParam()
keepChildren.grid=np.array([])
else:
keepChildren=read.modelGrid(paramList=curGenerationModel.
grid[fitnessIDX[:(populationSize-subPopulationNo)]])
keepChildren._initSpecs()
elif mode=='elitism':
keepChildren=read.modelGrid(paramList=curGenerationModel.
grid[fitnessIDX[:int(populationSize*elitism)]])
keepChildren._initSpecs()
#Now we get the population that is used for breeding and submit it to the breed function
breedPopulation=read.modelGrid(paramList=curGenerationModel.
grid[fitnessIDX[:generationGapNo]])
if mode=='subpopulation':
curGenerationSet=breed(breedPopulation,
popNum=subPopulationNo)
elif mode=='elitism':
curGenerationSet=breed(breedPopulation,
popNum=int(populationSize*(1-elitism)))
del curGenerationModel
#Time is kept
ficaTime=time.time()
#Network check for available nodes
gws.checkAvailability()
print gws.availability
#Calculating the new generation with elauncher.
curGenerationModel,tmp=elauncher.cloudLaunch(curGenerationSet.paramGrid,gws.getAvailGateWays(),origSpec=origSpec)
#Printing time statements
print "Took %s for the fica runs"%(time.time()-ficaTime)
print "Took %s seconds for last loop"%(time.time()-curTime)