def __init__(self, runDirs=None, multiParam=None, paramList=None, origSpec=None):
tmpParam = []
if paramList != None:
self.grid = np.array(paramList)
del paramList
else:
for dir, param in zip(runDirs, multiParam.paramGrid.flatten()):
# print "Reading Directory %s"%dir
tmpParam.append(model(param, dir))
self.grid = np.array(tmpParam)
del tmpParam
# self.grid.reshape(multiParam.paramGrid.shape)
if origSpec != None:
self.origSpec = origSpec
else:
self.origSpec = config.getOrigSpec(preProcess=True)
self._initSpecs()
self.specMask = self.specFlag == 0
self._initSpecs()
def cloudLaunch(params,gateways,origSpec=None,baseDir=None):
if origSpec==None: origSpec=config.getOrigSpec(preProcess=True)
if baseDir==None: baseDir = os.getcwd()
noModels=len(params)
models=modelList()
proxyModels=weakref.proxy(models)
##CONSTANTS
params=list(params)
machines=zip(*gateways)[0]
machineSlots=dict([[machine,0] for machine in machines])
startTime=time.time()
#Configuring which machines to use
def callbackFica(confTuple):
#execution of fica after the program has run
protocol, i = confTuple[:2]
gwConfig = gateways[i]
machineName = gwConfig[0]
#Reading model and appending
if protocol == 'centralRead':
ficaWorkDir=confTuple[2]
model=getModel(ficaWorkDir,origSpec)
model.machine = machineName
proxyModels.append(model)
elif protocol == 'localRead':
model = pickle.loads(confTuple[2])
model.machine = machineName
proxyModels.append(model)
threads = gwConfig[2]
gw=gwConfig[1]
machineSlots[machineName] -= 1
try: param=params.pop(0)
except IndexError: return
machineSlots[machineName] += 1
ch=launch(param, machineConfig[machine], threads,baseDir, origSpec, gw, i, genWorker)
ch.setcallback(callbackFica)
channels=[]
for i,gwConfig in enumerate(gateways):
try: param=params.pop(0)
except IndexError: break
machineName=gwConfig[0]
gw=gwConfig[1]
threads=gwConfig[2]
machineSlots[machineName]+=1
ch=launch(param,machineConfig[machine],threads,baseDir,origSpec,gw,i,genWorker)
ch.setcallback(callbackFica)
########
#Printing the current progress (works only on VT-100 compliant terminals)
while True:
for machineName in machineSlots:
print "%s: %02d"%(machineName,machineSlots[machineName])
lines=len(machineSlots)+1
if len(models)==noModels:break
if params!=[]:
print "Next process to launch is %s/%s"%\
(noModels-len(params)+1,noModels)
sys.stdout.write("\x1b[%sA"%lines)
else:
print "Launched all parameter sets, Still %s items in queue"%\
(noModels-len(models))
sys.stdout.write("\x1b[%sA"%lines)
time.sleep(checkUpTimeStep)
wrefModels=weakref.ref(models)
modelGrid=model.modelGrid(paramList=models,origSpec=origSpec)
del models
timedelta = time.time()-startTime
print "Fica run took %.4f min and %.4f s/model" % (timedelta/60, timedelta/noModels)
return modelGrid
def evolve2(generations=200,populationSize=150,savePath='.',continueGA=False,mutationRate=0.2,mutationScale=0.05):
#Initializing random seed
random.seed(seed)
evolveHeader='#gen specno fitness %s\n'%' '.join(['lum','vph']+selElements)
evolveHeaderFMT='%d %d %s '+' '.join(['%s']*len(['lum','vph']+selElements))
evolveDBPath=os.path.join(savePath,'evolution.dat')
file(evolveDBPath,'w').write(evolveHeader)
generationGapNo=int(populationSize*generationGapFraction)
subPopulationNo=int(populationSize*generationGapFraction*subPopulationFraction)
gws=elauncher.gateways()
basePath=config.getAutoDir()
#Smoothing UV
origSpec=config.getOrigSpec(preProcess=True)
try:
os.remove('break_after_loop')
except:
pass
try:
os.remove('debug_after_loop')
except:
pass
#continue GA if stuff there if not make new one
if continueGA:
x=[]
y=[]
y2=[]
yerr=[]
for fname in np.sort(glob(os.path.join(savePath,'generation*.pkl'))):
x.append(int(re.search('\d+',os.path.basename(fname)).group()))
fitness=np.loadtxt(fname.replace('.pkl','.dat'))
y.append(np.mean(fitness))
y2.append(np.max(fitness))
yerr.append(np.std(fitness))
plotSet.genPlotGenVSFitness(x,y,y2,yerr,outName='gen_vs_fitness_log.png')
plotSet.genPlotGenVSFitness(x,y,y2,yerr,logPlot=False,outName='gen_vs_fitness_linear.png')
curGenerationModel=cPickle.load(file(os.path.join(savePath,'generation%04d.pkl'%np.max(x))))
firstGeneration=np.max(x)+1
else:
x=[]#generation Number
y=[]#fitness medium
y2=[]#fitness max
yerr=[]#fitness std
curGenerationSet=createRandomParamSet(populationSize)
firstGeneration=0
gws.checkAvailability()
print gws.availability
curGenerationModel,tmp=elauncher.cloudLaunch(curGenerationSet.paramGrid,gws.getAvailGateWays(),origSpec=origSpec)
keepChildren=param.multiParam()
keepChildren.grid=np.array([])
for i in range(firstGeneration,generations):
curTime=time.time()
#Skipping for continueGA
if not continueGA:
curGenerationModel.grid=np.concatenate((keepChildren.grid,curGenerationModel.grid))
curGenerationModel._initSpecs()
if i%5 == 0:
fname="generation%04d.pkl"%i
print "Saving current generation to %s"%fname
cPickle.dump(curGenerationModel,file(os.path.join(savePath,fname),'w'))
print "Saved"
else:
continueGA=False
fitness=curGenerationModel['fitness']
fitnessIDX=np.argsort(fitness)[::-1]
#TEST CASE FOR VARYING OPEN PARAMETERS ________ TEST _____ TEST
if i == 1:
openGAParameters=['lum','vph']
#TEST CASE FOR VARYING OPEN PARAMETERS ________ TEST _____ TEST
#Saving the fitness, making plots
np.savetxt(os.path.join(savePath,'generation%04d.dat'%i),fitness)
x.append(i)
y.append(np.mean(fitness))
y2.append(np.max(fitness))
yerr.append(np.std(fitness))
#Saving to evolution DB
evolveDB=zip([i]*populationSize,range(populationSize),
*[curGenerationModel[item] for item in ['fitness','lum','vph']+selElements])
np.savetxt(file(evolveDBPath,'a'),evolveDB,fmt=evolveHeaderFMT)
#Plotting the generation
plotSet.genPlotSingleGeneration(curGenerationModel,
fitness,
generation=i,
no=10,
pdfName=os.path.join(savePath,'generation%04d.pdf'%i))
plotSet.genPlotStatus(curGenerationModel,
pdfName=os.path.join(savePath,'gen_status%04d.pdf'%i))
plotSet.genPlotHistogram(curGenerationModel,
fitness=fitness,
pdfName=os.path.join(savePath,'gen_histogram%04d.pdf'%i))
plotSet.genPlotGenVSFitness(x,y,y2,yerr,
outName=os.path.join(savePath,'gen_vs_fitness_log.png'))
plotSet.genPlotGenVSFitness(x,y,y2,yerr,
logPlot=False,
outName=os.path.join(savePath,'gen_vs_fitness_linear.png'))
#Checking for break conditions
if os.path.exists('break_after_loop') or os.path.exists(os.path.join(savePath,'break_after_loop')): break
if os.path.exists('debug_after_loop'):
try:
os.remove('debug_after_loop')
except:
pass
pdb.set_trace()
#made plots
#start selective breeding
#First the children that are kept for the subpopulation are put into a seperate variable
if mode=='subpopulation':
if populationSize==subPopulationNo:
keepChildren=param.multiParam()
keepChildren.grid=np.array([])
else:
keepChildren=read.modelGrid(paramList=curGenerationModel.
grid[fitnessIDX[:(populationSize-subPopulationNo)]])
keepChildren._initSpecs()
elif mode=='elitism':
keepChildren=read.modelGrid(paramList=curGenerationModel.
grid[fitnessIDX[:int(populationSize*elitism)]])
keepChildren._initSpecs()
#Now we get the population that is used for breeding and submit it to the breed function
breedPopulation=read.modelGrid(paramList=curGenerationModel.
grid[fitnessIDX[:generationGapNo]])
if mode=='subpopulation':
curGenerationSet=breed(breedPopulation,
popNum=subPopulationNo)
elif mode=='elitism':
curGenerationSet=breed(breedPopulation,
popNum=int(populationSize*(1-elitism)))
del curGenerationModel
#Time is kept
ficaTime=time.time()
#Network check for available nodes
gws.checkAvailability()
print gws.availability
#Calculating the new generation with elauncher.
curGenerationModel,tmp=elauncher.cloudLaunch(curGenerationSet.paramGrid,gws.getAvailGateWays(),origSpec=origSpec)
#Printing time statements
print "Took %s for the fica runs"%(time.time()-ficaTime)
print "Took %s seconds for last loop"%(time.time()-curTime)
