def runManual(param, gateways=None):
if gateways==None:
gateways = elauncher.gateways()
runPath = os.path.join(os.getcwd(), uuid.uuid4().hex)
param.targetDir = runPath
model = elauncher.cloudLaunch([param,], gateways.getAvailGateWays())
return model[0]
def evolve(conn, SNSpecID, GARunID=None, description=None, generations=20, populationSize=150,
breedFunc = breed, selectFunc=geneticDalekAlgo.selectRoulette,
crossFunc=geneticDalekAlgo.crossSingle,
randomParamFunc = geneticDalekAlgo.createRandomLogNormalValueW7):
#Run the GA
startTime = datetime.datetime.now()
#Initializing cursor
curs = conn.cursor()
#trying to remove existing break and debug switches
try:
os.remove('break_after_loop')
except:
pass
try:
os.remove('debug_after_loop')
except:
pass
#Initializing random seed
random.seed(config.GAConfDict['seed'])
#Initializing mode dependent constants:
generationGapNo=int(populationSize * config.GAConfDict['generationGapFraction'])
subPopulationNo=int(populationSize * config.GAConfDict['generationGapFraction']
* config.GAConfDict['subPopulationFraction'])
#Launching the gateways
gws=elauncher.gateways()
#getting origSpec and preparing it
rawOrigSpec = curs.execute('select SPECTRUM from SN_SPECTRA where ID=%d' % SNSpecID).fetchall()[0]
origSpec = initialize.preProcessOrigSpec(rawOrigSpec[0])
breedSource = dalekDB.makeZipPickle(inspect.getsource(breedFunc))
GAConfSource = dalekDB.makeZipPickle(config.GAConfDict)
crossSource = dalekDB.makeZipPickle(inspect.getsource(crossFunc))
selectSource = dalekDB.makeZipPickle(inspect.getsource(selectFunc))
fitSource = dalekDB.makeZipPickle(inspect.getsource(genFitness.fitFunc))
#Getting SNConfigDict
SNConfigDict = config.getSNConfigDict(conn)
SNConfigDict['t'] = config.getTimeFromExplosion(conn, SNSpecID, SNConfigDict)
#setting time
param.SNConfigDict = SNConfigDict
#Checking for continue or new
if GARunID!=None:
raise NotImplementedError('soon.....')
if GARunID == None: #or GA_CUR_GEN=0
#creating new GA_RUN entry and inserting the code of various functions
curs.execute('insert into GA_RUN(DESCRIPTION, SN_ID, START_TIME,'
'SN_SPECTRUM, GA_POP_SIZE, GA_GENERATION_SIZE, GA_CONF_DICT, GA_BREED_FUNC,'
'GA_CROSS_FUNC, GA_SELECT_FUNC, GA_FITNESS_FUNC)'
' VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
(description, SNSpecID, startTime,
origSpec, populationSize, generations, GAConfSource, breedSource,
crossSource, selectSource, fitSource))
GARunID = curs.lastrowid
curs.execute('insert into GA_GENERATION(GA_RUN_ID) VALUES(?)', (GARunID,))
generationID = curs.lastrowid
curGenerationSet = geneticDalekAlgo.createRandomParamSet(populationSize)
#Inserting the dica into the database, this dica will be used throughout the run
dicaID = dalekDB.insertDica(conn, curGenerationSet.paramGrid[0].dica)
firstGeneration = 0
# Checking availability on gateways
gws.checkAvailability()
pp.pprint(gws.availability)
#Launching
curGenerationModel = elauncher.cloudLaunch(curGenerationSet.paramGrid, gws.getAvailGateWays(), origSpec=origSpec)
#getting fitnesses from the model
fitness = curGenerationModel['fitness']
fitnessIDX = np.argsort(fitness)[::-1]
keepChildren = param.multiParam()
keepChildren.grid = np.array([])
keepModelIDs = np.array([])
keepFitness = np.array([])
for i in range(firstGeneration,generations):
#getting current time
curTime=time.time()
#saving model to db
modelIDs = curGenerationModel.toDB(conn, dicaID=dicaID, storeLList=False, storeWParam=False)
#main link between models and the GA for the keeping
dalekDB.insertGAIndividual(conn, generationID, keepModelIDs, keepFitness)
#main link between models and the GA
dalekDB.insertGAIndividual(conn, generationID, modelIDs, fitness)
curs.execute('update GA_RUN set GA_CUR_GEN=? where ID=?', (generationID, GARunID))
#getting new generation
curs.execute('insert into GA_GENERATION(GA_RUN_ID) VALUES(?)', (GARunID,))
generationID = curs.lastrowid
#uniting old keepChildren and current generation model
curGenerationModel.grid=np.concatenate((keepChildren.grid,curGenerationModel.grid))
curGenerationModel._initSpecs()
modelIDs = np.concatenate((keepModelIDs, modelIDs))
#getting fitnesses from the model
fitness = curGenerationModel['fitness']
fitnessIDX = np.argsort(fitness)[::-1]
#Checking for break conditions
if os.path.exists('break_after_loop'): break
if os.path.exists('debug_after_loop'):
try:
os.remove('debug_after_loop')
except:
pass
pdb.set_trace()
#start selective breeding
#First the children that are kept for the subpopulation are put into a seperate variable
if config.GAConfDict['mode'] == 'subpopulation':
if populationSize==subPopulationNo:
keepChildren=param.multiParam()
keepChildren.grid=np.array([])
else:
keepChildren = model.modelGrid(paramList=curGenerationModel.
grid[fitnessIDX[:(populationSize-subPopulationNo)]],
origSpec=origSpec)
keepModelIDs = modelIDs[fitnessIDX[:(populationSize - subPopulationNo)]]
keepFitness = fitness[fitnessIDX[:(populationSize - subPopulationNo)]]
keepChildren._initSpecs()
elif config.GAConfDict['mode'] == 'elitism':
keepChildren = model.modelGrid(paramList=curGenerationModel.
grid[fitnessIDX[:int(populationSize * config.GAConfDict['elitism'])]],
origSpec=origSpec)
keepModelIDs = modelIDs[fitnessIDX[:(populationSize * config.GAConfDict['elitism'])]]
keepFitness = fitness[fitnessIDX[:(populationSize * config.GAConfDict['elitism'])]]
keepChildren._initSpecs()
#Now we get the population that is used for breeding and submit it to the breed function
breedPopulation=model.modelGrid(paramList=curGenerationModel.
grid[fitnessIDX[:generationGapNo]],
origSpec=origSpec)
if config.GAConfDict['mode'] == 'subpopulation':
curGenerationSet=breed(breedPopulation,
popNum=subPopulationNo, select=selectFunc, cross=crossFunc)
elif config.GAConfDict['mode'] == 'elitism':
curGenerationSet=breed(breedPopulation,
popNum=int(populationSize*(1-config.GAConfDict['elitism'])), select=selectFunc, cross=crossFunc)
del curGenerationModel
#Time is kept
ficaTime=time.time()
#Network check for available nodes
gws.checkAvailability()
pp.pprint(gws.availability)
#Calculating the new generation with elauncher
curGenerationModel = elauncher.cloudLaunch(curGenerationSet.paramGrid, gws.getAvailGateWays(), origSpec=origSpec)
conn.commit()
#Printing time statements
print "Took %.3f for the fica runs" % (time.time()-ficaTime)
print "Took %.3f seconds for last loop" % (time.time()-curTime)
curs.execute('update GA_RUN set END_TIME=? where ID=?', (datetime.datetime.now(), GARunID))
def evolve2(generations=200,populationSize=150,savePath='.',continueGA=False,mutationRate=0.2,mutationScale=0.05):
#Initializing random seed
random.seed(seed)
evolveHeader='#gen specno fitness %s\n'%' '.join(['lum','vph']+selElements)
evolveHeaderFMT='%d %d %s '+' '.join(['%s']*len(['lum','vph']+selElements))
evolveDBPath=os.path.join(savePath,'evolution.dat')
file(evolveDBPath,'w').write(evolveHeader)
generationGapNo=int(populationSize*generationGapFraction)
subPopulationNo=int(populationSize*generationGapFraction*subPopulationFraction)
gws=elauncher.gateways()
basePath=config.getAutoDir()
#Smoothing UV
origSpec=config.getOrigSpec(preProcess=True)
try:
os.remove('break_after_loop')
except:
pass
try:
os.remove('debug_after_loop')
except:
pass
#continue GA if stuff there if not make new one
if continueGA:
x=[]
y=[]
y2=[]
yerr=[]
for fname in np.sort(glob(os.path.join(savePath,'generation*.pkl'))):
x.append(int(re.search('\d+',os.path.basename(fname)).group()))
fitness=np.loadtxt(fname.replace('.pkl','.dat'))
y.append(np.mean(fitness))
y2.append(np.max(fitness))
yerr.append(np.std(fitness))
plotSet.genPlotGenVSFitness(x,y,y2,yerr,outName='gen_vs_fitness_log.png')
plotSet.genPlotGenVSFitness(x,y,y2,yerr,logPlot=False,outName='gen_vs_fitness_linear.png')
curGenerationModel=cPickle.load(file(os.path.join(savePath,'generation%04d.pkl'%np.max(x))))
firstGeneration=np.max(x)+1
else:
x=[]#generation Number
y=[]#fitness medium
y2=[]#fitness max
yerr=[]#fitness std
curGenerationSet=createRandomParamSet(populationSize)
firstGeneration=0
gws.checkAvailability()
print gws.availability
curGenerationModel,tmp=elauncher.cloudLaunch(curGenerationSet.paramGrid,gws.getAvailGateWays(),origSpec=origSpec)
keepChildren=param.multiParam()
keepChildren.grid=np.array([])
for i in range(firstGeneration,generations):
curTime=time.time()
#Skipping for continueGA
if not continueGA:
curGenerationModel.grid=np.concatenate((keepChildren.grid,curGenerationModel.grid))
curGenerationModel._initSpecs()
if i%5 == 0:
fname="generation%04d.pkl"%i
print "Saving current generation to %s"%fname
cPickle.dump(curGenerationModel,file(os.path.join(savePath,fname),'w'))
print "Saved"
else:
continueGA=False
fitness=curGenerationModel['fitness']
fitnessIDX=np.argsort(fitness)[::-1]
#TEST CASE FOR VARYING OPEN PARAMETERS ________ TEST _____ TEST
if i == 1:
openGAParameters=['lum','vph']
#TEST CASE FOR VARYING OPEN PARAMETERS ________ TEST _____ TEST
#Saving the fitness, making plots
np.savetxt(os.path.join(savePath,'generation%04d.dat'%i),fitness)
x.append(i)
y.append(np.mean(fitness))
y2.append(np.max(fitness))
yerr.append(np.std(fitness))
#Saving to evolution DB
evolveDB=zip([i]*populationSize,range(populationSize),
*[curGenerationModel[item] for item in ['fitness','lum','vph']+selElements])
np.savetxt(file(evolveDBPath,'a'),evolveDB,fmt=evolveHeaderFMT)
#Plotting the generation
plotSet.genPlotSingleGeneration(curGenerationModel,
fitness,
generation=i,
no=10,
pdfName=os.path.join(savePath,'generation%04d.pdf'%i))
plotSet.genPlotStatus(curGenerationModel,
pdfName=os.path.join(savePath,'gen_status%04d.pdf'%i))
plotSet.genPlotHistogram(curGenerationModel,
fitness=fitness,
pdfName=os.path.join(savePath,'gen_histogram%04d.pdf'%i))
plotSet.genPlotGenVSFitness(x,y,y2,yerr,
outName=os.path.join(savePath,'gen_vs_fitness_log.png'))
plotSet.genPlotGenVSFitness(x,y,y2,yerr,
logPlot=False,
outName=os.path.join(savePath,'gen_vs_fitness_linear.png'))
#Checking for break conditions
if os.path.exists('break_after_loop') or os.path.exists(os.path.join(savePath,'break_after_loop')): break
if os.path.exists('debug_after_loop'):
try:
os.remove('debug_after_loop')
except:
pass
pdb.set_trace()
#made plots
#start selective breeding
#First the children that are kept for the subpopulation are put into a seperate variable
if mode=='subpopulation':
if populationSize==subPopulationNo:
keepChildren=param.multiParam()
keepChildren.grid=np.array([])
else:
keepChildren=read.modelGrid(paramList=curGenerationModel.
grid[fitnessIDX[:(populationSize-subPopulationNo)]])
keepChildren._initSpecs()
elif mode=='elitism':
keepChildren=read.modelGrid(paramList=curGenerationModel.
grid[fitnessIDX[:int(populationSize*elitism)]])
keepChildren._initSpecs()
#Now we get the population that is used for breeding and submit it to the breed function
breedPopulation=read.modelGrid(paramList=curGenerationModel.
grid[fitnessIDX[:generationGapNo]])
if mode=='subpopulation':
curGenerationSet=breed(breedPopulation,
popNum=subPopulationNo)
elif mode=='elitism':
curGenerationSet=breed(breedPopulation,
popNum=int(populationSize*(1-elitism)))
del curGenerationModel
#Time is kept
ficaTime=time.time()
#Network check for available nodes
gws.checkAvailability()
print gws.availability
#Calculating the new generation with elauncher.
curGenerationModel,tmp=elauncher.cloudLaunch(curGenerationSet.paramGrid,gws.getAvailGateWays(),origSpec=origSpec)
#Printing time statements
print "Took %s for the fica runs"%(time.time()-ficaTime)
print "Took %s seconds for last loop"%(time.time()-curTime)