def createSingleTrainingSet(setSize,iter,SNName='2002bo'):
t=config.getTimeFromExplosion()
outnames=[]
for i in range(iter):
curGenerationSet=geneticDalek.createRandomParamSet(setSize)
print "At Generation %s"%i
curGenerationModel=launcherSteps.getModel(curGenerationSet)[0]
outname='%st%.2f.ts%0d.pkl'%(SNName,t,i)
print "Saving current generation to %s"%outname
cPickle.dump(curGenerationModel,file(outname,'w'))
outnames.append(outname)
return outnames
def initLumVphGrid(step=1,gridSize=10,doPickle=True):
t=config.getTimeFromExplosion()
vph=initialize.time2vph(t)
mainConfigDir=config.getMainConfigDir()
modelW7=initialize.readW7Data(os.path.join(mainConfigDir,'w7.combined.dat'))
initDica=config.getVanillaDica()
initDica['t']=t
#Preparing the normalization
initComp=config.getVanillaComp()
initComp.update(initialize.getW7Comp(modelW7,t))
initComp=abund.setNiDecay(initComp,t)
initComp=abund.setCONe(initComp)
initComp=abund.normAbundances(initComp)
initRunDir='init_lumvph_run/'
initialVph=[0.6*vph,1.4*vph]
return lumVphGrid(initialLum,initialVph,1,initRunDir,initComp=initComp,initDica=initDica,gridSize=gridSize,doPickle=doPickle)
def getPrevDica():
dicaFile=os.path.join(config.getLastMainDir(),'manual','dica.dat')
dica=fileio.dicafile(dicaFile).read_data()
print "Time=%s"%config.getTimeFromExplosion()
dica['t']=config.getTimeFromExplosion()
return dica
def getCurVph():
t=config.getTimeFromExplosion()
return time2vph(t)
def evolve(conn, SNSpecID, GARunID=None, description=None, generations=20, populationSize=150,
breedFunc = breed, selectFunc=geneticDalekAlgo.selectRoulette,
crossFunc=geneticDalekAlgo.crossSingle,
randomParamFunc = geneticDalekAlgo.createRandomLogNormalValueW7):
#Run the GA
startTime = datetime.datetime.now()
#Initializing cursor
curs = conn.cursor()
#trying to remove existing break and debug switches
try:
os.remove('break_after_loop')
except:
pass
try:
os.remove('debug_after_loop')
except:
pass
#Initializing random seed
random.seed(config.GAConfDict['seed'])
#Initializing mode dependent constants:
generationGapNo=int(populationSize * config.GAConfDict['generationGapFraction'])
subPopulationNo=int(populationSize * config.GAConfDict['generationGapFraction']
* config.GAConfDict['subPopulationFraction'])
#Launching the gateways
gws=elauncher.gateways()
#getting origSpec and preparing it
rawOrigSpec = curs.execute('select SPECTRUM from SN_SPECTRA where ID=%d' % SNSpecID).fetchall()[0]
origSpec = initialize.preProcessOrigSpec(rawOrigSpec[0])
breedSource = dalekDB.makeZipPickle(inspect.getsource(breedFunc))
GAConfSource = dalekDB.makeZipPickle(config.GAConfDict)
crossSource = dalekDB.makeZipPickle(inspect.getsource(crossFunc))
selectSource = dalekDB.makeZipPickle(inspect.getsource(selectFunc))
fitSource = dalekDB.makeZipPickle(inspect.getsource(genFitness.fitFunc))
#Getting SNConfigDict
SNConfigDict = config.getSNConfigDict(conn)
SNConfigDict['t'] = config.getTimeFromExplosion(conn, SNSpecID, SNConfigDict)
#setting time
param.SNConfigDict = SNConfigDict
#Checking for continue or new
if GARunID!=None:
raise NotImplementedError('soon.....')
if GARunID == None: #or GA_CUR_GEN=0
#creating new GA_RUN entry and inserting the code of various functions
curs.execute('insert into GA_RUN(DESCRIPTION, SN_ID, START_TIME,'
'SN_SPECTRUM, GA_POP_SIZE, GA_GENERATION_SIZE, GA_CONF_DICT, GA_BREED_FUNC,'
'GA_CROSS_FUNC, GA_SELECT_FUNC, GA_FITNESS_FUNC)'
' VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
(description, SNSpecID, startTime,
origSpec, populationSize, generations, GAConfSource, breedSource,
crossSource, selectSource, fitSource))
GARunID = curs.lastrowid
curs.execute('insert into GA_GENERATION(GA_RUN_ID) VALUES(?)', (GARunID,))
generationID = curs.lastrowid
curGenerationSet = geneticDalekAlgo.createRandomParamSet(populationSize)
#Inserting the dica into the database, this dica will be used throughout the run
dicaID = dalekDB.insertDica(conn, curGenerationSet.paramGrid[0].dica)
firstGeneration = 0
# Checking availability on gateways
gws.checkAvailability()
pp.pprint(gws.availability)
#Launching
curGenerationModel = elauncher.cloudLaunch(curGenerationSet.paramGrid, gws.getAvailGateWays(), origSpec=origSpec)
#getting fitnesses from the model
fitness = curGenerationModel['fitness']
fitnessIDX = np.argsort(fitness)[::-1]
keepChildren = param.multiParam()
keepChildren.grid = np.array([])
keepModelIDs = np.array([])
keepFitness = np.array([])
for i in range(firstGeneration,generations):
#getting current time
curTime=time.time()
#saving model to db
modelIDs = curGenerationModel.toDB(conn, dicaID=dicaID, storeLList=False, storeWParam=False)
#main link between models and the GA for the keeping
dalekDB.insertGAIndividual(conn, generationID, keepModelIDs, keepFitness)
#main link between models and the GA
dalekDB.insertGAIndividual(conn, generationID, modelIDs, fitness)
curs.execute('update GA_RUN set GA_CUR_GEN=? where ID=?', (generationID, GARunID))
#getting new generation
curs.execute('insert into GA_GENERATION(GA_RUN_ID) VALUES(?)', (GARunID,))
generationID = curs.lastrowid
#uniting old keepChildren and current generation model
curGenerationModel.grid=np.concatenate((keepChildren.grid,curGenerationModel.grid))
curGenerationModel._initSpecs()
modelIDs = np.concatenate((keepModelIDs, modelIDs))
#getting fitnesses from the model
fitness = curGenerationModel['fitness']
fitnessIDX = np.argsort(fitness)[::-1]
#Checking for break conditions
if os.path.exists('break_after_loop'): break
if os.path.exists('debug_after_loop'):
try:
os.remove('debug_after_loop')
except:
pass
pdb.set_trace()
#start selective breeding
#First the children that are kept for the subpopulation are put into a seperate variable
if config.GAConfDict['mode'] == 'subpopulation':
if populationSize==subPopulationNo:
keepChildren=param.multiParam()
keepChildren.grid=np.array([])
else:
keepChildren = model.modelGrid(paramList=curGenerationModel.
grid[fitnessIDX[:(populationSize-subPopulationNo)]],
origSpec=origSpec)
keepModelIDs = modelIDs[fitnessIDX[:(populationSize - subPopulationNo)]]
keepFitness = fitness[fitnessIDX[:(populationSize - subPopulationNo)]]
keepChildren._initSpecs()
elif config.GAConfDict['mode'] == 'elitism':
keepChildren = model.modelGrid(paramList=curGenerationModel.
grid[fitnessIDX[:int(populationSize * config.GAConfDict['elitism'])]],
origSpec=origSpec)
keepModelIDs = modelIDs[fitnessIDX[:(populationSize * config.GAConfDict['elitism'])]]
keepFitness = fitness[fitnessIDX[:(populationSize * config.GAConfDict['elitism'])]]
keepChildren._initSpecs()
#Now we get the population that is used for breeding and submit it to the breed function
breedPopulation=model.modelGrid(paramList=curGenerationModel.
grid[fitnessIDX[:generationGapNo]],
origSpec=origSpec)
if config.GAConfDict['mode'] == 'subpopulation':
curGenerationSet=breed(breedPopulation,
popNum=subPopulationNo, select=selectFunc, cross=crossFunc)
elif config.GAConfDict['mode'] == 'elitism':
curGenerationSet=breed(breedPopulation,
popNum=int(populationSize*(1-config.GAConfDict['elitism'])), select=selectFunc, cross=crossFunc)
del curGenerationModel
#Time is kept
ficaTime=time.time()
#Network check for available nodes
gws.checkAvailability()
pp.pprint(gws.availability)
#Calculating the new generation with elauncher
curGenerationModel = elauncher.cloudLaunch(curGenerationSet.paramGrid, gws.getAvailGateWays(), origSpec=origSpec)
conn.commit()
#Printing time statements
print "Took %.3f for the fica runs" % (time.time()-ficaTime)
print "Took %.3f seconds for last loop" % (time.time()-curTime)
curs.execute('update GA_RUN set END_TIME=? where ID=?', (datetime.datetime.now(), GARunID))
def prepGATry(gaNo,description):
wikiRoot='/home/wkerzend/public_html/dalek/data/pages'
wikiMediaRoot='/home/wkerzend/public_html/dalek/data/media'
gaStore='/priv/miner3/skymap/wkerzend/ga_store2'
SNName=config.getName().lower()
t=config.getTimeFromExplosion()
GeneralOverviewFile=os.path.join(wikiRoot,'sn_gas.txt')
SNOverviewFile=os.path.join(wikiRoot,'%s_gas.txt'%SNName)
SNTimeFile=os.path.join(wikiRoot,'%s_t%.2f.txt'%(SNName,t))
GATryFile=os.path.join(wikiRoot,'%s_t%.2f_ga%d.txt'%(SNName,t,gaNo))
mediaNamespace=":%s:%s:%s:"%(SNName,'%s_t%.2f'%(SNName,t),'ga_try%d'%gaNo)
try:
os.makedirs(os.path.join(gaStore,SNName,'%s_t%.2f'%(SNName,t),'ga_try%d'%gaNo))
except:
pass
os.system('ln -s %s .'%os.path.join(gaStore,SNName,'%s_t%.2f'%(SNName,t),'ga_try%d'%gaNo))
paramString="""
lumLimits=%s
vphLimits=%s
selElements=%s
selParameters=%s
seed=%s
mode=%s
generationGapFraction=%.3f
subPopulationFraction=%.3f
elitism=%.3f
scaleFitness=%s
Cmult=%s
mutationParams=%s
crossOverProbability=%s
"""%(lumLimits,vphLimits,selElements,selParameters,seed,mode,generationGapFraction,subPopulationFraction,elitism,scaleFitness,Cmult,mutationParams,crossOverProbability)
templatePage="""{{%sgen_vs_fitness_linear.png?400}}
{{%sgen_vs_fitness_log.png?400}}
x=generation
y=mean(fitness) in this case sum square error
yerr=std(fitness)
===Current Parameter===
%s
===Description===
%s
^ Spectra ^ Status ^ Histograms ^
| {{filelist>%sgener*.pdf&style=table&tableheader=1&tableshowdate=1&tableshowsize=1&order=desc}} | {{filelist>%sgen_st*.pdf&style=table&tableheader=1&tableshowdate=1&tableshowsize=1&order=desc}} | {{filelist>%sgen_his*.pdf&style=table&tableheader=1&tableshowdate=1&tableshowsize=1&order=desc}} |
"""%(mediaNamespace,mediaNamespace,paramString,description,mediaNamespace,mediaNamespace,mediaNamespace)
#Create page structure and create media structure
if os.path.exists(SNOverviewFile):
if os.path.exists(SNTimeFile):
file(SNTimeFile,'a').write("[[%s_t%.2f_ga%d|%s t=%.2f ga try=%d]]\n\n"%(SNName,t,gaNo,SNName,t,gaNo))
file(GATryFile,'w').write(templatePage)
else:
file(SNOverviewFile,'a').write('[[%s_t%.2f|%s t=%.3f]]\n\n'%(SNName,t,SNName,t))
file(SNTimeFile,'a').write("[[%s_t%.2f_ga%d|%s t=%.2f ga try=%d]]\n\n"%(SNName,t,gaNo,SNName,t,gaNo))
file(GATryFile,'w').write(templatePage)
else:
file(GeneralOverviewFile,'a').write("[[%s_gas|%s GA overview]]\n\n"%(SNName,SNName))
file(SNOverviewFile,'a').write('[[%s_t%.2f|%s t=%.3f]]\n\n'%(SNName,t,SNName,t))
file(SNTimeFile,'a').write("[[%s_t%.2f_ga%d|%s t=%.2f ga try=%d]]\n\n"%(SNName,t,gaNo,SNName,t,gaNo))
file(GATryFile,'w').write(templatePage)
