def createSingleTrainingSet(setSize,iter,SNName='2002bo'): t=config.getTimeFromExplosion() outnames=[] for i in range(iter): curGenerationSet=geneticDalek.createRandomParamSet(setSize) print "At Generation %s"%i curGenerationModel=launcherSteps.getModel(curGenerationSet)[0] outname='%st%.2f.ts%0d.pkl'%(SNName,t,i) print "Saving current generation to %s"%outname cPickle.dump(curGenerationModel,file(outname,'w')) outnames.append(outname) return outnames
def initLumVphGrid(step=1,gridSize=10,doPickle=True): t=config.getTimeFromExplosion() vph=initialize.time2vph(t) mainConfigDir=config.getMainConfigDir() modelW7=initialize.readW7Data(os.path.join(mainConfigDir,'w7.combined.dat')) initDica=config.getVanillaDica() initDica['t']=t #Preparing the normalization initComp=config.getVanillaComp() initComp.update(initialize.getW7Comp(modelW7,t)) initComp=abund.setNiDecay(initComp,t) initComp=abund.setCONe(initComp) initComp=abund.normAbundances(initComp) initRunDir='init_lumvph_run/' initialVph=[0.6*vph,1.4*vph] return lumVphGrid(initialLum,initialVph,1,initRunDir,initComp=initComp,initDica=initDica,gridSize=gridSize,doPickle=doPickle)
def getPrevDica(): dicaFile=os.path.join(config.getLastMainDir(),'manual','dica.dat') dica=fileio.dicafile(dicaFile).read_data() print "Time=%s"%config.getTimeFromExplosion() dica['t']=config.getTimeFromExplosion() return dica
def getCurVph(): t=config.getTimeFromExplosion() return time2vph(t)
def evolve(conn, SNSpecID, GARunID=None, description=None, generations=20, populationSize=150, breedFunc = breed, selectFunc=geneticDalekAlgo.selectRoulette, crossFunc=geneticDalekAlgo.crossSingle, randomParamFunc = geneticDalekAlgo.createRandomLogNormalValueW7): #Run the GA startTime = datetime.datetime.now() #Initializing cursor curs = conn.cursor() #trying to remove existing break and debug switches try: os.remove('break_after_loop') except: pass try: os.remove('debug_after_loop') except: pass #Initializing random seed random.seed(config.GAConfDict['seed']) #Initializing mode dependent constants: generationGapNo=int(populationSize * config.GAConfDict['generationGapFraction']) subPopulationNo=int(populationSize * config.GAConfDict['generationGapFraction'] * config.GAConfDict['subPopulationFraction']) #Launching the gateways gws=elauncher.gateways() #getting origSpec and preparing it rawOrigSpec = curs.execute('select SPECTRUM from SN_SPECTRA where ID=%d' % SNSpecID).fetchall()[0] origSpec = initialize.preProcessOrigSpec(rawOrigSpec[0]) breedSource = dalekDB.makeZipPickle(inspect.getsource(breedFunc)) GAConfSource = dalekDB.makeZipPickle(config.GAConfDict) crossSource = dalekDB.makeZipPickle(inspect.getsource(crossFunc)) selectSource = dalekDB.makeZipPickle(inspect.getsource(selectFunc)) fitSource = dalekDB.makeZipPickle(inspect.getsource(genFitness.fitFunc)) #Getting SNConfigDict SNConfigDict = config.getSNConfigDict(conn) SNConfigDict['t'] = config.getTimeFromExplosion(conn, SNSpecID, SNConfigDict) #setting time param.SNConfigDict = SNConfigDict #Checking for continue or new if GARunID!=None: raise NotImplementedError('soon.....') if GARunID == None: #or GA_CUR_GEN=0 #creating new GA_RUN entry and inserting the code of various functions curs.execute('insert into GA_RUN(DESCRIPTION, SN_ID, START_TIME,' 'SN_SPECTRUM, GA_POP_SIZE, GA_GENERATION_SIZE, GA_CONF_DICT, GA_BREED_FUNC,' 'GA_CROSS_FUNC, GA_SELECT_FUNC, GA_FITNESS_FUNC)' ' VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (description, SNSpecID, startTime, origSpec, populationSize, generations, GAConfSource, breedSource, crossSource, selectSource, fitSource)) GARunID = curs.lastrowid curs.execute('insert into GA_GENERATION(GA_RUN_ID) VALUES(?)', (GARunID,)) generationID = curs.lastrowid curGenerationSet = geneticDalekAlgo.createRandomParamSet(populationSize) #Inserting the dica into the database, this dica will be used throughout the run dicaID = dalekDB.insertDica(conn, curGenerationSet.paramGrid[0].dica) firstGeneration = 0 # Checking availability on gateways gws.checkAvailability() pp.pprint(gws.availability) #Launching curGenerationModel = elauncher.cloudLaunch(curGenerationSet.paramGrid, gws.getAvailGateWays(), origSpec=origSpec) #getting fitnesses from the model fitness = curGenerationModel['fitness'] fitnessIDX = np.argsort(fitness)[::-1] keepChildren = param.multiParam() keepChildren.grid = np.array([]) keepModelIDs = np.array([]) keepFitness = np.array([]) for i in range(firstGeneration,generations): #getting current time curTime=time.time() #saving model to db modelIDs = curGenerationModel.toDB(conn, dicaID=dicaID, storeLList=False, storeWParam=False) #main link between models and the GA for the keeping dalekDB.insertGAIndividual(conn, generationID, keepModelIDs, keepFitness) #main link between models and the GA dalekDB.insertGAIndividual(conn, generationID, modelIDs, fitness) curs.execute('update GA_RUN set GA_CUR_GEN=? where ID=?', (generationID, GARunID)) #getting new generation curs.execute('insert into GA_GENERATION(GA_RUN_ID) VALUES(?)', (GARunID,)) generationID = curs.lastrowid #uniting old keepChildren and current generation model curGenerationModel.grid=np.concatenate((keepChildren.grid,curGenerationModel.grid)) curGenerationModel._initSpecs() modelIDs = np.concatenate((keepModelIDs, modelIDs)) #getting fitnesses from the model fitness = curGenerationModel['fitness'] fitnessIDX = np.argsort(fitness)[::-1] #Checking for break conditions if os.path.exists('break_after_loop'): break if os.path.exists('debug_after_loop'): try: os.remove('debug_after_loop') except: pass pdb.set_trace() #start selective breeding #First the children that are kept for the subpopulation are put into a seperate variable if config.GAConfDict['mode'] == 'subpopulation': if populationSize==subPopulationNo: keepChildren=param.multiParam() keepChildren.grid=np.array([]) else: keepChildren = model.modelGrid(paramList=curGenerationModel. grid[fitnessIDX[:(populationSize-subPopulationNo)]], origSpec=origSpec) keepModelIDs = modelIDs[fitnessIDX[:(populationSize - subPopulationNo)]] keepFitness = fitness[fitnessIDX[:(populationSize - subPopulationNo)]] keepChildren._initSpecs() elif config.GAConfDict['mode'] == 'elitism': keepChildren = model.modelGrid(paramList=curGenerationModel. grid[fitnessIDX[:int(populationSize * config.GAConfDict['elitism'])]], origSpec=origSpec) keepModelIDs = modelIDs[fitnessIDX[:(populationSize * config.GAConfDict['elitism'])]] keepFitness = fitness[fitnessIDX[:(populationSize * config.GAConfDict['elitism'])]] keepChildren._initSpecs() #Now we get the population that is used for breeding and submit it to the breed function breedPopulation=model.modelGrid(paramList=curGenerationModel. grid[fitnessIDX[:generationGapNo]], origSpec=origSpec) if config.GAConfDict['mode'] == 'subpopulation': curGenerationSet=breed(breedPopulation, popNum=subPopulationNo, select=selectFunc, cross=crossFunc) elif config.GAConfDict['mode'] == 'elitism': curGenerationSet=breed(breedPopulation, popNum=int(populationSize*(1-config.GAConfDict['elitism'])), select=selectFunc, cross=crossFunc) del curGenerationModel #Time is kept ficaTime=time.time() #Network check for available nodes gws.checkAvailability() pp.pprint(gws.availability) #Calculating the new generation with elauncher curGenerationModel = elauncher.cloudLaunch(curGenerationSet.paramGrid, gws.getAvailGateWays(), origSpec=origSpec) conn.commit() #Printing time statements print "Took %.3f for the fica runs" % (time.time()-ficaTime) print "Took %.3f seconds for last loop" % (time.time()-curTime) curs.execute('update GA_RUN set END_TIME=? where ID=?', (datetime.datetime.now(), GARunID))
def prepGATry(gaNo,description): wikiRoot='/home/wkerzend/public_html/dalek/data/pages' wikiMediaRoot='/home/wkerzend/public_html/dalek/data/media' gaStore='/priv/miner3/skymap/wkerzend/ga_store2' SNName=config.getName().lower() t=config.getTimeFromExplosion() GeneralOverviewFile=os.path.join(wikiRoot,'sn_gas.txt') SNOverviewFile=os.path.join(wikiRoot,'%s_gas.txt'%SNName) SNTimeFile=os.path.join(wikiRoot,'%s_t%.2f.txt'%(SNName,t)) GATryFile=os.path.join(wikiRoot,'%s_t%.2f_ga%d.txt'%(SNName,t,gaNo)) mediaNamespace=":%s:%s:%s:"%(SNName,'%s_t%.2f'%(SNName,t),'ga_try%d'%gaNo) try: os.makedirs(os.path.join(gaStore,SNName,'%s_t%.2f'%(SNName,t),'ga_try%d'%gaNo)) except: pass os.system('ln -s %s .'%os.path.join(gaStore,SNName,'%s_t%.2f'%(SNName,t),'ga_try%d'%gaNo)) paramString=""" lumLimits=%s vphLimits=%s selElements=%s selParameters=%s seed=%s mode=%s generationGapFraction=%.3f subPopulationFraction=%.3f elitism=%.3f scaleFitness=%s Cmult=%s mutationParams=%s crossOverProbability=%s """%(lumLimits,vphLimits,selElements,selParameters,seed,mode,generationGapFraction,subPopulationFraction,elitism,scaleFitness,Cmult,mutationParams,crossOverProbability) templatePage="""{{%sgen_vs_fitness_linear.png?400}} {{%sgen_vs_fitness_log.png?400}} x=generation y=mean(fitness) in this case sum square error yerr=std(fitness) ===Current Parameter=== %s ===Description=== %s ^ Spectra ^ Status ^ Histograms ^ | {{filelist>%sgener*.pdf&style=table&tableheader=1&tableshowdate=1&tableshowsize=1&order=desc}} | {{filelist>%sgen_st*.pdf&style=table&tableheader=1&tableshowdate=1&tableshowsize=1&order=desc}} | {{filelist>%sgen_his*.pdf&style=table&tableheader=1&tableshowdate=1&tableshowsize=1&order=desc}} | """%(mediaNamespace,mediaNamespace,paramString,description,mediaNamespace,mediaNamespace,mediaNamespace) #Create page structure and create media structure if os.path.exists(SNOverviewFile): if os.path.exists(SNTimeFile): file(SNTimeFile,'a').write("[[%s_t%.2f_ga%d|%s t=%.2f ga try=%d]]\n\n"%(SNName,t,gaNo,SNName,t,gaNo)) file(GATryFile,'w').write(templatePage) else: file(SNOverviewFile,'a').write('[[%s_t%.2f|%s t=%.3f]]\n\n'%(SNName,t,SNName,t)) file(SNTimeFile,'a').write("[[%s_t%.2f_ga%d|%s t=%.2f ga try=%d]]\n\n"%(SNName,t,gaNo,SNName,t,gaNo)) file(GATryFile,'w').write(templatePage) else: file(GeneralOverviewFile,'a').write("[[%s_gas|%s GA overview]]\n\n"%(SNName,SNName)) file(SNOverviewFile,'a').write('[[%s_t%.2f|%s t=%.3f]]\n\n'%(SNName,t,SNName,t)) file(SNTimeFile,'a').write("[[%s_t%.2f_ga%d|%s t=%.2f ga try=%d]]\n\n"%(SNName,t,gaNo,SNName,t,gaNo)) file(GATryFile,'w').write(templatePage)