Example #1
0
def runManual(param, gateways=None):
    if gateways==None:
        gateways = elauncher.gateways()
    runPath = os.path.join(os.getcwd(), uuid.uuid4().hex)
    param.targetDir = runPath
    model = elauncher.cloudLaunch([param,], gateways.getAvailGateWays())
    return model[0]
Example #2
0
def evolve(conn, SNSpecID, GARunID=None, description=None, generations=20, populationSize=150,
           breedFunc = breed, selectFunc=geneticDalekAlgo.selectRoulette,
           crossFunc=geneticDalekAlgo.crossSingle,
           randomParamFunc = geneticDalekAlgo.createRandomLogNormalValueW7):
    
    
    
    
    #Run the GA

    startTime = datetime.datetime.now()
    #Initializing cursor
    curs = conn.cursor()
    
    
    #trying to remove existing break and debug switches
    try:
        os.remove('break_after_loop')
    except:
        pass
    try:
        os.remove('debug_after_loop')
    except:
        pass
    
    #Initializing random seed
    random.seed(config.GAConfDict['seed'])
    
    #Initializing mode dependent constants:
    generationGapNo=int(populationSize * config.GAConfDict['generationGapFraction'])
    
    subPopulationNo=int(populationSize * config.GAConfDict['generationGapFraction']
                        * config.GAConfDict['subPopulationFraction'])
    
    #Launching the gateways
    gws=elauncher.gateways()
    
    
    
    #getting origSpec and preparing it
    rawOrigSpec = curs.execute('select SPECTRUM from SN_SPECTRA where ID=%d' % SNSpecID).fetchall()[0]
    origSpec = initialize.preProcessOrigSpec(rawOrigSpec[0])
    
    
    breedSource = dalekDB.makeZipPickle(inspect.getsource(breedFunc))
    GAConfSource = dalekDB.makeZipPickle(config.GAConfDict)
    crossSource = dalekDB.makeZipPickle(inspect.getsource(crossFunc))
    selectSource = dalekDB.makeZipPickle(inspect.getsource(selectFunc))
    fitSource = dalekDB.makeZipPickle(inspect.getsource(genFitness.fitFunc))
    
    
    #Getting SNConfigDict
    SNConfigDict = config.getSNConfigDict(conn)
    SNConfigDict['t'] = config.getTimeFromExplosion(conn, SNSpecID, SNConfigDict)
    
    
    
    #setting time
    param.SNConfigDict = SNConfigDict
    
    #Checking for continue or new
    if GARunID!=None:
        raise NotImplementedError('soon.....')
        
    
    
    if GARunID == None: #or GA_CUR_GEN=0    
        #creating new GA_RUN entry and inserting the code of various functions
        
        curs.execute('insert into GA_RUN(DESCRIPTION, SN_ID, START_TIME,'
                     'SN_SPECTRUM, GA_POP_SIZE, GA_GENERATION_SIZE, GA_CONF_DICT, GA_BREED_FUNC,'
                     'GA_CROSS_FUNC, GA_SELECT_FUNC, GA_FITNESS_FUNC)'
                     ' VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
                     (description, SNSpecID, startTime, 
                      origSpec, populationSize, generations, GAConfSource, breedSource,
                      crossSource, selectSource, fitSource))
        
        GARunID = curs.lastrowid
        
        curs.execute('insert into GA_GENERATION(GA_RUN_ID) VALUES(?)', (GARunID,))
        
        generationID = curs.lastrowid
        
        curGenerationSet = geneticDalekAlgo.createRandomParamSet(populationSize)
        
        #Inserting the dica into the database, this dica will be used throughout the run
        dicaID = dalekDB.insertDica(conn, curGenerationSet.paramGrid[0].dica)
        
        firstGeneration = 0
        
        # Checking availability on gateways
        gws.checkAvailability()
        pp.pprint(gws.availability)
        
        #Launching 
        curGenerationModel = elauncher.cloudLaunch(curGenerationSet.paramGrid, gws.getAvailGateWays(), origSpec=origSpec)
        #getting fitnesses from the model
        fitness = curGenerationModel['fitness']
        fitnessIDX = np.argsort(fitness)[::-1]
        
        keepChildren = param.multiParam()
        keepChildren.grid = np.array([])
        keepModelIDs = np.array([])
        keepFitness = np.array([])
        
    
    
    for i in range(firstGeneration,generations):
    
        #getting current time
        curTime=time.time()
        
        
        
        #saving model to db
        modelIDs = curGenerationModel.toDB(conn, dicaID=dicaID, storeLList=False, storeWParam=False)
        
        
        #main link between models and the GA for the keeping
        dalekDB.insertGAIndividual(conn, generationID, keepModelIDs, keepFitness)
        
        #main link between models and the GA 
        dalekDB.insertGAIndividual(conn, generationID, modelIDs, fitness)
        
        curs.execute('update GA_RUN set GA_CUR_GEN=? where ID=?', (generationID, GARunID))
        
        #getting new generation
        curs.execute('insert into GA_GENERATION(GA_RUN_ID) VALUES(?)', (GARunID,))
        
        generationID = curs.lastrowid

        
        #uniting old keepChildren and current generation model
        curGenerationModel.grid=np.concatenate((keepChildren.grid,curGenerationModel.grid))
        curGenerationModel._initSpecs()
        modelIDs = np.concatenate((keepModelIDs, modelIDs))
        
        #getting fitnesses from the model
        fitness = curGenerationModel['fitness']
        fitnessIDX = np.argsort(fitness)[::-1]
        
        #Checking for break conditions
        if os.path.exists('break_after_loop'): break
        if os.path.exists('debug_after_loop'):
            try:
                os.remove('debug_after_loop')
            except:
                pass
            pdb.set_trace()
        
        
        #start selective breeding
        #First the children that are kept for the subpopulation are put into a seperate variable
        if config.GAConfDict['mode'] == 'subpopulation':
            if populationSize==subPopulationNo:
                keepChildren=param.multiParam()
                keepChildren.grid=np.array([])
            else:
                keepChildren = model.modelGrid(paramList=curGenerationModel.
                                grid[fitnessIDX[:(populationSize-subPopulationNo)]],
                                origSpec=origSpec)
                keepModelIDs = modelIDs[fitnessIDX[:(populationSize - subPopulationNo)]]
                keepFitness = fitness[fitnessIDX[:(populationSize - subPopulationNo)]]
                keepChildren._initSpecs()
        
        elif config.GAConfDict['mode'] == 'elitism':
            keepChildren = model.modelGrid(paramList=curGenerationModel.
                            grid[fitnessIDX[:int(populationSize * config.GAConfDict['elitism'])]],
                            origSpec=origSpec)
            keepModelIDs = modelIDs[fitnessIDX[:(populationSize * config.GAConfDict['elitism'])]]
            keepFitness = fitness[fitnessIDX[:(populationSize * config.GAConfDict['elitism'])]]
            keepChildren._initSpecs()
        
        #Now we get the population that is used for breeding and submit it to the breed function
        breedPopulation=model.modelGrid(paramList=curGenerationModel.
                                       grid[fitnessIDX[:generationGapNo]],
                                       origSpec=origSpec)
        
        if config.GAConfDict['mode'] == 'subpopulation':
            curGenerationSet=breed(breedPopulation,
                  popNum=subPopulationNo, select=selectFunc, cross=crossFunc)
        elif config.GAConfDict['mode'] == 'elitism':
            curGenerationSet=breed(breedPopulation,
                  popNum=int(populationSize*(1-config.GAConfDict['elitism'])), select=selectFunc, cross=crossFunc)
        
        del curGenerationModel
        
        #Time is kept
        ficaTime=time.time()
    
        #Network check for available nodes
        gws.checkAvailability()
        pp.pprint(gws.availability)
        
        
        #Calculating the new generation with elauncher
        curGenerationModel = elauncher.cloudLaunch(curGenerationSet.paramGrid, gws.getAvailGateWays(), origSpec=origSpec)
        
        conn.commit()
        
        #Printing time statements
        print "Took %.3f for the fica runs" % (time.time()-ficaTime)
        print "Took %.3f seconds for last loop" % (time.time()-curTime)
        
    curs.execute('update GA_RUN set END_TIME=? where ID=?', (datetime.datetime.now(), GARunID))
Example #3
0
def evolve2(generations=200,populationSize=150,savePath='.',continueGA=False,mutationRate=0.2,mutationScale=0.05):

#Initializing random seed
    random.seed(seed)

    evolveHeader='#gen specno fitness %s\n'%' '.join(['lum','vph']+selElements)
    evolveHeaderFMT='%d %d %s '+' '.join(['%s']*len(['lum','vph']+selElements))
    evolveDBPath=os.path.join(savePath,'evolution.dat')
    
    file(evolveDBPath,'w').write(evolveHeader)
    
    generationGapNo=int(populationSize*generationGapFraction)
    
    subPopulationNo=int(populationSize*generationGapFraction*subPopulationFraction)
    
    gws=elauncher.gateways()
    
    basePath=config.getAutoDir()
    
    #Smoothing UV
    origSpec=config.getOrigSpec(preProcess=True)
    try:
            os.remove('break_after_loop')
    except:
        pass
    try:
        os.remove('debug_after_loop')
    except:
        pass
    #continue GA if stuff there if not make new one
    
    if continueGA:
        x=[]
        y=[]
        y2=[]
        yerr=[]
        for fname in np.sort(glob(os.path.join(savePath,'generation*.pkl'))):
            x.append(int(re.search('\d+',os.path.basename(fname)).group()))
            fitness=np.loadtxt(fname.replace('.pkl','.dat'))
            y.append(np.mean(fitness))
            y2.append(np.max(fitness))
            yerr.append(np.std(fitness))
        plotSet.genPlotGenVSFitness(x,y,y2,yerr,outName='gen_vs_fitness_log.png')
        plotSet.genPlotGenVSFitness(x,y,y2,yerr,logPlot=False,outName='gen_vs_fitness_linear.png')
        curGenerationModel=cPickle.load(file(os.path.join(savePath,'generation%04d.pkl'%np.max(x))))
        firstGeneration=np.max(x)+1
    else:
        x=[]#generation Number
        y=[]#fitness medium
        y2=[]#fitness max
        yerr=[]#fitness std
        curGenerationSet=createRandomParamSet(populationSize)
        firstGeneration=0
        gws.checkAvailability()
        print gws.availability
        curGenerationModel,tmp=elauncher.cloudLaunch(curGenerationSet.paramGrid,gws.getAvailGateWays(),origSpec=origSpec)        
        keepChildren=param.multiParam()
        keepChildren.grid=np.array([])
        
    for i in range(firstGeneration,generations):
        
        curTime=time.time()
        #Skipping for continueGA
        if not continueGA:
            curGenerationModel.grid=np.concatenate((keepChildren.grid,curGenerationModel.grid))
            curGenerationModel._initSpecs()
            if i%5 == 0:
                fname="generation%04d.pkl"%i
                print "Saving current generation to %s"%fname
                cPickle.dump(curGenerationModel,file(os.path.join(savePath,fname),'w'))
                print "Saved"
        else:
            continueGA=False
        fitness=curGenerationModel['fitness']
        fitnessIDX=np.argsort(fitness)[::-1]
        #TEST CASE FOR VARYING OPEN PARAMETERS ________ TEST _____ TEST
        if i == 1:
            openGAParameters=['lum','vph']
        
        #TEST CASE FOR VARYING OPEN PARAMETERS ________ TEST _____ TEST
        #Saving the fitness, making plots
        np.savetxt(os.path.join(savePath,'generation%04d.dat'%i),fitness)
        x.append(i)
        y.append(np.mean(fitness))
        y2.append(np.max(fitness))
        yerr.append(np.std(fitness))
        
        #Saving to evolution DB
        evolveDB=zip([i]*populationSize,range(populationSize),
            *[curGenerationModel[item] for item in ['fitness','lum','vph']+selElements])
        np.savetxt(file(evolveDBPath,'a'),evolveDB,fmt=evolveHeaderFMT)
        
        #Plotting the generation
        plotSet.genPlotSingleGeneration(curGenerationModel,
                                        fitness,
                                        generation=i,
                                        no=10,
                                        pdfName=os.path.join(savePath,'generation%04d.pdf'%i))
        
        plotSet.genPlotStatus(curGenerationModel,
                              pdfName=os.path.join(savePath,'gen_status%04d.pdf'%i))
        
        plotSet.genPlotHistogram(curGenerationModel,
                                 fitness=fitness,
                                 pdfName=os.path.join(savePath,'gen_histogram%04d.pdf'%i))
        
        plotSet.genPlotGenVSFitness(x,y,y2,yerr,
                                    outName=os.path.join(savePath,'gen_vs_fitness_log.png'))
        plotSet.genPlotGenVSFitness(x,y,y2,yerr,
                                    logPlot=False,
                                    outName=os.path.join(savePath,'gen_vs_fitness_linear.png'))
        #Checking for break conditions
        if os.path.exists('break_after_loop') or os.path.exists(os.path.join(savePath,'break_after_loop')): break
        if os.path.exists('debug_after_loop'):
            try:
                os.remove('debug_after_loop')
            except:
                pass
            pdb.set_trace()
        
        #made plots
        
        #start selective breeding
        #First the children that are kept for the subpopulation are put into a seperate variable
        if mode=='subpopulation':
            if populationSize==subPopulationNo:
                keepChildren=param.multiParam()
                keepChildren.grid=np.array([])
            else:
                keepChildren=read.modelGrid(paramList=curGenerationModel.
                             grid[fitnessIDX[:(populationSize-subPopulationNo)]])
                keepChildren._initSpecs()
        elif mode=='elitism':
            keepChildren=read.modelGrid(paramList=curGenerationModel.
                             grid[fitnessIDX[:int(populationSize*elitism)]])
            keepChildren._initSpecs()
        #Now we get the population that is used for breeding and submit it to the breed function
        breedPopulation=read.modelGrid(paramList=curGenerationModel.
                                       grid[fitnessIDX[:generationGapNo]])
        
        if mode=='subpopulation':
            curGenerationSet=breed(breedPopulation,
                  popNum=subPopulationNo)
        elif mode=='elitism':
            curGenerationSet=breed(breedPopulation,
                  popNum=int(populationSize*(1-elitism)))
        del curGenerationModel
        
        #Time is kept
        
        ficaTime=time.time()
    
        #Network check for available nodes
        gws.checkAvailability()
        print gws.availability
        
        #Calculating the new generation with elauncher.
        curGenerationModel,tmp=elauncher.cloudLaunch(curGenerationSet.paramGrid,gws.getAvailGateWays(),origSpec=origSpec)
        
        
        #Printing time statements
        print "Took %s for the fica runs"%(time.time()-ficaTime)
        print "Took %s seconds for last loop"%(time.time()-curTime)