Beispiel #1
0
def cloudLaunch(params,gateways,origSpec=None,baseDir=None):
    if origSpec==None: origSpec=config.getOrigSpec(preProcess=True)
    if baseDir==None: baseDir = os.getcwd()
    noModels=len(params)
    models=modelList()
    proxyModels=weakref.proxy(models)
    ##CONSTANTS
    params=list(params)
    machines=zip(*gateways)[0]
    machineSlots=dict([[machine,0] for machine in machines])
    startTime=time.time()
    
    #Configuring which machines to use
    
    def callbackFica(confTuple):
        #execution of fica after the program has run
        protocol, i = confTuple[:2]
        gwConfig = gateways[i]
        machineName = gwConfig[0]
        
        #Reading model and appending
        if protocol == 'centralRead':
            ficaWorkDir=confTuple[2]
            model=getModel(ficaWorkDir,origSpec)
            model.machine = machineName
            proxyModels.append(model)
        elif protocol == 'localRead':
            model = pickle.loads(confTuple[2])
            model.machine = machineName
            proxyModels.append(model)
        
        
        threads = gwConfig[2]
        gw=gwConfig[1]
        machineSlots[machineName] -= 1
        
        try: param=params.pop(0)
        except IndexError: return
        machineSlots[machineName] += 1
        ch=launch(param, machineConfig[machine], threads,baseDir, origSpec, gw, i, genWorker)
        ch.setcallback(callbackFica)
    
    channels=[]
    
    for i,gwConfig in enumerate(gateways):
        try: param=params.pop(0)
        except IndexError: break
        machineName=gwConfig[0]
        gw=gwConfig[1]
        threads=gwConfig[2]
        machineSlots[machineName]+=1
        ch=launch(param,machineConfig[machine],threads,baseDir,origSpec,gw,i,genWorker)
        ch.setcallback(callbackFica)
        
    ########
    #Printing the current progress (works only on VT-100 compliant terminals)
    while True:
        for machineName in machineSlots:
            print "%s: %02d"%(machineName,machineSlots[machineName])
        lines=len(machineSlots)+1
        if len(models)==noModels:break
        if params!=[]:
            print "Next process to launch is %s/%s"%\
            (noModels-len(params)+1,noModels)
            sys.stdout.write("\x1b[%sA"%lines)
        else:
            print "Launched all parameter sets, Still %s items in queue"%\
            (noModels-len(models))
            sys.stdout.write("\x1b[%sA"%lines)
        
        time.sleep(checkUpTimeStep)
    wrefModels=weakref.ref(models)
    modelGrid=model.modelGrid(paramList=models,origSpec=origSpec)
    
    del models
    
    timedelta = time.time()-startTime
    
    print "Fica run took %.4f min and %.4f s/model" % (timedelta/60, timedelta/noModels)
    
    return modelGrid
Beispiel #2
0
def evolve(conn, SNSpecID, GARunID=None, description=None, generations=20, populationSize=150,
           breedFunc = breed, selectFunc=geneticDalekAlgo.selectRoulette,
           crossFunc=geneticDalekAlgo.crossSingle,
           randomParamFunc = geneticDalekAlgo.createRandomLogNormalValueW7):
    
    
    
    
    #Run the GA

    startTime = datetime.datetime.now()
    #Initializing cursor
    curs = conn.cursor()
    
    
    #trying to remove existing break and debug switches
    try:
        os.remove('break_after_loop')
    except:
        pass
    try:
        os.remove('debug_after_loop')
    except:
        pass
    
    #Initializing random seed
    random.seed(config.GAConfDict['seed'])
    
    #Initializing mode dependent constants:
    generationGapNo=int(populationSize * config.GAConfDict['generationGapFraction'])
    
    subPopulationNo=int(populationSize * config.GAConfDict['generationGapFraction']
                        * config.GAConfDict['subPopulationFraction'])
    
    #Launching the gateways
    gws=elauncher.gateways()
    
    
    
    #getting origSpec and preparing it
    rawOrigSpec = curs.execute('select SPECTRUM from SN_SPECTRA where ID=%d' % SNSpecID).fetchall()[0]
    origSpec = initialize.preProcessOrigSpec(rawOrigSpec[0])
    
    
    breedSource = dalekDB.makeZipPickle(inspect.getsource(breedFunc))
    GAConfSource = dalekDB.makeZipPickle(config.GAConfDict)
    crossSource = dalekDB.makeZipPickle(inspect.getsource(crossFunc))
    selectSource = dalekDB.makeZipPickle(inspect.getsource(selectFunc))
    fitSource = dalekDB.makeZipPickle(inspect.getsource(genFitness.fitFunc))
    
    
    #Getting SNConfigDict
    SNConfigDict = config.getSNConfigDict(conn)
    SNConfigDict['t'] = config.getTimeFromExplosion(conn, SNSpecID, SNConfigDict)
    
    
    
    #setting time
    param.SNConfigDict = SNConfigDict
    
    #Checking for continue or new
    if GARunID!=None:
        raise NotImplementedError('soon.....')
        
    
    
    if GARunID == None: #or GA_CUR_GEN=0    
        #creating new GA_RUN entry and inserting the code of various functions
        
        curs.execute('insert into GA_RUN(DESCRIPTION, SN_ID, START_TIME,'
                     'SN_SPECTRUM, GA_POP_SIZE, GA_GENERATION_SIZE, GA_CONF_DICT, GA_BREED_FUNC,'
                     'GA_CROSS_FUNC, GA_SELECT_FUNC, GA_FITNESS_FUNC)'
                     ' VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
                     (description, SNSpecID, startTime, 
                      origSpec, populationSize, generations, GAConfSource, breedSource,
                      crossSource, selectSource, fitSource))
        
        GARunID = curs.lastrowid
        
        curs.execute('insert into GA_GENERATION(GA_RUN_ID) VALUES(?)', (GARunID,))
        
        generationID = curs.lastrowid
        
        curGenerationSet = geneticDalekAlgo.createRandomParamSet(populationSize)
        
        #Inserting the dica into the database, this dica will be used throughout the run
        dicaID = dalekDB.insertDica(conn, curGenerationSet.paramGrid[0].dica)
        
        firstGeneration = 0
        
        # Checking availability on gateways
        gws.checkAvailability()
        pp.pprint(gws.availability)
        
        #Launching 
        curGenerationModel = elauncher.cloudLaunch(curGenerationSet.paramGrid, gws.getAvailGateWays(), origSpec=origSpec)
        #getting fitnesses from the model
        fitness = curGenerationModel['fitness']
        fitnessIDX = np.argsort(fitness)[::-1]
        
        keepChildren = param.multiParam()
        keepChildren.grid = np.array([])
        keepModelIDs = np.array([])
        keepFitness = np.array([])
        
    
    
    for i in range(firstGeneration,generations):
    
        #getting current time
        curTime=time.time()
        
        
        
        #saving model to db
        modelIDs = curGenerationModel.toDB(conn, dicaID=dicaID, storeLList=False, storeWParam=False)
        
        
        #main link between models and the GA for the keeping
        dalekDB.insertGAIndividual(conn, generationID, keepModelIDs, keepFitness)
        
        #main link between models and the GA 
        dalekDB.insertGAIndividual(conn, generationID, modelIDs, fitness)
        
        curs.execute('update GA_RUN set GA_CUR_GEN=? where ID=?', (generationID, GARunID))
        
        #getting new generation
        curs.execute('insert into GA_GENERATION(GA_RUN_ID) VALUES(?)', (GARunID,))
        
        generationID = curs.lastrowid

        
        #uniting old keepChildren and current generation model
        curGenerationModel.grid=np.concatenate((keepChildren.grid,curGenerationModel.grid))
        curGenerationModel._initSpecs()
        modelIDs = np.concatenate((keepModelIDs, modelIDs))
        
        #getting fitnesses from the model
        fitness = curGenerationModel['fitness']
        fitnessIDX = np.argsort(fitness)[::-1]
        
        #Checking for break conditions
        if os.path.exists('break_after_loop'): break
        if os.path.exists('debug_after_loop'):
            try:
                os.remove('debug_after_loop')
            except:
                pass
            pdb.set_trace()
        
        
        #start selective breeding
        #First the children that are kept for the subpopulation are put into a seperate variable
        if config.GAConfDict['mode'] == 'subpopulation':
            if populationSize==subPopulationNo:
                keepChildren=param.multiParam()
                keepChildren.grid=np.array([])
            else:
                keepChildren = model.modelGrid(paramList=curGenerationModel.
                                grid[fitnessIDX[:(populationSize-subPopulationNo)]],
                                origSpec=origSpec)
                keepModelIDs = modelIDs[fitnessIDX[:(populationSize - subPopulationNo)]]
                keepFitness = fitness[fitnessIDX[:(populationSize - subPopulationNo)]]
                keepChildren._initSpecs()
        
        elif config.GAConfDict['mode'] == 'elitism':
            keepChildren = model.modelGrid(paramList=curGenerationModel.
                            grid[fitnessIDX[:int(populationSize * config.GAConfDict['elitism'])]],
                            origSpec=origSpec)
            keepModelIDs = modelIDs[fitnessIDX[:(populationSize * config.GAConfDict['elitism'])]]
            keepFitness = fitness[fitnessIDX[:(populationSize * config.GAConfDict['elitism'])]]
            keepChildren._initSpecs()
        
        #Now we get the population that is used for breeding and submit it to the breed function
        breedPopulation=model.modelGrid(paramList=curGenerationModel.
                                       grid[fitnessIDX[:generationGapNo]],
                                       origSpec=origSpec)
        
        if config.GAConfDict['mode'] == 'subpopulation':
            curGenerationSet=breed(breedPopulation,
                  popNum=subPopulationNo, select=selectFunc, cross=crossFunc)
        elif config.GAConfDict['mode'] == 'elitism':
            curGenerationSet=breed(breedPopulation,
                  popNum=int(populationSize*(1-config.GAConfDict['elitism'])), select=selectFunc, cross=crossFunc)
        
        del curGenerationModel
        
        #Time is kept
        ficaTime=time.time()
    
        #Network check for available nodes
        gws.checkAvailability()
        pp.pprint(gws.availability)
        
        
        #Calculating the new generation with elauncher
        curGenerationModel = elauncher.cloudLaunch(curGenerationSet.paramGrid, gws.getAvailGateWays(), origSpec=origSpec)
        
        conn.commit()
        
        #Printing time statements
        print "Took %.3f for the fica runs" % (time.time()-ficaTime)
        print "Took %.3f seconds for last loop" % (time.time()-curTime)
        
    curs.execute('update GA_RUN set END_TIME=? where ID=?', (datetime.datetime.now(), GARunID))