Exemple #1
0
  def _restart(self): 
    """ Updates self from SHELVECAR.
    
        Reads the last functional in the shelvecar and updates offspring,
        population, errors, and generation. Nothing else is changed.

        If the shelvecar does not exist, then creates one with current values.
    """
    from os.path import exists
    from shelve import open as shelve_open
    from ..error import internal
    from ..misc import LockFile
    if '_directory' not in self.__dict__:
      raise internal("Directory has not yet been set.")
    with LockFile(self.shelvepath, timeout=10) as lock:
      if not exists(self.shelvepath):
        try:
          shelvecar = shelve_open(self.shelvepath)
          shelvecar['individuals'] = {}
          shelvecar['functionals'] = [self]
          shelvecar['added']       = []
          shelvecar['removed']     = []
          shelvecar['new']         = []
        finally: shelvecar.close()
      else:
        try:
          shelvecar = shelve_open(self.shelvepath, writeback=True)
          previous = shelvecar['functionals']
        finally: shelvecar.close()
        self.offspring  = previous[-1].offspring
        self.population = previous[-1].population
        self.errors     = previous[-1].errors
        self.generation = previous[-1].generation
Exemple #2
0
    def __init__(self, name='sid', dir=path_join(WORK_DIR, 'sessions'),
            path=None, domain=None, max_age=None):

        self._name = name
        now = datetime.utcnow();

        # blank cookie
        self._cookie = SimpleCookie()

        if environ.has_key('HTTP_COOKIE'):
            # cookie already exists, see what's in it
            self._cookie.load(environ['HTTP_COOKIE'])

        try:
            # what's our session ID?
            self.sid = self._cookie[name].value;
        except KeyError:
            # there isn't any, make a new session ID
            remote = environ.get('REMOTE_ADDR')
            self.sid = sha224('%s-%s' % (remote, now)).hexdigest()

        self._cookie.clear();
        self._cookie[name] = self.sid

        # set/reset path
        if path:
            self._cookie[name]['path'] = path
        else:
            self._cookie[name]['path'] = ''

        # set/reset domain
        if domain:
            self._cookie[name]['domain'] = domain
        else:
            self._cookie[name]['domain'] = ''

        # set/reset expiration date
        if max_age:
            if isinstance(max_age, int):
                max_age = timedelta(seconds=max_age)
            expires = now + max_age
            self._cookie[name]['expires'] = expires.strftime('%a, %d %b %Y %H:%M:%S')
        else:
            self._cookie[name]['expires'] = ''

        # to protect against cookie-stealing JS, make our cookie
        # available only to the browser, and not to any scripts
        try:
            # This will not work for Python 2.5 and older
            self._cookie[name]['httponly'] = True
        except CookieError:
            pass

        # if the sessions dir doesn't exist, create it
        if not exists(dir):
            mkdir(dir)
        # persist the session data
        self._shelf_file = path_join(dir, self.sid)
        # -1 signifies the highest available protocol version
        self._shelf = shelve_open(self._shelf_file, protocol=-1, writeback=True)
Exemple #3
0
    def __init__(self, name='sid', dir=path_join(WORK_DIR, 'sessions'),
            path=None, domain=None, max_age=None):

        self._name = name
        now = datetime.utcnow();

        # blank cookie
        self._cookie = SimpleCookie()

        if environ.has_key('HTTP_COOKIE'):
            # cookie already exists, see what's in it
            self._cookie.load(environ['HTTP_COOKIE'])

        try:
            # what's our session ID?
            self.sid = self._cookie[name].value;
        except KeyError:
            # there isn't any, make a new session ID
            remote = environ.get('REMOTE_ADDR')
            self.sid = sha224('%s-%s' % (remote, now)).hexdigest()

        self._cookie.clear();
        self._cookie[name] = self.sid

        # set/reset path
        if path:
            self._cookie[name]['path'] = path
        else:
            self._cookie[name]['path'] = ''

        # set/reset domain
        if domain:
            self._cookie[name]['domain'] = domain
        else:
            self._cookie[name]['domain'] = ''

        # set/reset expiration date
        if max_age:
            if isinstance(max_age, int):
                max_age = timedelta(seconds=max_age)
            expires = now + max_age
            self._cookie[name]['expires'] = expires.strftime('%a, %d %b %Y %H:%M:%S')
        else:
            self._cookie[name]['expires'] = ''

        # to protect against cookie-stealing JS, make our cookie
        # available only to the browser, and not to any scripts
        try:
            # This will not work for Python 2.5 and older
            self._cookie[name]['httponly'] = True
        except CookieError:
            pass

        # if the sessions dir doesn't exist, create it
        if not exists(dir):
            mkdir(dir)
        # persist the session data
        self._shelf_file = path_join(dir, self.sid)
        # -1 signifies the highest available protocol version
        self._shelf = shelve_open(self._shelf_file, protocol=-1, writeback=True)
Exemple #4
0
  def __init__(self, directory=None):
    """ Initializes an extraction object. 
  
        :param directory: 
          Directory where the SHELVECAR can be found.
          Defaults to current working directory.

        :raises RuntimeError: if no appropriate SHELVECAR can be found.
    """
    from os.path import exists, isfile, isdir
    from shelve import open as shelve_open
    from ..misc import RelativePath, LockFile
    super(Extract, self).__init__()

    self._directory = RelativePath(directory)
    if not exists(self.directory):return
    if not isdir(self.directory): return
    if not exists(self.shelvepath): return
    if not isfile(self.shelvepath): return

    with LockFile(self.shelvepath, timeout=10) as lock:
      shelve = shelve_open(self.shelvepath)
      try:
        if set(shelve.keys()) !=  set(['individuals', 'functionals', 'removed', 'added', 'new']):
          raise RuntimeError('{0} is not a GA SHELVECAR file.'.format(self.shelvepath))
      finally: shelve.close()
Exemple #5
0
 def functionals(self):
   """ Functionals at each generation. """
   from shelve import open as shelve_open
   from ..misc import LockFile
   with LockFile(self.shelvepath, timeout=10) as lock:
     try:
       shelve = shelve_open(self.shelvepath)
       return shelve['functionals']
     finally: shelve.close()
Exemple #6
0
 def individuals(self):
   """ Dictionary containing all individuals. """
   from shelve import open as shelve_open
   from ..misc import LockFile
   with LockFile(self.shelvepath, timeout=10) as lock:
     try: shelve = shelve_open(self.shelvepath)
     except: raise
     else: return shelve['individuals']
     finally: shelve.close()
Exemple #7
0
def open_db():
    lock = get_write_lock()
    db = None
    try:
        db = shelve_open(Config.db)
    except:
        close_db(db, lock)
        raise Exception(EX_UNAVAILABLE, 'Can\'t open database for writing')
    return (lock, db)
Exemple #8
0
 def update_func(self):
   """ Updates functional in SHELVECAR file. """
   from shelve import open as shelve_open
   from ..misc import LockFile
   with LockFile(self.shelvepath, timeout=10) as file:
     try:
       shelvecar = shelve_open(self.shelvepath, writeback=True)
       shelvecar['functionals'][-1] = self
     finally: shelvecar.close()
Exemple #9
0
def open_db():
    lock = get_write_lock()
    db = None
    try:
        db = shelve_open(Config.db)
    except:
        close_db(db, lock)
        raise Exception(EX_UNAVAILABLE, 'Can\'t open database for writing')
    return (lock, db)
Exemple #10
0
 def removed(self):
   """ Individuals removed at each generation. """
   from shelve import open as shelve_open
   from ..misc import LockFile
   with LockFile(self.shelvepath, timeout=10) as lock:
     try:
       shelve = shelve_open(self.shelvepath)
       result = []
       for r in shelve['removed']:
         result.extend(shelve['individuals'][i] for i in r)
       return result
     finally: shelve.close()
Exemple #11
0
  def __call__(self, outdir, comm): 
    """ Performs GA run until exhaustion sets in. """
    import sys
    from itertools import chain
    from os.path import join
    from shelve import open as shelve_open
    from ..misc import RelativePath, LockFile

    # sets ouput directory for this run.
    self._directory = RelativePath(outdir).path
    # reloads population, offspring, errors, generation from previous run
    # this has to come after setting the directory.
    self._restart()

    # try loops resets old stdout, stderr
    oldouterr = sys.stdout, sys.stderr
    try: 
      sys.stdout = open(join(self._directory, "out"), 'a')
      sys.stderr = open(join(self._directory, "err"), 'a')
     
      print 'COMMUNICATOR', comm 

      # first creates a random population if necessary.
      if len(self.population) == 0 and len(self.offspring) == 0: 
        self.new_random_individuals(n=self.target_popsize)
        self.update_func()
        sys.stdout.flush()

      # then  creates the jobfolder and start the process.
      self.process.start(comm)
      assert hasattr(self.process, '_comm')

      # finally, loop till eternity comes.
      while True:
        if not self.go_to_next_iteration(): continue

        # Now we create new generation. 
        # First we get those individuals that were successfuly computed.
        successfuls = self.find_successfuls()

        # update fitness of whole population, in case it depends upon whole population.
        iterator = (self.offspring[i] for i in successfuls.itervalues())
        self.update_fitness(chain(iterator, self.population))

        # prints out successful individuals.
        if len(successfuls) != 0:
          print "Newly calculated individuals. """
          for index in successfuls.itervalues():
            print " {0.uuid}(conception {0.conception}): {0.fitness}"\
                  .format(self.offspring[index])
          sys.stdout.flush()
         
        # removes worst individuals.
        removed, population, offspring = self.survival(successfuls)
        self.population = population
        self.offspring = offspring

        # increment generation count.
        self.generation += 1

        # create new offspring via mating
        nboffspring = int(self.target_popsize*self.rate+0.1) - len(offspring) \
                      + (self.target_popsize - self.popsize)
        self.mating(nboffspring)

        # udpate jobfolder
        self._update_jobfolder(removed, offspring, population)

        # update shelvecar
        with LockFile(self.shelvepath, timeout=10) as lock:
          try:
            shelvecar = shelve_open(self.shelvepath, writeback=True)
            individuals = shelvecar['individuals']
            for individual in self.population: 
              if not hasattr(individual, 'birth'):
                individual.birth = self.generation
              individuals[str(individual.uuid)] = individual
            for uuid in removed: individuals[str(uuid)].removed = self.generation
            shelvecar['individuals'] = individuals
            shelvecar['functionals'].append(self)
            shelvecar['removed'].append(removed)
            shelvecar['added'].append(successfuls.keys())
            shelvecar['new'].append( [ u.uuid for u in self.offspring\
                                       if u.conception == self.generation-1] )
          finally: shelvecar.close()
        print "\n Starting generation {0}.".format(self.generation)
        sys.stdout.flush()

        if  self.checkpoints(): break

    finally:
      sys.stdout.close()
      sys.stderr.close()
      sys.stdout, sys.stderr = oldouterr 
      if '_process' in self.__dict__:
        self._process.terminate()
        del self._process
      self.__dict__.pop('_directory', None)
      self.__dict__.pop('_jobfolder', None)
    # returns an extraction object. 
    # Might never get here, but that's ok.
    return self.Extract(outdir)
Exemple #12
0
 def __init__(self, *pargs, **kwargs):
     """
     Creates ShelveProxy object, passing constructor arguments to shelve.open
     """
     self._s = shelve_open(*pargs, **kwargs)
     self._cache = {}