def _restart(self):
""" Updates self from SHELVECAR.
Reads the last functional in the shelvecar and updates offspring,
population, errors, and generation. Nothing else is changed.
If the shelvecar does not exist, then creates one with current values.
"""
from os.path import exists
from shelve import open as shelve_open
from ..error import internal
from ..misc import LockFile
if '_directory' not in self.__dict__:
raise internal("Directory has not yet been set.")
with LockFile(self.shelvepath, timeout=10) as lock:
if not exists(self.shelvepath):
try:
shelvecar = shelve_open(self.shelvepath)
shelvecar['individuals'] = {}
shelvecar['functionals'] = [self]
shelvecar['added'] = []
shelvecar['removed'] = []
shelvecar['new'] = []
finally: shelvecar.close()
else:
try:
shelvecar = shelve_open(self.shelvepath, writeback=True)
previous = shelvecar['functionals']
finally: shelvecar.close()
self.offspring = previous[-1].offspring
self.population = previous[-1].population
self.errors = previous[-1].errors
self.generation = previous[-1].generation
def __init__(self, name='sid', dir=path_join(WORK_DIR, 'sessions'),
path=None, domain=None, max_age=None):
self._name = name
now = datetime.utcnow();
# blank cookie
self._cookie = SimpleCookie()
if environ.has_key('HTTP_COOKIE'):
# cookie already exists, see what's in it
self._cookie.load(environ['HTTP_COOKIE'])
try:
# what's our session ID?
self.sid = self._cookie[name].value;
except KeyError:
# there isn't any, make a new session ID
remote = environ.get('REMOTE_ADDR')
self.sid = sha224('%s-%s' % (remote, now)).hexdigest()
self._cookie.clear();
self._cookie[name] = self.sid
# set/reset path
if path:
self._cookie[name]['path'] = path
else:
self._cookie[name]['path'] = ''
# set/reset domain
if domain:
self._cookie[name]['domain'] = domain
else:
self._cookie[name]['domain'] = ''
# set/reset expiration date
if max_age:
if isinstance(max_age, int):
max_age = timedelta(seconds=max_age)
expires = now + max_age
self._cookie[name]['expires'] = expires.strftime('%a, %d %b %Y %H:%M:%S')
else:
self._cookie[name]['expires'] = ''
# to protect against cookie-stealing JS, make our cookie
# available only to the browser, and not to any scripts
try:
# This will not work for Python 2.5 and older
self._cookie[name]['httponly'] = True
except CookieError:
pass
# if the sessions dir doesn't exist, create it
if not exists(dir):
mkdir(dir)
# persist the session data
self._shelf_file = path_join(dir, self.sid)
# -1 signifies the highest available protocol version
self._shelf = shelve_open(self._shelf_file, protocol=-1, writeback=True)
def __init__(self, name='sid', dir=path_join(WORK_DIR, 'sessions'),
path=None, domain=None, max_age=None):
self._name = name
now = datetime.utcnow();
# blank cookie
self._cookie = SimpleCookie()
if environ.has_key('HTTP_COOKIE'):
# cookie already exists, see what's in it
self._cookie.load(environ['HTTP_COOKIE'])
try:
# what's our session ID?
self.sid = self._cookie[name].value;
except KeyError:
# there isn't any, make a new session ID
remote = environ.get('REMOTE_ADDR')
self.sid = sha224('%s-%s' % (remote, now)).hexdigest()
self._cookie.clear();
self._cookie[name] = self.sid
# set/reset path
if path:
self._cookie[name]['path'] = path
else:
self._cookie[name]['path'] = ''
# set/reset domain
if domain:
self._cookie[name]['domain'] = domain
else:
self._cookie[name]['domain'] = ''
# set/reset expiration date
if max_age:
if isinstance(max_age, int):
max_age = timedelta(seconds=max_age)
expires = now + max_age
self._cookie[name]['expires'] = expires.strftime('%a, %d %b %Y %H:%M:%S')
else:
self._cookie[name]['expires'] = ''
# to protect against cookie-stealing JS, make our cookie
# available only to the browser, and not to any scripts
try:
# This will not work for Python 2.5 and older
self._cookie[name]['httponly'] = True
except CookieError:
pass
# if the sessions dir doesn't exist, create it
if not exists(dir):
mkdir(dir)
# persist the session data
self._shelf_file = path_join(dir, self.sid)
# -1 signifies the highest available protocol version
self._shelf = shelve_open(self._shelf_file, protocol=-1, writeback=True)
def __init__(self, directory=None):
""" Initializes an extraction object.
:param directory:
Directory where the SHELVECAR can be found.
Defaults to current working directory.
:raises RuntimeError: if no appropriate SHELVECAR can be found.
"""
from os.path import exists, isfile, isdir
from shelve import open as shelve_open
from ..misc import RelativePath, LockFile
super(Extract, self).__init__()
self._directory = RelativePath(directory)
if not exists(self.directory):return
if not isdir(self.directory): return
if not exists(self.shelvepath): return
if not isfile(self.shelvepath): return
with LockFile(self.shelvepath, timeout=10) as lock:
shelve = shelve_open(self.shelvepath)
try:
if set(shelve.keys()) != set(['individuals', 'functionals', 'removed', 'added', 'new']):
raise RuntimeError('{0} is not a GA SHELVECAR file.'.format(self.shelvepath))
finally: shelve.close()
def functionals(self):
""" Functionals at each generation. """
from shelve import open as shelve_open
from ..misc import LockFile
with LockFile(self.shelvepath, timeout=10) as lock:
try:
shelve = shelve_open(self.shelvepath)
return shelve['functionals']
finally: shelve.close()
def individuals(self):
""" Dictionary containing all individuals. """
from shelve import open as shelve_open
from ..misc import LockFile
with LockFile(self.shelvepath, timeout=10) as lock:
try: shelve = shelve_open(self.shelvepath)
except: raise
else: return shelve['individuals']
finally: shelve.close()
def open_db():
lock = get_write_lock()
db = None
try:
db = shelve_open(Config.db)
except:
close_db(db, lock)
raise Exception(EX_UNAVAILABLE, 'Can\'t open database for writing')
return (lock, db)
def update_func(self):
""" Updates functional in SHELVECAR file. """
from shelve import open as shelve_open
from ..misc import LockFile
with LockFile(self.shelvepath, timeout=10) as file:
try:
shelvecar = shelve_open(self.shelvepath, writeback=True)
shelvecar['functionals'][-1] = self
finally: shelvecar.close()
def open_db():
lock = get_write_lock()
db = None
try:
db = shelve_open(Config.db)
except:
close_db(db, lock)
raise Exception(EX_UNAVAILABLE, 'Can\'t open database for writing')
return (lock, db)
def removed(self):
""" Individuals removed at each generation. """
from shelve import open as shelve_open
from ..misc import LockFile
with LockFile(self.shelvepath, timeout=10) as lock:
try:
shelve = shelve_open(self.shelvepath)
result = []
for r in shelve['removed']:
result.extend(shelve['individuals'][i] for i in r)
return result
finally: shelve.close()
def __call__(self, outdir, comm):
""" Performs GA run until exhaustion sets in. """
import sys
from itertools import chain
from os.path import join
from shelve import open as shelve_open
from ..misc import RelativePath, LockFile
# sets ouput directory for this run.
self._directory = RelativePath(outdir).path
# reloads population, offspring, errors, generation from previous run
# this has to come after setting the directory.
self._restart()
# try loops resets old stdout, stderr
oldouterr = sys.stdout, sys.stderr
try:
sys.stdout = open(join(self._directory, "out"), 'a')
sys.stderr = open(join(self._directory, "err"), 'a')
print 'COMMUNICATOR', comm
# first creates a random population if necessary.
if len(self.population) == 0 and len(self.offspring) == 0:
self.new_random_individuals(n=self.target_popsize)
self.update_func()
sys.stdout.flush()
# then creates the jobfolder and start the process.
self.process.start(comm)
assert hasattr(self.process, '_comm')
# finally, loop till eternity comes.
while True:
if not self.go_to_next_iteration(): continue
# Now we create new generation.
# First we get those individuals that were successfuly computed.
successfuls = self.find_successfuls()
# update fitness of whole population, in case it depends upon whole population.
iterator = (self.offspring[i] for i in successfuls.itervalues())
self.update_fitness(chain(iterator, self.population))
# prints out successful individuals.
if len(successfuls) != 0:
print "Newly calculated individuals. """
for index in successfuls.itervalues():
print " {0.uuid}(conception {0.conception}): {0.fitness}"\
.format(self.offspring[index])
sys.stdout.flush()
# removes worst individuals.
removed, population, offspring = self.survival(successfuls)
self.population = population
self.offspring = offspring
# increment generation count.
self.generation += 1
# create new offspring via mating
nboffspring = int(self.target_popsize*self.rate+0.1) - len(offspring) \
+ (self.target_popsize - self.popsize)
self.mating(nboffspring)
# udpate jobfolder
self._update_jobfolder(removed, offspring, population)
# update shelvecar
with LockFile(self.shelvepath, timeout=10) as lock:
try:
shelvecar = shelve_open(self.shelvepath, writeback=True)
individuals = shelvecar['individuals']
for individual in self.population:
if not hasattr(individual, 'birth'):
individual.birth = self.generation
individuals[str(individual.uuid)] = individual
for uuid in removed: individuals[str(uuid)].removed = self.generation
shelvecar['individuals'] = individuals
shelvecar['functionals'].append(self)
shelvecar['removed'].append(removed)
shelvecar['added'].append(successfuls.keys())
shelvecar['new'].append( [ u.uuid for u in self.offspring\
if u.conception == self.generation-1] )
finally: shelvecar.close()
print "\n Starting generation {0}.".format(self.generation)
sys.stdout.flush()
if self.checkpoints(): break
finally:
sys.stdout.close()
sys.stderr.close()
sys.stdout, sys.stderr = oldouterr
if '_process' in self.__dict__:
self._process.terminate()
del self._process
self.__dict__.pop('_directory', None)
self.__dict__.pop('_jobfolder', None)
# returns an extraction object.
# Might never get here, but that's ok.
return self.Extract(outdir)
def __init__(self, *pargs, **kwargs):
"""
Creates ShelveProxy object, passing constructor arguments to shelve.open
"""
self._s = shelve_open(*pargs, **kwargs)
self._cache = {}
