def retrieveMigrants(self):
"""
@description
Retrieve some migrants chosen from the 'pooled_db' file,
if they exist.
@arguments
<<none>>
@return
migrants -- list of Ind -- some migrants
"""
#corner cases: not interested in migration
if self.pooled_db_file is None: return []
if self.ss.migration_rate == 0.0: return []
#corner case: no pooled db file exists
# (though it may exist at other times in the run of this engine)
if not os.path.exists(self.pooled_db_file): return []
#main case...
#try to load candidate migrants
#-this may fail if we are in conflict with another process
# trying to access the pooled DB file. No problem, just
# do it another time
try:
pooled_state = loadSynthState(self.pooled_db_file, self.ps)
cand_migrants = pooled_state.allInds()
except:
log.warning('Could not open pooled_db file for migration')
return []
#choose num_migrants based on num_inds_per_age_layer and migration_rate
N = self.ss.num_inds_per_age_layer
min_num_migrants = 1
max_num_migrants = N / 2
num_migrants = int(self.ss.migration_rate * N)
num_migrants = max(min_num_migrants, min(max_num_migrants,
num_migrants))
#unique-ify cand_migrants
cand_migrants = uniqueIndsByPerformance(cand_migrants)
#choose migrants
if len(cand_migrants) <= num_migrants:
migrants = cand_migrants
else:
migrants = random.sample(cand_migrants, num_migrants)
log.info('From %d candidate migrants, retrieving %d for pop' %
(len(cand_migrants), len(migrants)))
return migrants
def retrieveMigrants(self):
"""
@description
Retrieve some migrants chosen from the 'pooled_db' file,
if they exist.
@arguments
<<none>>
@return
migrants -- list of Ind -- some migrants
"""
#corner cases: not interested in migration
if self.pooled_db_file is None: return []
if self.ss.migration_rate == 0.0: return []
#corner case: no pooled db file exists
# (though it may exist at other times in the run of this engine)
if not os.path.exists(self.pooled_db_file): return []
#main case...
#try to load candidate migrants
#-this may fail if we are in conflict with another process
# trying to access the pooled DB file. No problem, just
# do it another time
try:
pooled_state = loadSynthState(self.pooled_db_file, self.ps)
cand_migrants = pooled_state.allInds()
except:
log.warning('Could not open pooled_db file for migration')
return []
#choose num_migrants based on num_inds_per_age_layer and migration_rate
N = self.ss.num_inds_per_age_layer
min_num_migrants = 1
max_num_migrants = N / 2
num_migrants = int(self.ss.migration_rate * N)
num_migrants = max(min_num_migrants,min(max_num_migrants,num_migrants))
#unique-ify cand_migrants
cand_migrants = uniqueIndsByPerformance(cand_migrants)
#choose migrants
if len(cand_migrants) <= num_migrants:
migrants = cand_migrants
else:
migrants = random.sample(cand_migrants, num_migrants)
log.info('From %d candidate migrants, retrieving %d for pop' %
(len(cand_migrants), len(migrants)))
return migrants
def __init__(self, ps, ss, output_dir, pooled_db_file, restart_file):
#initialize state, possibly from previous run
if restart_file is None:
self.state = SynthState(ps, ss, AgeLayeredPop())
else:
restart_file = os.path.abspath(restart_file)
try:
self.state = loadSynthState(restart_file, ps)
except:
log.error('Could not open restart_file=%s; quitting' %
restart_file)
sys.exit(0)
#clear up output directory
self.output_dir = os.path.abspath(output_dir)
if self.output_dir[-1] != '/':
self.output_dir += '/'
if os.path.exists(self.output_dir):
log.warning("Output path '%s' already exists; will rewrite" %
self.output_dir)
shutil.rmtree(self.output_dir)
os.mkdir(self.output_dir)
#if we had a restart file, we can ensure that its info wasn't lost
# due to clearing up the output directory
if restart_file is not None and not os.path.exists(restart_file):
self.saveState()
self.simfile_dir = self.output_dir + 'autogen_simfiles/'
os.mkdir(self.simfile_dir)
assert 0.0 <= ss.migration_rate <= 0.5, \
"migration rate must be in [0.0, 0.5]"
if pooled_db_file is None:
self.pooled_db_file = None
else:
self.pooled_db_file = os.path.abspath(pooled_db_file)
def __init__(self, ps, ss, output_dir, pooled_db_file, restart_file):
#initialize state, possibly from previous run
if restart_file is None:
self.state = SynthState(ps, ss, AgeLayeredPop())
else:
restart_file = os.path.abspath(restart_file)
try:
self.state = loadSynthState(restart_file, ps)
except:
log.error('Could not open restart_file=%s; quitting' %
restart_file)
sys.exit(0)
#clear up output directory
self.output_dir = os.path.abspath(output_dir)
if self.output_dir[-1] != '/':
self.output_dir += '/'
if os.path.exists(self.output_dir):
log.warning("Output path '%s' already exists; will rewrite" %
self.output_dir)
shutil.rmtree(self.output_dir, ignore_errors=True)
os.mkdir(self.output_dir)
#if we had a restart file, we can ensure that its info wasn't lost
# due to clearing up the output directory
if restart_file is not None and not os.path.exists(restart_file):
self.saveState()
self.simfile_dir = self.output_dir + 'autogen_simfiles/'
os.mkdir(self.simfile_dir)
assert 0.0 <= ss.migration_rate <= 0.5, \
"migration rate must be in [0.0, 0.5]"
if pooled_db_file is None:
self.pooled_db_file = None
else:
self.pooled_db_file = os.path.abspath(pooled_db_file)
def run__OneIteration(self):
"""Run one iteration of Pooler"""
log.info('======================================================')
log.info('Begin new Pooler iteration')
#1. read db_dirs_file to get latest set of dbs to go for
f = open(self.db_dirs_file, 'r')
lines_list = f.readlines()
f.close()
db_dirs = string.join(lines_list).split()
log.info('Will try to read from these db_dirs: %s' % db_dirs)
#2. read each db in (it's ok if a db doesn't exist)
#3. pool all the db results into one big db
all_inds = []
num_dbs_found = 0
synth_ss = None #set this using the first decent db we find
for db_dir in db_dirs:
#try to retrieve a db file; catch possible problems.
if len(db_dir) > 0 and db_dir[-1] != '/':
db_dir = db_dir + '/'
log.info('Try retrieving from db_dir=%s ...' % db_dir)
if not os.path.exists(db_dir):
log.info(' Could not find db_dir, so ignoring it')
continue
db_file = self._newestStateGenFile(db_dir)
if db_file is None:
log.info(' No relevant dbs in db_dir yet')
continue
try:
synth_state = loadSynthState(db_file, self.ps)
except:
log.info(' Could not loadSynthState from db_file=%s' %
db_file)
continue
#retrieved a db file, so add to all_inds
num_dbs_found += 1
inds = synth_state.allInds()
log.info(' Found %d inds in %s' % (len(inds), db_file))
max_num_inds_from_db = max(1, self.ss.max_pool_size / 3)
if synth_ss is None:
synth_ss = synth_state.ss
inds_from_db = self._bestInds(inds, max_num_inds_from_db,
synth_ss.metric_weights)
all_inds.extend(inds_from_db)
log.info(' Added %d of those inds to pooled_inds (pruned some)' %
len(inds_from_db))
# -no info to work with, so try later
log.info('Num dbs found = %d' % num_dbs_found)
if num_dbs_found == 0:
time.sleep(max(self.ss.loop_wait_time, 60))
return
#4. save the pooled file
# -DO prune it down to the best inds
log.info('Prune down pooled_inds via fast nondominated sort...')
target_num_best = self.ss.max_pool_size
pooled_inds = self._bestInds(all_inds, target_num_best,
synth_ss.metric_weights)
log.info('Done prune')
synth_state = SynthState(self.ps, synth_ss,
AgeLayeredPop([pooled_inds]))
# -we may have trouble saving to the file if another process
# is accessing it, so just keep retrying until good
while True:
log.info('Try saving %d pruned-pooled inds to file %s' %
(len(pooled_inds), self.pooled_db_file))
try:
synth_state.save(self.pooled_db_file)
log.info('Successfully saved')
break
except:
time.sleep(3)
pass
#5. wait a while
log.info('Done pooler iteration; pause for %s seconds' %
self.ss.loop_wait_time)
time.sleep(self.ss.loop_wait_time)
def run__OneIteration(self):
"""Run one iteration of Pooler"""
log.info('======================================================')
log.info('Begin new Pooler iteration')
#1. read db_dirs_file to get latest set of dbs to go for
f = open(self.db_dirs_file, 'r')
lines_list = f.readlines()
f.close()
db_dirs = string.join(lines_list).split()
log.info('Will try to read from these db_dirs: %s' % db_dirs)
#2. read each db in (it's ok if a db doesn't exist)
#3. pool all the db results into one big db
all_inds = []
num_dbs_found = 0
synth_ss = None #set this using the first decent db we find
for db_dir in db_dirs:
#try to retrieve a db file; catch possible problems.
if len(db_dir)>0 and db_dir[-1] != '/':
db_dir = db_dir + '/'
log.info('Try retrieving from db_dir=%s ...' % db_dir)
if not os.path.exists(db_dir):
log.info(' Could not find db_dir, so ignoring it')
continue
db_file = self._newestStateGenFile(db_dir)
if db_file is None:
log.info(' No relevant dbs in db_dir yet')
continue
try:
synth_state = loadSynthState(db_file, self.ps)
except:
log.info(' Could not loadSynthState from db_file=%s' % db_file)
continue
#retrieved a db file, so add to all_inds
num_dbs_found += 1
inds = synth_state.allInds()
log.info(' Found %d inds in %s' % (len(inds), db_file))
max_num_inds_from_db = max(1, self.ss.max_pool_size / 3)
if synth_ss is None:
synth_ss = synth_state.ss
inds_from_db = self._bestInds(inds, max_num_inds_from_db,
synth_ss.metric_weights)
all_inds.extend(inds_from_db)
log.info(' Added %d of those inds to pooled_inds (pruned some)' %
len(inds_from_db))
# -no info to work with, so try later
log.info('Num dbs found = %d' % num_dbs_found)
if num_dbs_found == 0:
time.sleep(max(self.ss.loop_wait_time, 60))
return
#4. save the pooled file
# -DO prune it down to the best inds
log.info('Prune down pooled_inds via fast nondominated sort...')
target_num_best = self.ss.max_pool_size
pooled_inds = self._bestInds(all_inds, target_num_best,
synth_ss.metric_weights)
log.info('Done prune')
synth_state = SynthState(self.ps, synth_ss,
AgeLayeredPop([pooled_inds]))
# -we may have trouble saving to the file if another process
# is accessing it, so just keep retrying until good
while True:
log.info('Try saving %d pruned-pooled inds to file %s' %
(len(pooled_inds), self.pooled_db_file))
try:
synth_state.save(self.pooled_db_file)
log.info('Successfully saved')
break
except:
time.sleep(3)
pass
#5. wait a while
log.info('Done pooler iteration; pause for %s seconds' %
self.ss.loop_wait_time)
time.sleep(self.ss.loop_wait_time)