def main():
redis_conn = Redis(decode_responses=True)
use_connection(redis_conn)
q = Queue()
sql_con, sql_cur = setup_db()
conns = Connections(q, sql_con, sql_cur)
cohort_size = int(input("Enter cohort size: "))
gen = int(input("Enter current generation (0 to start from scratch): "))
if gen == 0:
gen = 1
cohort = evolution.first_generation(cohort_size)
queue_cohort(conns, gen, cohort)
while True:
wait_for_gen(conns, gen)
old_cohort = fetch_results_from_gen(conns, gen)
highscore = max(i[1] for i in old_cohort)
logger.info("Generation complete. Max score: %.2f", highscore)
if gen % 10 == 0:
send_message(
"Generation Complete",
"Gen {0}, max score {1:.2f}".format(gen, highscore))
new_cohort = evolution.new_generation(old_cohort)
members = len(new_cohort)
if members < cohort_size:
# timeout or fails
new_cohort += evolution.first_generation(cohort_size - members)
gen += 1
logger.info("Enqueueing tasks for generation %d...", gen)
queue_cohort(conns, gen, new_cohort)
logger.info("Tasks queued, waiting for completion...")
def wait_for_gen(conns, gen):
logger.info("Waiting for generation %d...", gen)
jobs = fetch_jobs_from_gen(conns, gen)
saved_jobs = []
t0 = time.time()
old_finished = 0
while True:
finished = 0
failed = 0
started = 0
queued = 0
for job in jobs:
if job.is_finished:
if job not in saved_jobs:
save_score(conns, job._id, job.result)
saved_jobs.append(job)
finished += 1
elif job.is_failed:
failed += 1
elif job.is_started:
started += 1
elif job.is_queued:
queued += 1
logger.info("generation %d: %d finished, %d started, "
"%d queued, %d failed jobs",
gen, finished, started, queued, failed)
if finished == len(jobs):
logger.info("All jobs finished, proceeding...")
return
elif finished + failed == len(jobs):
logger.warn("All jobs finished or failed, proceeding...")
send_message("Failures", "Generation {0}".format(gen))
return
else:
if old_finished != finished:
t0 = time.time()
elif time.time() - t0 > 60:
logger.warn("It's been a minute with no progress! "
"Moving on with what we've got.")
send_message("Timeout", "Generation {0}".format(gen))
return
old_finished = finished
time.sleep(5)