def main():
parser = argparse.ArgumentParser(description='Proces trail on an elite.')
parser.add_argument('run_id',
type=int,
help='The run ID to fetch animations for.')
parser.add_argument('-g',
'--generation',
type=int,
default=None,
help='The generation to evaluate elite for.')
parser.add_argument('--no-shorten',
action='store_true',
help='Disables URL shortening with Google.')
parser.add_argument('--debug',
action='store_true',
help='Enables debug information.')
args = parser.parse_args()
# Read the configuration file.
with open(CONFIG_FILE) as fh:
config = json.load(fh)
pgdb = DBUtils(CONFIG_FILE)
elite = pgdb.get_elite(args.run_id, args.generation)
run_info = pgdb.get_run_info(args.run_id)
moves = eval_trail(elite, run_info["moves_limit"], run_info["networks_id"],
run_info["trails_id"])
actual_url = BASE_URL.format(config["base-viewer-url"],
run_info["trails_id"],
"".join([str(x) for x in moves]))
if args.debug:
print "DEBUG: Base URL is {0}.".format(actual_url)
if not args.no_shorten:
print_url = shorten_url(actual_url, config["url-shorten-key"],
args.debug)
else:
print_url = actual_url
print "Animation URL: {0}".format(print_url)
def main():
parser = argparse.ArgumentParser(
description='Proces trail on an elite.')
parser.add_argument('run_id', type=int,
help='The run ID to fetch animations for.')
parser.add_argument('-g', '--generation', type=int,
default=None,
help='The generation to evaluate elite for.')
parser.add_argument('--no-shorten', action='store_true',
help='Disables URL shortening with Google.')
parser.add_argument('--debug', action='store_true',
help='Enables debug information.')
args = parser.parse_args()
# Read the configuration file.
with open(CONFIG_FILE) as fh:
config = json.load(fh)
pgdb = DBUtils(CONFIG_FILE)
elite = pgdb.get_elite(args.run_id, args.generation)
run_info = pgdb.get_run_info(args.run_id)
moves = eval_trail(elite, run_info["moves_limit"],
run_info["networks_id"], run_info["trails_id"])
actual_url = BASE_URL.format(
config["base-viewer-url"],
run_info["trails_id"],
"".join([str(x) for x in moves]))
if args.debug:
print "DEBUG: Base URL is {0}.".format(actual_url)
if not args.no_shorten:
print_url = shorten_url(actual_url, config["url-shorten-key"],
args.debug)
else:
print_url = actual_url
print "Animation URL: {0}".format(print_url)
SWEEP_BUTTON_STR = [
"Delay Line Length", "Hidden Perceptron Count",
"DL Len / Hidden Perceptron", "P(Mutate)", "P(Crossover)", "Selection",
"Moves Limit", "Population", "Generations", "P(Mutate)/P(Crossover)",
"Tournament Size", "Lambda Size"
]
VALID_PIL_EXTENSION = [
"bmp", "eps", "gif", "jpg", "jpeg", "png", "pdf", "tiff", "tif"
]
PIL_DROPDOWN_LIST = ["bmp", "eps", "gif", "jpg", "png", "pdf", "tif"]
app = Flask(__name__)
pgdb = DBUtils()
# Enable logging
if not app.debug:
import logging
from logging.handlers import RotatingFileHandler
file_handler = RotatingFileHandler("errors.log", backupCount=5)
file_handler.setLevel(logging.WARNING)
app.logger.addHandler(file_handler)
### Begin Helper Functions ###
def str_to_bool(s_in):
""" Checks if a string is "true" or "True" and resturns True if so,
False otherwise as a bool.
"""
def main(args):
run_date = time.time()
# Configure Logging
root = logging.getLogger()
if(args.debug):
root.setLevel(logging.DEBUG)
else:
root.setLevel(logging.INFO)
if args.quiet:
root.propogate = False
# Set up the database.
pgdb = DBUtils(config_file=DB_CONFIG_FILE)
# Get the name of this agent trail for later use
at = AgentTrail()
at.readTrail(args.trail, DB_CONFIG_FILE)
trail_name = at.getName()
if not args.quiet and not args.debug and not args.script_mode:
try:
TOTAL_GENERATIONS = (len(args.network) *
args.generations * args.repeat)
widgets = ['Processed: ', progressbar.Percentage(), ' ',
progressbar.Bar(marker=progressbar.RotatingMarker()),
' ', progressbar.ETA()]
pbar = progressbar.ProgressBar(
widgets=widgets,
maxval=TOTAL_GENERATIONS).start()
except:
pbar = None
else:
pbar = None
current_overall_gen = 0
for curr_network in args.network:
# Query the database to get the network information.
pybrain_network = pgdb.getNetworkByID(curr_network)
temp_f_h, temp_f_network = tempfile.mkstemp()
os.close(temp_f_h)
with open(temp_f_network, "w") as f:
pickle.dump(pybrain_network, f)
# TODO: Need to fix this for chemistry support here.
if "Chemical" in pybrain_network.name:
chem_re = re.compile(
"JL NN Chemical DL([0-9]+) \([0-9]+,[0-9]+,[0-9]+\) v[0-9]+")
chem_dl_length = int(chem_re.findall(pybrain_network.name)[0])
network_params_len = len(pybrain_network.params) + chem_dl_length * 3
else:
network_params_len = len(pybrain_network.params)
# Query the database to get the trail information.
(data_matrix,
db_trail_name,
init_rot) = pgdb.getTrailData(args.trail)
# Calculate the maximum amount of food for potential later comparison.
MAX_FOOD = np.bincount(np.array(data_matrix).flatten())[1]
for curr_repeat in range(0, args.repeat):
repeat_start_time = datetime.datetime.now()
gens_stat_list = [None] * args.generations
# Create an empty array to store the launches for SCOOP.
launches = []
# Prepare the array for storing hall of fame.
hof_array = np.zeros((args.generations,
network_params_len))
toolbox = base.Toolbox()
toolbox.register("map", scoop.futures.map)
toolbox.register("attr_float", random.uniform,
a=args.weight_min, b=args.weight_max)
toolbox.register("individual", tools.initRepeat, creator.Individual,
toolbox.attr_float, n=network_params_len)
toolbox.register("population", tools.initRepeat, list,
toolbox.individual)
an_temp = AgentNetwork()
an_temp.readNetworkFromFile(temp_f_network)
at_temp = AgentTrail()
at_temp.readTrailInstant(data_matrix, db_trail_name, init_rot)
toolbox.register("evaluate", __singleMazeTask, moves=args.moves,
network=pickle.dumps(an_temp), trail=pickle.dumps(at_temp))
toolbox.register("mate", tools.cxTwoPoint)
if args.mutate_type == 1:
toolbox.register("mutate",
tools.mutFlipBit,
indpb=P_BIT_MUTATE)
elif args.mutate_type == 2:
toolbox.register("mutate",
mutUniformFloat,
low=args.weight_min,
up=args.weight_max,
indpb=P_BIT_MUTATE)
elif args.mutate_type == 3:
toolbox.register("mutate",
mutUniformFloat,
low=args.weight_min,
up=args.weight_max,
indpb=0.30)
elif args.mutate_type == 4:
toolbox.register("mutate",
mutUniformFloat,
low=args.weight_min,
up=args.weight_max,
indpb=0.10)
elif args.mutate_type == 5:
toolbox.register("mutate",
tools.mutGaussian,
mu=0,
indpb=0.05)
else:
print "ERROR: Please selct a valid mutate type!"
sys.exit(10)
if args.selection == 1:
# Selection is tournment. Must use argument from user.
toolbox.register("select", tools.selTournament,
tournsize=args.tournament_size)
elif args.selection == 2:
toolbox.register("select", tools.selRoulette)
elif args.selection == 3:
toolbox.register("select", tools.selNSGA2)
elif args.selection == 4:
toolbox.register("select", tools.selSPEA2)
elif args.selection == 5:
toolbox.register("select", tools.selRandom)
elif args.selection == 6:
toolbox.register("select", tools.selBest)
elif args.selection == 7:
toolbox.register("select", tools.selWorst)
elif args.selection == 8:
toolbox.register("select", tools.selTournamentDCD)
else:
print "ERROR: Something is wrong with selection method!"
sys.exit(10)
# Start a new evolution
population = toolbox.population(n=args.population)
halloffame = tools.HallOfFame(maxsize=1)
food_stats = tools.Statistics(key=lambda ind: ind.fitness.values[0])
move_stats = tools.Statistics(key=lambda ind: ind.fitness.values[1])
mstats = tools.MultiStatistics(food=food_stats, moves=move_stats)
mstats.register("min", np.min)
mstats.register("avg", np.mean)
mstats.register("max", np.max)
mstats.register("std", np.std)
mstats.register("mode", mode)
# Record the start of this run.
log_time = datetime.datetime.now()
# Evaluate and record the first generation here.
invalid_ind = [ind for ind in population if not ind.fitness.valid]
fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
# Determine the current generations statistics.
record = mstats.compile(population)
if args.debug:
print "DEBUG: Completed generation 1"
hof_indiv = np.array(tools.selBest(population, k=1)[0])
hof_array[0] = hof_indiv
# Add the hall of fame to launches.
launches.append(
scoop.futures.submit(__singleMazeTask,
hof_indiv,
args.moves,
pickle.dumps(an_temp),
pickle.dumps(at_temp),
1,
record)
)
# Keep track of the average food history.
mean_food_history = []
smart_term_msg = ""
# Begin the generational process
for gen in range(2, args.generations + 1):
# Vary the pool of individuals
if args.variation in [1]:
offspring = algorithms.varAnd(population, toolbox,
cxpb=args.prob_crossover, mutpb=args.prob_mutate)
elif args.variation in [2, 3, 4]:
offspring = algorithms.varOr(population, toolbox,
lambda_=args.lambda_,
cxpb=args.prob_crossover, mutpb=args.prob_mutate)
elif args.variation in [5]:
# Take and modify the varAnd from DEAP.
offspring = [toolbox.clone(ind) for ind in population]
# Apply crossover and mutation on the offspring
for i in range(1, len(offspring), 2):
if random.random() < args.prob_crossover:
offspring[i-1], offspring[i] = toolbox.mate(
offspring[i-1], offspring[i])
del (offspring[i-1].fitness.values,
offspring[i].fitness.values)
for i in range(len(offspring)):
if random.random() < args.prob_mutate:
if args.mutate_type in [5]:
offspring[i], = toolbox.mutate(
offspring[i],
sigma=np.std(offspring[i]))
else:
offspring[i], = toolbox.mutate(
offspring[i], offspring[i])
del offspring[i].fitness.values
else:
print ("ERROR: Something is really wrong! " +
"Reached an invalid variation type!")
sys.exit(5)
# Evaluate the individuals with an invalid fitness
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
# Update the hall of fame with the generated individuals
if halloffame is not None:
halloffame.update(offspring)
# Replace the current population by the offspring
if args.variation in [2, 3]:
population[:] = toolbox.select(offspring, args.population)
elif args.variation in [4, 5]:
population[:] = toolbox.select(offspring + population,
args.population)
else:
population[:] = offspring
# Determine the current generations statistics.
record = mstats.compile(population)
if args.debug:
print "DEBUG: Completed generation {0}.".format(gen)
print (
"DEBUG: Food (Min / Max / Avg / Std / Mode): "
"{0} / {1} / {2} / {3} / {4}".format(
record["food"]["min"],
record["food"]["max"],
record["food"]["avg"],
record["food"]["std"],
record["food"]["mode"]))
print (
"DEBUG: Moves (Min / Max / Avg / Std / Mode): "
"{0} / {1} / {2} / {3} / {4}".format(
record["moves"]["min"],
record["moves"]["max"],
record["moves"]["avg"],
record["moves"]["std"],
record["moves"]["mode"]))
hof_indiv = np.array(tools.selBest(population, k=1)[0])
hof_array[gen - 1] = hof_indiv
# Add the hall of fame to launches.
launches.append(
scoop.futures.submit(__singleMazeTask,
hof_indiv,
args.moves,
pickle.dumps(an_temp),
pickle.dumps(at_temp),
gen,
record)
)
# Update the mean food history.
mean_food_history.append(record["food"]["avg"])
# Update the progress bar
if pbar:
current_overall_gen += 1
pbar.update(current_overall_gen)
# Check if it is time to quit if variation is 3. Critera are
# any of the following:
# 1) All food has been collected.
# 2) Mean has not changed for args.mean_check_length
# 3) Run out of generations (happens without this if)
if args.variation in [3, 4, 5] and not args.no_early_quit:
if (int(record["food"]["max"]) == int(MAX_FOOD)):
smart_term_msg = ("Exited at generation {0} because "
"all food was consumed.").format(gen)
break
elif(len(mean_food_history) >= args.mean_check_length and
(np.std(mean_food_history[-args.mean_check_length:])
< 0.1)):
smart_term_msg = ("Exited at generation {0} because "
"mean check length has been met.").format(gen)
break
# Evaluate the Hall of Fame individual for each generation here
# in a multithreaded fashion to speed things up.
for this_future in scoop.futures.as_completed(launches):
result = this_future.result()
gens_stat_list[result[0] - 1] = result[1]
# Remove all of the None values from the gen_stat_list
gens_stat_list = filter(lambda a: a is not None, gens_stat_list)
# Record the statistics on this run.
run_info = {}
run_info["trails_id"] = args.trail
run_info["networks_id"] = curr_network
run_info["selection_id"] = args.selection
run_info["mutate_id"] = args.mutate_type
run_info["host_type_id"] = 1 # Only one host type for now.
run_info["variations_id"] = args.variation
run_info["run_date"] = log_time
run_info["hostname"] = socket.getfqdn()
run_info["generations"] = args.generations
run_info["population"] = args.population
run_info["moves_limit"] = args.moves
run_info["sel_tourn_size"] = args.tournament_size
if args.variation in [1, 5]:
run_info["lambda"] = 0
else:
run_info["lambda"] = args.lambda_
run_info["p_mutate"] = args.prob_mutate
run_info["p_crossover"] = args.prob_crossover
run_info["weight_min"] = args.weight_min
run_info["weight_max"] = args.weight_max
run_info["debug"] = args.debug
# Version for if anything changes in python GA Algorithm
run_info["algorithm_ver"] = 2
run_info["mean_check_length"] = args.mean_check_length
run_info["runtime"] = (datetime.datetime.now() -
repeat_start_time)
if not args.disable_db:
run_id = pgdb.recordRun(run_info, gens_stat_list)
else:
run_id = -1
if args.script_mode:
if run_id > 0:
print (
"Completed repeat {0} with run ID {1}. {2}".format(
curr_repeat,
run_id,
smart_term_msg
))
else:
print (
"Completed repeat {0} without logging to DB. {1}".format(
curr_repeat,
smart_term_msg
))
# Delete the temporary file
os.remove(temp_f_network)
# Calculate and display the total runtime
if pbar:
pbar.finish()
total_time_s = time.time() - run_date
if run_id > 0:
print "Final Run ID {0} completed all runs in {1}. {2}".format(
run_id,
time.strftime('%H:%M:%S', time.gmtime(total_time_s)),
smart_term_msg)
else:
print "UNLOGGED Run completed in {0}. {1}".format(
time.strftime('%H:%M:%S', time.gmtime(total_time_s)),
smart_term_msg)