def distrib_workers(population, gen, worker_pop_size, num_survive, advice, BD_table, biases, configs):
num_workers = int(configs['number_of_workers'])
output_dir = configs['output_directory']
debug = util.boool(configs['debug'])
if (debug == True): # sequential debug
for w in range(1, num_workers + 1):
dump_file = output_dir + "to_workers/" + str(gen) + "/" + str(w)
seed = population[0].copy()
randSeeds = os.urandom(sysRand().randint(0, 1000000))
worker_args = [w, seed, worker_pop_size, min(worker_pop_size, num_survive), randSeeds, advice, BD_table, biases, configs]
with open(dump_file, 'wb') as file:
pickle.dump(worker_args, file)
# pool.map_async(minion.evolve_minion, (dump_file,))
minion.evolve_minion(dump_file, gen, w, output_dir)
sleep(.0001)
else:
for w in range(1, num_workers + 1):
dump_file = output_dir + "/to_workers/" + str(gen) + "/" + str(w)
seed = population[w % num_survive].copy()
randSeeds = os.urandom(sysRand().randint(0, 1000000))
assert (seed != population[w % num_survive])
worker_args = [w, seed, worker_pop_size, min(worker_pop_size, num_survive), randSeeds, advice, BD_table, biases, configs]
with open(dump_file, 'wb') as file:
pickle.dump(worker_args, file)
del population
if (debug == True): util.cluster_print(output_dir, "debug is ON")
def report_timing(t_start, rank, gen, output_dir):
t_end = time.time()
time_elapsed = t_end - t_start
if ((rank == 1 or rank == 63) and gen % 100 == 0):
util.cluster_print(
output_dir, "Worker #" + str(rank) + " finishing after " +
str(time_elapsed) + " seconds")
def watch(configs, gen, num_workers, output_dir, num_survive, fitness_direction):
dump_dir = output_dir + "/to_master/" + str(gen)
t_start = time.time()
popn, num_finished, dir_checks = [], 0,0
ids = [str(i) for i in range(1, num_workers + 1)]
while (num_finished < num_workers):
time.sleep(1)
dir_checks+=1
for root, dirs, files in os.walk(dump_dir):
for f in files:
if f in ids:
if (os.path.getmtime(root + "/" + f) + 1 < time.time()):
dump_file = output_dir + "/to_master/" + str(gen) + "/" + str(f)
with open(dump_file, 'rb') as file:
try:
worker_pop = pickle.load(file)
popn += worker_pop[:num_survive]
num_finished += 1
ids.remove(f)
except: pass
#sort and delete some
sorted_popn = fitness.eval_fitness(popn, fitness_direction)
popn = sorted_popn[:num_survive]
del sorted_popn
assert (not ids)
t_end = time.time()
time_elapsed = t_end - t_start
if (gen % 100 == 0): util.cluster_print(output_dir,"master finished extracting workers after " + str(time_elapsed) + " seconds, and making " + str(dir_checks) + " dir checks.")
return popn
def init_sim(configs, num_sims, sim_num, orig_output_dir):
if (num_sims > 1 and sim_num == 0): # check where to pick up the run
this_dir = False
while (not this_dir):
if (sim_num >= num_sims):
util.cluster_print(
orig_output_dir,
"All simulations already finished, exiting...\n")
return
configs['output_directory'] = orig_output_dir + "_" + str(i)
this_dir = True # remains true if any of the following fail
if os.path.exists(configs['output_directory'] + "/progress.txt"):
with open(configs['output_directory'] +
"/progress.txt") as progress:
line = progress.readline()
if (line.strip() == 'Done' or line.strip() == 'done'):
this_dir = False
sim_num += 1
if (num_sims > 1):
configs['output_directory'] = orig_output_dir + "sim_" + str(
sim_num) + "/"
configs['instance_file'] = (util.slash(configs['output_directory']) +
"/instances/" + configs['stamp'])
if (rank == 0):
if not os.path.exists(configs['output_directory']):
os.makedirs(configs['output_directory'])
else:
while (not os.path.exists(configs['output_directory'])):
sleep(1)
def evolve(rank, num_workers, config_file):
configs = init.load_sim_configs(config_file, rank, num_workers)
orig_output_dir = configs['output_directory']
num_sims = int(configs['num_sims'])
for i in range(num_sims):
err = init_sim(configs, num_sims, i, orig_output_dir, rank)
#WARNING: this area might need more work, esp if mult isms
if rank == 0 and not err: # MASTER
log_text = 'Evolve_root(): in dir ' + str(
os.getcwd()) + ', config file = ' + str(
config_file) + ', num_workers = ' + str(num_workers) + "\n"
import master
util.cluster_print(configs['output_directory'], log_text)
master.evolve_master(configs)
elif not err: # WORKERS
import minion
minion.work(configs, rank)
if (num_sims > 1 and rank == 0):
close_out_mult_sims(num_sims, orig_output_dir)
def master_info(population, gen, size, pop_size, num_survive, advice, BD_table, configs):
output_dir = configs['output_directory']
num_data_output = int(configs['num_data_output'])
num_net_output = int(configs['num_net_output'])
num_instance_output = int(configs['num_instance_output'])
instance_file = configs['instance_file']
if (num_instance_output==0): instance_file = None
stop_condition = configs['stop_condition']
if (stop_condition == 'size'):
end = int(configs['ending_size'])
#this sort of assumes simulation starts near size 0
elif (stop_condition == 'generation'): end = int(configs['max_generations'])
else: assert(False)
if (num_data_output > 0):
if (gen % int(end / num_data_output) == 0):
popn_data(population, output_dir, gen)
util.cluster_print(output_dir, "Master at gen " + str(gen) + ", with net size = " + str(size) + " nodes and " + str(len(population[0].net.edges())) + " edges, " + str(num_survive) + "<=" + str(len(population)) + " survive out of " + str(pop_size))
nx.write_edgelist(population[0].net, output_dir + "/fittest_net.edgelist")
if (num_instance_output != 0):
if (gen % int(end / num_instance_output) == 0):
# if first gen, have already pressurized w/net[0]
if (gen != 0): pressurize.pressurize(configs, population[0], instance_file + "Xitern" + str(gen) + ".csv", advice, BD_table)
if (num_net_output > 0):
if (gen % int(end / num_net_output) == 0):
nx.write_edgelist(population[0].net, output_dir + "/nets/" + str(gen))
pickle_file = output_dir + "/pickle_nets/" + str(gen) + "_pickle"
with open(pickle_file, 'wb') as file:
pickle.dump(population[0].net, file)
deg_distrib_csv(output_dir, population, gen)
def add_this_edge(net,
configs,
node1=None,
node2=None,
sign=None,
random_direction=False,
given_bias=None):
reverse_allowed = util.boool(configs['reverse_edges_allowed'])
bias_on = configs['bias_on']
node1_set, node2_set = node1, node2 #to save their states
if not sign:
sign = rd.randint(0, 1)
if (sign == 0): sign = -1
pre_size = post_size = len(net.edges())
i = 0
while (pre_size == post_size): # ensure that net adds
if not node1_set and node1_set != 0:
node = rd.sample(net.nodes(), 1)
node1 = node[0]
if not node2_set and node2_set != 0:
node2 = node1
while (node2 == node1):
node2 = rd.sample(net.nodes(), 1)
node2 = node2[0]
if random_direction: #chance to swap nodes 1 & 2
if (rd.random() < .5):
node3 = node2
node2 = node1
node1 = node3
if reverse_allowed:
if not net.has_edge(node1, node2):
net.add_edge(node1, node2, sign=sign)
else:
if not net.has_edge(node1, node2) and not net.has_edge(
node2, node1):
net.add_edge(node1, node2, sign=sign)
post_size = len(net.edges())
i += 1
if (i == 10000000):
util.cluster_print(
configs['output_directory'],
"\n\n\nWARNING mutate.add_this_edge() is looping a lot.\nNode1 = "
+ str(node1_set) + ", Node2 = " + str(node2_set) + "\n\n\n")
if (bias and bias_on == 'edges'):
bias.assign_an_edge_bias(net, [node1, node2],
configs['bias_distribution'],
given_bias=given_bias)
def final_master_info(population, gen, configs):
output_dir = configs['output_directory']
nx.write_edgelist(population[0].net, output_dir+"/nets/"+str(gen))
pickle_file = output_dir + "/pickle_nets/" + str(gen) + "_pickle"
with open(pickle_file, 'wb') as file: pickle.dump(population[0].net, file)
popn_data(population, output_dir, gen)
#draw_nets.basic(population, output_dir, total_gens)
if util.boool(configs['biased']):
util.cluster_print(output_dir,"Pickling biases.")
bias.pickle_bias(population[0].net, output_dir+"/bias", configs['bias_on'])
def init_run(configs):
num_workers = int(configs['number_of_workers'])
output_dir = configs['output_directory']
init_type = str(configs['initial_net_type'])
start_size = int(configs['starting_size'])
fitness_direction = str(configs['fitness_direction'])
num_instance_output = int(configs['num_instance_output'])
instance_file = configs['instance_file']
if (num_instance_output==0): instance_file = None
population, gen, size, advice, BD_table, keep_running = None, None, None, None, None, None #avoiding annoying warnings
pop_size, num_survive = curr_gen_params(start_size, None, configs)
util.cluster_print(output_dir,"Master init: num survive: " + str(num_survive) + " out of total popn of " + str(pop_size))
prog_path = output_dir + "/progress.txt"
cont=False
if os.path.isfile(prog_path):
with open(prog_path) as file:
gen = file.readline()
if (gen == 'Done'):
util.cluster_print(output_dir, "Run already finished, exiting...\n")
return #here need another way to exit, otherwise returns wrong number of args, or package args in one doc
elif (int(gen) > 2): #IS CONTINUATION RUN
gen = int(gen)-2 #latest may not have finished
population = parse_worker_popn(num_workers, gen, output_dir, num_survive, fitness_direction)
size = len(population[0].net.nodes())
gen += 1
keep_running = util.test_stop_condition(size, gen, configs)
cont = True
if not cont: #FRESH START
init_dirs(num_workers, output_dir)
output.init_csv(output_dir, configs)
# draw_nets.init(output_dir)
population = init_nets.init_population(init_type, start_size, pop_size, configs)
advice = init.build_advice(population[0].net, configs)
#init fitness eval
pressurize.pressurize(configs, population[0],instance_file + "Xitern0.csv", advice)
gen, size = 0, start_size
keep_running = util.test_stop_condition(size, gen, configs)
return population, gen, size, advice, BD_table, num_survive, keep_running
def debug(population, worker_ID, output_dir):
pop_size = len(population)
if (worker_ID == 0):
print("Minion population fitness: ")
for p in range(pop_size):
util.cluster_util.cluster_print(output_dir,
population[p].fitness_parts[2])
# check that population is unique
for p in range(pop_size):
for q in range(0, p):
if (p != q): assert (population[p] != population[q])
util.cluster_print(output_dir, "Minion nets exist?")
for p in range(pop_size):
util.cluster_print(output_dir, population[p].net)
def rm_edges(net, num_rm, configs):
# constraints: doesn't leave 0 deg edges or mult connected components
biased = util.boool(configs['biased'])
bias_on = configs['bias_on']
orig_biases = []
for j in range(num_rm):
pre_size = post_size = len(net.edges())
i = 0
while (pre_size == post_size):
edge = rd.sample(net.edges(), 1)
edge = edge[0]
# don't allow 0 deg edges
while ((net.in_degree(edge[0]) + net.out_degree(edge[0]) == 1)
or (net.in_degree(edge[1]) + net.out_degree(edge[1]) == 1)):
edge = rd.sample(net.edges(), 1)
edge = edge[0]
sign_orig = net[edge[0]][edge[1]]['sign']
if biased and bias_on == 'edges':
bias_orig = net[edge[0]][edge[1]]['bias']
else:
bias_orig = None
orig_biases.append(bias_orig)
net.remove_edge(edge[0], edge[1])
post_size = len(net.edges())
i += 1
if (i == 10000000):
util.cluster_print(
configs['output_directory'],
"WARNING mutate.rm_edges() is looping a lot.\n")
ensure_single_cc(net,
configs,
node1=edge[0],
node2=edge[1],
sign_orig=sign_orig,
bias_orig=bias_orig)
return orig_biases
def load_sim_configs(param_file, rank, num_workers):
parameters = (open(param_file, 'r')).readlines()
assert len(parameters) > 0
configs = {}
for param in parameters:
if len(param) > 0: #ignore empty lines
if param[0] != '#': #ignore lines beginning with #
param = param.split('=')
if len(param) == 2:
key = param[0].strip().replace(' ', '_')
value = param[1].strip()
configs[key] = value
configs['KP_solver_name'] = configs['KP_solver_binary'].split(
'/')[-1].split('.')[0]
configs['timestamp'] = time.strftime("%B-%d-%Y-h%Hm%Ms%S")
configs['output_directory'] += "/"
#kp_only, stamp may need work
configs['instance_file'] = (
util.slash(configs['output_directory'])
) + "instances/" # + configs['stamp']) #TODO: 'stamp' needs to be redone is wanted
#--------------------------------------------
if rank == 0: #only master should create dir, prevents workers from fighting over creating the same dir
while not os.path.isdir(configs['output_directory']):
try:
os.makedirs(
configs['output_directory']
) # will raise an error if invalid path, which is good
except:
time.sleep(5)
continue
if (configs['number_of_workers'] != num_workers):
util.cluster_print(
configs['output_directory'],
"\nWARNING in init.load_sim_configs(): mpi #workers != config #workers! "
+ str(configs['number_of_workers']) + " vs " +
str(num_workers) + "\n"
) # not sure why this doesn't correctly get # config workers...
return configs
def read_progress_file(progress, output_dir, rank):
t_start = time.time()
while not os.path.isfile(progress): # master will create this file
time.sleep(2)
while not (os.path.getmtime(progress) + 2 < time.time()): # check that file has not been recently touched
time.sleep(2)
with open(progress, 'r') as file:
line = file.readline()
if (line == 'Done' or line == 'Done\n'):
if (rank == 1 or rank==32 or rank==63): util.cluster_print(output_dir,"Worker #" + str(rank) + " + exiting.")
return # no more work to be done
else:
gen = int(line.strip())
t_end = time.time()
#if ((rank == 1 or rank == 63 or rank == 128) and gen % 100 == 0): util.cluster_print(output_dir,"worker #" + str(rank) + " finished init in " + str(t_end-t_start) + " seconds.")
return gen
def evolve_master(configs):
# get configs
num_workers = int(configs['number_of_workers'])
output_dir = configs['output_directory']
worker_pop_size = int(configs['num_worker_nets'])
fitness_direction = str(configs['fitness_direction'])
biased = util.boool(configs['biased'])
num_sims = int(configs['num_sims'])
run_data = init_run(configs)
if not run_data: return #for ex if run already finished
population, gen, size, advice, BD_table, num_survive, keep_running = run_data
while keep_running:
t_start = time.time()
pop_size, num_survive = curr_gen_params(size, num_survive, configs)
output.master_info(population, gen, size, pop_size, num_survive, advice, BD_table, configs)
write_mpi_info(output_dir, gen)
if biased: biases = bias.gen_biases(configs) #all nets must have same bias to have comparable fitness
else: biases = None
distrib_workers(population, gen, worker_pop_size, num_survive, advice, BD_table, biases, configs)
report_timing(t_start, gen, output_dir)
population = watch(configs, gen, num_workers, output_dir, num_survive, fitness_direction)
size = len(population[0].net.nodes())
gen += 1
keep_running = util.test_stop_condition(size, gen, configs)
with open(output_dir + "/progress.txt", 'w') as out: out.write("Done")
output.final_master_info(population, gen, configs)
del_mpi_dirs(output_dir)
util.cluster_print(output_dir,"Evolution finished, generating images.")
if (num_sims == 1): plot_nets.single_run_plots(output_dir)
util.cluster_print(output_dir,"Master finished config file.\n")
return pop_size, num_survive
def report_timing(t_start, gen, output_dir, report_freq=.001):
if report_freq == 0: return
t_end = time.time()
t_elapsed = t_end - t_start
if (gen % int(1 / report_freq) == 0): util.cluster_print(output_dir, "Master finishing after " + str(t_elapsed) + " seconds.\n")
if __name__ == "__main__":
# note that phone calls this directly
rank = 0 #ie master only
config_file = sys.argv[1]
num_workers = 1
configs = init.load_sim_configs(config_file, rank, num_workers)
orig_output_dir = configs['output_directory']
num_sims = int(configs['num_sims'])
assert(num_sims == 1) #not ready for more
log_text = 'Evolve_root(): in dir ' + str(os.getcwd()) + ', config file = ' + str(config_file) + ', num_workers = ' + str(num_workers) + "\n"
util.cluster_print(configs['output_directory'], log_text)
evolve_master(configs)
print("\nDone.\n")
def report_timing(t_start, gen, output_dir, report_freq=.001):
if report_freq == 0: return
t_end = time.time()
t_elapsed = t_end - t_start
if (gen % int(1 / report_freq) == 0): util.cluster_print(output_dir, "Master finishing after " + str(t_elapsed) + " seconds.\n")
def evolve_minion(worker_file, gen, rank, output_dir):
t_start = time.time()
with open(str(worker_file), 'rb') as file:
worker_ID, seed, worker_gens, pop_size, num_return, randSeed, curr_gen, configs = pickle.load(
file)
file.close()
survive_fraction = float(configs['worker_percent_survive']) / 100
num_survive = math.ceil(survive_fraction * pop_size)
output_dir = configs['output_directory'].replace(
"v4nu_minknap_1X_both_reverse/", '')
#output_dir += str(worker_ID)
max_gen = int(configs['max_generations'])
control = configs['control']
if (control == "None"): control = None
fitness_direction = str(configs['fitness_direction'])
node_edge_ratio = float(configs['edge_to_node_ratio'])
random.seed(randSeed)
population = gen_population_from_seed(seed, pop_size)
start_size = len(seed.net.nodes())
pressurize_time = 0
mutate_time = 0
for g in range(worker_gens):
gen_percent = float(curr_gen / max_gen)
if (g != 0):
for p in range(num_survive, pop_size):
population[p] = population[p % num_survive].copy()
#assert (population[p] != population[p%num_survive])
#assert (population[p].net != population[p % num_survive].net)
for p in range(pop_size):
t0 = ptime()
mutate.mutate(configs, population[p].net, gen_percent,
node_edge_ratio)
t1 = ptime()
mutate_time += t1 - t0
if (control == None):
pressure_results = pressurize.pressurize(
configs, population[p].net, None
) # false: don't track node fitness, None: don't write instances to file
population[p].fitness_parts[0], population[p].fitness_parts[
1], population[p].fitness_parts[2] = pressure_results[
0], pressure_results[1], pressure_results[2]
else:
util.cluster_print(
output_dir, "ERROR in minion(): unknown control config: " +
str(control))
old_popn = population
population = fitness.eval_fitness(old_popn, fitness_direction)
del old_popn
#debug(population,worker_ID, output_dir)
curr_gen += 1
write_out_worker(output_dir + "/to_master/" + str(gen) + "/" + str(rank),
population, num_return)
# some output, probably outdated
if (worker_ID == 0):
orig_dir = configs['output_directory']
end_size = len(population[0].net.nodes())
growth = end_size - start_size
output.minion_csv(orig_dir, pressurize_time, growth, end_size)
#debug(population, worker_ID, output_dir)
#if (worker_ID==0): util.cluster_print(output_dir,"Pressurizing took " + str(pressurize_time) + " secs, while mutate took " + str(mutate_time) + " secs.")
t_end = time.time()
time_elapsed = t_end - t_start
if (rank == 1 or rank == 32 or rank == 63):
util.cluster_print(
output_dir, "Worker #" + str(rank) + " finishing after " +
str(time_elapsed) + " seconds")
def work(configs, rank):
output_dir = configs['output_directory']
max_gen = int(configs['max_generations'])
gen = 0
print("\t\t\t\tworker #" + str(rank) + " is working,\t")
progress = output_dir + "/progress.txt"
#init, see if cont run
t_start = time.time()
while not os.path.isfile(progress): # master will create this file
time.sleep(2)
while not (os.path.getmtime(progress) + .5 <
time.time()): # check that file has not been recently touched
time.sleep(.5)
if (True): #lol just to avoid re-indenting in vi
with open(progress, 'r') as file:
line = file.readline()
if (line == 'Done' or line == 'Done\n'):
if (rank == 1 or rank == 32 or rank == 63):
util.cluster_print(output_dir,
"Worker #" + str(rank) + " + exiting.")
return # no more work to be done
else:
gen = int(line.strip())
#util.cluster_print(output_dir,"Worker #" + str(rank) + " got gen " + str(gen) + " from progress file.")
t_end = time.time()
if ((rank == 1 or rank == 32 or rank == 63 or rank == 128)
and gen % 100 == 0):
util.cluster_print(
output_dir, "worker #" + str(rank) + " finished init in " +
str(t_end - t_start) + " seconds.")
estim_time = 4
while gen < max_gen:
t_start = time.time()
worker_file = str(output_dir) + "/to_workers/" + str(gen) + "/" + str(
rank)
#util.cluster_print(output_dir,"worker #" + str(rank) + " looking for file: " + str(worker_file))
i = 1
num_estim = 0
while not os.path.isfile(worker_file):
if (num_estim < 4):
time.sleep(estim_time / 4)
num_estim += 1
else:
time.sleep(4)
estim_time += 4
i += 1
while not (os.path.getmtime(worker_file) + .4 < time.time()):
time.sleep(.5)
t_end = time.time()
t_elapsed = t_end - t_start
if ((rank == 1 or rank == 32 or rank == 63 or rank == 128)):
util.cluster_print(
output_dir,
"Worker #" + str(rank) + " starting evolution after waiting " +
str(t_elapsed) + " seconds and checking dir " + str(i) +
" times. Starts at gen " + str(gen))
evolve_minion(worker_file, gen, rank, output_dir)
gen += 1