def distrib_workers(population, gen, worker_pop_size, num_survive, advice, BD_table, biases, configs): num_workers = int(configs['number_of_workers']) output_dir = configs['output_directory'] debug = util.boool(configs['debug']) if (debug == True): # sequential debug for w in range(1, num_workers + 1): dump_file = output_dir + "to_workers/" + str(gen) + "/" + str(w) seed = population[0].copy() randSeeds = os.urandom(sysRand().randint(0, 1000000)) worker_args = [w, seed, worker_pop_size, min(worker_pop_size, num_survive), randSeeds, advice, BD_table, biases, configs] with open(dump_file, 'wb') as file: pickle.dump(worker_args, file) # pool.map_async(minion.evolve_minion, (dump_file,)) minion.evolve_minion(dump_file, gen, w, output_dir) sleep(.0001) else: for w in range(1, num_workers + 1): dump_file = output_dir + "/to_workers/" + str(gen) + "/" + str(w) seed = population[w % num_survive].copy() randSeeds = os.urandom(sysRand().randint(0, 1000000)) assert (seed != population[w % num_survive]) worker_args = [w, seed, worker_pop_size, min(worker_pop_size, num_survive), randSeeds, advice, BD_table, biases, configs] with open(dump_file, 'wb') as file: pickle.dump(worker_args, file) del population if (debug == True): util.cluster_print(output_dir, "debug is ON")
def report_timing(t_start, rank, gen, output_dir): t_end = time.time() time_elapsed = t_end - t_start if ((rank == 1 or rank == 63) and gen % 100 == 0): util.cluster_print( output_dir, "Worker #" + str(rank) + " finishing after " + str(time_elapsed) + " seconds")
def watch(configs, gen, num_workers, output_dir, num_survive, fitness_direction): dump_dir = output_dir + "/to_master/" + str(gen) t_start = time.time() popn, num_finished, dir_checks = [], 0,0 ids = [str(i) for i in range(1, num_workers + 1)] while (num_finished < num_workers): time.sleep(1) dir_checks+=1 for root, dirs, files in os.walk(dump_dir): for f in files: if f in ids: if (os.path.getmtime(root + "/" + f) + 1 < time.time()): dump_file = output_dir + "/to_master/" + str(gen) + "/" + str(f) with open(dump_file, 'rb') as file: try: worker_pop = pickle.load(file) popn += worker_pop[:num_survive] num_finished += 1 ids.remove(f) except: pass #sort and delete some sorted_popn = fitness.eval_fitness(popn, fitness_direction) popn = sorted_popn[:num_survive] del sorted_popn assert (not ids) t_end = time.time() time_elapsed = t_end - t_start if (gen % 100 == 0): util.cluster_print(output_dir,"master finished extracting workers after " + str(time_elapsed) + " seconds, and making " + str(dir_checks) + " dir checks.") return popn
def init_sim(configs, num_sims, sim_num, orig_output_dir): if (num_sims > 1 and sim_num == 0): # check where to pick up the run this_dir = False while (not this_dir): if (sim_num >= num_sims): util.cluster_print( orig_output_dir, "All simulations already finished, exiting...\n") return configs['output_directory'] = orig_output_dir + "_" + str(i) this_dir = True # remains true if any of the following fail if os.path.exists(configs['output_directory'] + "/progress.txt"): with open(configs['output_directory'] + "/progress.txt") as progress: line = progress.readline() if (line.strip() == 'Done' or line.strip() == 'done'): this_dir = False sim_num += 1 if (num_sims > 1): configs['output_directory'] = orig_output_dir + "sim_" + str( sim_num) + "/" configs['instance_file'] = (util.slash(configs['output_directory']) + "/instances/" + configs['stamp']) if (rank == 0): if not os.path.exists(configs['output_directory']): os.makedirs(configs['output_directory']) else: while (not os.path.exists(configs['output_directory'])): sleep(1)
def evolve(rank, num_workers, config_file): configs = init.load_sim_configs(config_file, rank, num_workers) orig_output_dir = configs['output_directory'] num_sims = int(configs['num_sims']) for i in range(num_sims): err = init_sim(configs, num_sims, i, orig_output_dir, rank) #WARNING: this area might need more work, esp if mult isms if rank == 0 and not err: # MASTER log_text = 'Evolve_root(): in dir ' + str( os.getcwd()) + ', config file = ' + str( config_file) + ', num_workers = ' + str(num_workers) + "\n" import master util.cluster_print(configs['output_directory'], log_text) master.evolve_master(configs) elif not err: # WORKERS import minion minion.work(configs, rank) if (num_sims > 1 and rank == 0): close_out_mult_sims(num_sims, orig_output_dir)
def master_info(population, gen, size, pop_size, num_survive, advice, BD_table, configs): output_dir = configs['output_directory'] num_data_output = int(configs['num_data_output']) num_net_output = int(configs['num_net_output']) num_instance_output = int(configs['num_instance_output']) instance_file = configs['instance_file'] if (num_instance_output==0): instance_file = None stop_condition = configs['stop_condition'] if (stop_condition == 'size'): end = int(configs['ending_size']) #this sort of assumes simulation starts near size 0 elif (stop_condition == 'generation'): end = int(configs['max_generations']) else: assert(False) if (num_data_output > 0): if (gen % int(end / num_data_output) == 0): popn_data(population, output_dir, gen) util.cluster_print(output_dir, "Master at gen " + str(gen) + ", with net size = " + str(size) + " nodes and " + str(len(population[0].net.edges())) + " edges, " + str(num_survive) + "<=" + str(len(population)) + " survive out of " + str(pop_size)) nx.write_edgelist(population[0].net, output_dir + "/fittest_net.edgelist") if (num_instance_output != 0): if (gen % int(end / num_instance_output) == 0): # if first gen, have already pressurized w/net[0] if (gen != 0): pressurize.pressurize(configs, population[0], instance_file + "Xitern" + str(gen) + ".csv", advice, BD_table) if (num_net_output > 0): if (gen % int(end / num_net_output) == 0): nx.write_edgelist(population[0].net, output_dir + "/nets/" + str(gen)) pickle_file = output_dir + "/pickle_nets/" + str(gen) + "_pickle" with open(pickle_file, 'wb') as file: pickle.dump(population[0].net, file) deg_distrib_csv(output_dir, population, gen)
def add_this_edge(net, configs, node1=None, node2=None, sign=None, random_direction=False, given_bias=None): reverse_allowed = util.boool(configs['reverse_edges_allowed']) bias_on = configs['bias_on'] node1_set, node2_set = node1, node2 #to save their states if not sign: sign = rd.randint(0, 1) if (sign == 0): sign = -1 pre_size = post_size = len(net.edges()) i = 0 while (pre_size == post_size): # ensure that net adds if not node1_set and node1_set != 0: node = rd.sample(net.nodes(), 1) node1 = node[0] if not node2_set and node2_set != 0: node2 = node1 while (node2 == node1): node2 = rd.sample(net.nodes(), 1) node2 = node2[0] if random_direction: #chance to swap nodes 1 & 2 if (rd.random() < .5): node3 = node2 node2 = node1 node1 = node3 if reverse_allowed: if not net.has_edge(node1, node2): net.add_edge(node1, node2, sign=sign) else: if not net.has_edge(node1, node2) and not net.has_edge( node2, node1): net.add_edge(node1, node2, sign=sign) post_size = len(net.edges()) i += 1 if (i == 10000000): util.cluster_print( configs['output_directory'], "\n\n\nWARNING mutate.add_this_edge() is looping a lot.\nNode1 = " + str(node1_set) + ", Node2 = " + str(node2_set) + "\n\n\n") if (bias and bias_on == 'edges'): bias.assign_an_edge_bias(net, [node1, node2], configs['bias_distribution'], given_bias=given_bias)
def final_master_info(population, gen, configs): output_dir = configs['output_directory'] nx.write_edgelist(population[0].net, output_dir+"/nets/"+str(gen)) pickle_file = output_dir + "/pickle_nets/" + str(gen) + "_pickle" with open(pickle_file, 'wb') as file: pickle.dump(population[0].net, file) popn_data(population, output_dir, gen) #draw_nets.basic(population, output_dir, total_gens) if util.boool(configs['biased']): util.cluster_print(output_dir,"Pickling biases.") bias.pickle_bias(population[0].net, output_dir+"/bias", configs['bias_on'])
def init_run(configs): num_workers = int(configs['number_of_workers']) output_dir = configs['output_directory'] init_type = str(configs['initial_net_type']) start_size = int(configs['starting_size']) fitness_direction = str(configs['fitness_direction']) num_instance_output = int(configs['num_instance_output']) instance_file = configs['instance_file'] if (num_instance_output==0): instance_file = None population, gen, size, advice, BD_table, keep_running = None, None, None, None, None, None #avoiding annoying warnings pop_size, num_survive = curr_gen_params(start_size, None, configs) util.cluster_print(output_dir,"Master init: num survive: " + str(num_survive) + " out of total popn of " + str(pop_size)) prog_path = output_dir + "/progress.txt" cont=False if os.path.isfile(prog_path): with open(prog_path) as file: gen = file.readline() if (gen == 'Done'): util.cluster_print(output_dir, "Run already finished, exiting...\n") return #here need another way to exit, otherwise returns wrong number of args, or package args in one doc elif (int(gen) > 2): #IS CONTINUATION RUN gen = int(gen)-2 #latest may not have finished population = parse_worker_popn(num_workers, gen, output_dir, num_survive, fitness_direction) size = len(population[0].net.nodes()) gen += 1 keep_running = util.test_stop_condition(size, gen, configs) cont = True if not cont: #FRESH START init_dirs(num_workers, output_dir) output.init_csv(output_dir, configs) # draw_nets.init(output_dir) population = init_nets.init_population(init_type, start_size, pop_size, configs) advice = init.build_advice(population[0].net, configs) #init fitness eval pressurize.pressurize(configs, population[0],instance_file + "Xitern0.csv", advice) gen, size = 0, start_size keep_running = util.test_stop_condition(size, gen, configs) return population, gen, size, advice, BD_table, num_survive, keep_running
def debug(population, worker_ID, output_dir): pop_size = len(population) if (worker_ID == 0): print("Minion population fitness: ") for p in range(pop_size): util.cluster_util.cluster_print(output_dir, population[p].fitness_parts[2]) # check that population is unique for p in range(pop_size): for q in range(0, p): if (p != q): assert (population[p] != population[q]) util.cluster_print(output_dir, "Minion nets exist?") for p in range(pop_size): util.cluster_print(output_dir, population[p].net)
def rm_edges(net, num_rm, configs): # constraints: doesn't leave 0 deg edges or mult connected components biased = util.boool(configs['biased']) bias_on = configs['bias_on'] orig_biases = [] for j in range(num_rm): pre_size = post_size = len(net.edges()) i = 0 while (pre_size == post_size): edge = rd.sample(net.edges(), 1) edge = edge[0] # don't allow 0 deg edges while ((net.in_degree(edge[0]) + net.out_degree(edge[0]) == 1) or (net.in_degree(edge[1]) + net.out_degree(edge[1]) == 1)): edge = rd.sample(net.edges(), 1) edge = edge[0] sign_orig = net[edge[0]][edge[1]]['sign'] if biased and bias_on == 'edges': bias_orig = net[edge[0]][edge[1]]['bias'] else: bias_orig = None orig_biases.append(bias_orig) net.remove_edge(edge[0], edge[1]) post_size = len(net.edges()) i += 1 if (i == 10000000): util.cluster_print( configs['output_directory'], "WARNING mutate.rm_edges() is looping a lot.\n") ensure_single_cc(net, configs, node1=edge[0], node2=edge[1], sign_orig=sign_orig, bias_orig=bias_orig) return orig_biases
def load_sim_configs(param_file, rank, num_workers): parameters = (open(param_file, 'r')).readlines() assert len(parameters) > 0 configs = {} for param in parameters: if len(param) > 0: #ignore empty lines if param[0] != '#': #ignore lines beginning with # param = param.split('=') if len(param) == 2: key = param[0].strip().replace(' ', '_') value = param[1].strip() configs[key] = value configs['KP_solver_name'] = configs['KP_solver_binary'].split( '/')[-1].split('.')[0] configs['timestamp'] = time.strftime("%B-%d-%Y-h%Hm%Ms%S") configs['output_directory'] += "/" #kp_only, stamp may need work configs['instance_file'] = ( util.slash(configs['output_directory']) ) + "instances/" # + configs['stamp']) #TODO: 'stamp' needs to be redone is wanted #-------------------------------------------- if rank == 0: #only master should create dir, prevents workers from fighting over creating the same dir while not os.path.isdir(configs['output_directory']): try: os.makedirs( configs['output_directory'] ) # will raise an error if invalid path, which is good except: time.sleep(5) continue if (configs['number_of_workers'] != num_workers): util.cluster_print( configs['output_directory'], "\nWARNING in init.load_sim_configs(): mpi #workers != config #workers! " + str(configs['number_of_workers']) + " vs " + str(num_workers) + "\n" ) # not sure why this doesn't correctly get # config workers... return configs
def read_progress_file(progress, output_dir, rank): t_start = time.time() while not os.path.isfile(progress): # master will create this file time.sleep(2) while not (os.path.getmtime(progress) + 2 < time.time()): # check that file has not been recently touched time.sleep(2) with open(progress, 'r') as file: line = file.readline() if (line == 'Done' or line == 'Done\n'): if (rank == 1 or rank==32 or rank==63): util.cluster_print(output_dir,"Worker #" + str(rank) + " + exiting.") return # no more work to be done else: gen = int(line.strip()) t_end = time.time() #if ((rank == 1 or rank == 63 or rank == 128) and gen % 100 == 0): util.cluster_print(output_dir,"worker #" + str(rank) + " finished init in " + str(t_end-t_start) + " seconds.") return gen
def evolve_master(configs): # get configs num_workers = int(configs['number_of_workers']) output_dir = configs['output_directory'] worker_pop_size = int(configs['num_worker_nets']) fitness_direction = str(configs['fitness_direction']) biased = util.boool(configs['biased']) num_sims = int(configs['num_sims']) run_data = init_run(configs) if not run_data: return #for ex if run already finished population, gen, size, advice, BD_table, num_survive, keep_running = run_data while keep_running: t_start = time.time() pop_size, num_survive = curr_gen_params(size, num_survive, configs) output.master_info(population, gen, size, pop_size, num_survive, advice, BD_table, configs) write_mpi_info(output_dir, gen) if biased: biases = bias.gen_biases(configs) #all nets must have same bias to have comparable fitness else: biases = None distrib_workers(population, gen, worker_pop_size, num_survive, advice, BD_table, biases, configs) report_timing(t_start, gen, output_dir) population = watch(configs, gen, num_workers, output_dir, num_survive, fitness_direction) size = len(population[0].net.nodes()) gen += 1 keep_running = util.test_stop_condition(size, gen, configs) with open(output_dir + "/progress.txt", 'w') as out: out.write("Done") output.final_master_info(population, gen, configs) del_mpi_dirs(output_dir) util.cluster_print(output_dir,"Evolution finished, generating images.") if (num_sims == 1): plot_nets.single_run_plots(output_dir) util.cluster_print(output_dir,"Master finished config file.\n")
return pop_size, num_survive def report_timing(t_start, gen, output_dir, report_freq=.001): if report_freq == 0: return t_end = time.time() t_elapsed = t_end - t_start if (gen % int(1 / report_freq) == 0): util.cluster_print(output_dir, "Master finishing after " + str(t_elapsed) + " seconds.\n") if __name__ == "__main__": # note that phone calls this directly rank = 0 #ie master only config_file = sys.argv[1] num_workers = 1 configs = init.load_sim_configs(config_file, rank, num_workers) orig_output_dir = configs['output_directory'] num_sims = int(configs['num_sims']) assert(num_sims == 1) #not ready for more log_text = 'Evolve_root(): in dir ' + str(os.getcwd()) + ', config file = ' + str(config_file) + ', num_workers = ' + str(num_workers) + "\n" util.cluster_print(configs['output_directory'], log_text) evolve_master(configs) print("\nDone.\n")
def report_timing(t_start, gen, output_dir, report_freq=.001): if report_freq == 0: return t_end = time.time() t_elapsed = t_end - t_start if (gen % int(1 / report_freq) == 0): util.cluster_print(output_dir, "Master finishing after " + str(t_elapsed) + " seconds.\n")
def evolve_minion(worker_file, gen, rank, output_dir): t_start = time.time() with open(str(worker_file), 'rb') as file: worker_ID, seed, worker_gens, pop_size, num_return, randSeed, curr_gen, configs = pickle.load( file) file.close() survive_fraction = float(configs['worker_percent_survive']) / 100 num_survive = math.ceil(survive_fraction * pop_size) output_dir = configs['output_directory'].replace( "v4nu_minknap_1X_both_reverse/", '') #output_dir += str(worker_ID) max_gen = int(configs['max_generations']) control = configs['control'] if (control == "None"): control = None fitness_direction = str(configs['fitness_direction']) node_edge_ratio = float(configs['edge_to_node_ratio']) random.seed(randSeed) population = gen_population_from_seed(seed, pop_size) start_size = len(seed.net.nodes()) pressurize_time = 0 mutate_time = 0 for g in range(worker_gens): gen_percent = float(curr_gen / max_gen) if (g != 0): for p in range(num_survive, pop_size): population[p] = population[p % num_survive].copy() #assert (population[p] != population[p%num_survive]) #assert (population[p].net != population[p % num_survive].net) for p in range(pop_size): t0 = ptime() mutate.mutate(configs, population[p].net, gen_percent, node_edge_ratio) t1 = ptime() mutate_time += t1 - t0 if (control == None): pressure_results = pressurize.pressurize( configs, population[p].net, None ) # false: don't track node fitness, None: don't write instances to file population[p].fitness_parts[0], population[p].fitness_parts[ 1], population[p].fitness_parts[2] = pressure_results[ 0], pressure_results[1], pressure_results[2] else: util.cluster_print( output_dir, "ERROR in minion(): unknown control config: " + str(control)) old_popn = population population = fitness.eval_fitness(old_popn, fitness_direction) del old_popn #debug(population,worker_ID, output_dir) curr_gen += 1 write_out_worker(output_dir + "/to_master/" + str(gen) + "/" + str(rank), population, num_return) # some output, probably outdated if (worker_ID == 0): orig_dir = configs['output_directory'] end_size = len(population[0].net.nodes()) growth = end_size - start_size output.minion_csv(orig_dir, pressurize_time, growth, end_size) #debug(population, worker_ID, output_dir) #if (worker_ID==0): util.cluster_print(output_dir,"Pressurizing took " + str(pressurize_time) + " secs, while mutate took " + str(mutate_time) + " secs.") t_end = time.time() time_elapsed = t_end - t_start if (rank == 1 or rank == 32 or rank == 63): util.cluster_print( output_dir, "Worker #" + str(rank) + " finishing after " + str(time_elapsed) + " seconds")
def work(configs, rank): output_dir = configs['output_directory'] max_gen = int(configs['max_generations']) gen = 0 print("\t\t\t\tworker #" + str(rank) + " is working,\t") progress = output_dir + "/progress.txt" #init, see if cont run t_start = time.time() while not os.path.isfile(progress): # master will create this file time.sleep(2) while not (os.path.getmtime(progress) + .5 < time.time()): # check that file has not been recently touched time.sleep(.5) if (True): #lol just to avoid re-indenting in vi with open(progress, 'r') as file: line = file.readline() if (line == 'Done' or line == 'Done\n'): if (rank == 1 or rank == 32 or rank == 63): util.cluster_print(output_dir, "Worker #" + str(rank) + " + exiting.") return # no more work to be done else: gen = int(line.strip()) #util.cluster_print(output_dir,"Worker #" + str(rank) + " got gen " + str(gen) + " from progress file.") t_end = time.time() if ((rank == 1 or rank == 32 or rank == 63 or rank == 128) and gen % 100 == 0): util.cluster_print( output_dir, "worker #" + str(rank) + " finished init in " + str(t_end - t_start) + " seconds.") estim_time = 4 while gen < max_gen: t_start = time.time() worker_file = str(output_dir) + "/to_workers/" + str(gen) + "/" + str( rank) #util.cluster_print(output_dir,"worker #" + str(rank) + " looking for file: " + str(worker_file)) i = 1 num_estim = 0 while not os.path.isfile(worker_file): if (num_estim < 4): time.sleep(estim_time / 4) num_estim += 1 else: time.sleep(4) estim_time += 4 i += 1 while not (os.path.getmtime(worker_file) + .4 < time.time()): time.sleep(.5) t_end = time.time() t_elapsed = t_end - t_start if ((rank == 1 or rank == 32 or rank == 63 or rank == 128)): util.cluster_print( output_dir, "Worker #" + str(rank) + " starting evolution after waiting " + str(t_elapsed) + " seconds and checking dir " + str(i) + " times. Starts at gen " + str(gen)) evolve_minion(worker_file, gen, rank, output_dir) gen += 1