def solve_from_file(input_file, output_directory, params=[]): # print('Processing', input_file) input_data = utils.read_file(input_file) number_of_kingdoms, list_of_kingdom_names, starting_kingdom, adjacency_matrix = data_parser( input_data) closed_walk, conquered_kingdoms = solve(list_of_kingdom_names, starting_kingdom, adjacency_matrix, params=params) if closed_walk == "Error": print("Error") else: basename, filename = os.path.split(input_file) output_filename = utils.input_to_output(filename) output_file = f'{output_directory}/{output_filename}' if not os.path.exists(output_directory): os.makedirs(output_directory) utils.write_data_to_file(output_file, closed_walk, ' ') utils.write_to_file(output_file, '\n', append=True) utils.write_data_to_file(output_file, conquered_kingdoms, ' ', append=True)
def naive_solver(list_of_locations, list_of_homes, starting_car_location, adjacency_matrix): G, _ = adjacency_matrix_to_graph(adjacency_matrix) car_path = [int(starting_car_location)] drop_off = {int(starting_car_location): [int(h) for h in list_of_homes]} cost, _ = student_utils.cost_of_solution(G, car_path, drop_off) utils.write_data_to_file('logs/naive.log', [cost], separator='\n', append=True) return car_path, drop_off
def greedy_solver(list_of_locations, list_of_homes, starting_car_location, adjacency_matrix): G, _ = adjacency_matrix_to_graph(adjacency_matrix) all_pairs_shortest_path = dict(nx.floyd_warshall(G)) car_path, visit_order = nearest_neighbor_tour(list_of_homes, starting_car_location, all_pairs_shortest_path, G) drop_off = find_drop_off_mapping(car_path, list_of_homes, all_pairs_shortest_path) cost, _ = student_utils.cost_of_solution(G, car_path, drop_off) utils.write_data_to_file('logs/greedy.log', [cost], separator='\n', append=True) print(len(list_of_locations), 'locations', 'greedy:', cost) return car_path, drop_off
def ant_colony(list_of_locations, list_of_homes, starting_car_location, adjacency_matrix): G, _ = adjacency_matrix_to_graph(adjacency_matrix) all_pairs_shortest_path = dict(nx.floyd_warshall(G)) _, tour = nearest_neighbor_tour(list_of_homes, starting_car_location, all_pairs_shortest_path, G) tour = tour[1:] newGraph = build_tour_graph(G, tour, all_pairs_shortest_path) solution = ant_colony_tour(newGraph, starting_car_location) car_path = generate_full_path(solution, G) drop_off = find_drop_off_mapping(car_path, list_of_homes, all_pairs_shortest_path) cost, _ = student_utils.cost_of_solution(G, car_path, drop_off) utils.write_data_to_file('logs/ant_colony.log', [cost], separator='\n', append=True) print(len(list_of_locations), 'locations', 'ant_colony:', cost) return car_path, drop_off
def mst_solver(list_of_locations, list_of_homes, starting_car_location, adjacency_matrix): G, _ = adjacency_matrix_to_graph(adjacency_matrix) all_pairs_shortest_path = dict(nx.floyd_warshall(G)) verticies = set(list_of_homes) verticies.add(int(starting_car_location)) verticies = list(verticies) newGraph = build_tour_graph(G, verticies, all_pairs_shortest_path) mst = nx.minimum_spanning_tree(newGraph) mst_tour = list( nx.dfs_preorder_nodes(newGraph, source=int(starting_car_location))) mst_tour.append(int(starting_car_location)) car_path = generate_full_path(mst_tour, G) drop_off = find_drop_off_mapping(car_path, list_of_homes, all_pairs_shortest_path) cost, _ = student_utils.cost_of_solution(G, car_path, drop_off) utils.write_data_to_file('logs/mst.log', [cost], separator='\n', append=True) print(len(list_of_locations), 'locations', 'mst:', cost) return car_path, drop_off
def analyze_all(output_directory, costs_directory, params=[]): input_files = set(utils.get_files_with_extension('inputs', 'in')) num_outputs = len(os.listdir(output_directory)) # writing into csv table # data = np.empty((num_outputs + 1, 5)) header = ['output_dir', 'output_filename', 'travel_cost', 'conquer_cost', 'cost'] costs_file = f'{costs_directory}/{output_directory}.csv' if not os.path.exists(costs_directory): os.makedirs(costs_directory) utils.write_data_to_file(costs_file, header, ',') for name in os.listdir(output_directory): if name.endswith("out"): if 'inputs/' + name.replace('.out', '.in') not in input_files: continue num, ext = name.split(".") t_cost, c_cost, cost = analyze_from_file(f'inputs/{num}.in', f'{output_directory}/{name}', params=params) if cost == -1: continue line_data = [output_directory, name, t_cost, c_cost, cost] utils.write_to_file(costs_file, '\n', append=True) utils.write_data_to_file(costs_file, line_data, ',', append=True)
def greedy_clustering_three_opt_best_ratio(list_of_locations, list_of_homes, starting_car_location, adjacency_matrix): def findsubsets(s, n): result = [] for i in range(n): ls = [list(x) for x in list(itertools.combinations(s, i + 1))] result.extend(ls) return result G, _ = adjacency_matrix_to_graph(adjacency_matrix) shortest = dict(nx.floyd_warshall(G)) tour = [int(starting_car_location)] stops = [int(starting_car_location)] remain_bus_stop = set([int(l) for l in list_of_locations]) remain_bus_stop.remove(int(starting_car_location)) drop_off_map = find_drop_off_mapping(tour, list_of_homes, shortest) walk_cost = calc_walking_cost(drop_off_map, shortest) drive_cost = calc_driving_cost(tour, shortest) bestCost = walk_cost + drive_cost while True: bestTour = None bestStop = None best_ratio = 0 bstops = findsubsets(remain_bus_stop, 3) for bstop in bstops: new_tour = stops + bstop new_drop_off_map = find_drop_off_mapping(new_tour, list_of_homes, shortest) _, new_tour = nearest_neighbor_tour(new_tour, starting_car_location, shortest, G) new_tour = three_opt(new_tour, shortest) new_walk_cost = calc_walking_cost(new_drop_off_map, shortest) walk_cost_decrease = walk_cost - new_walk_cost new_drive_cost = calc_driving_cost(new_tour, shortest) drive_cost_increase = new_drive_cost - drive_cost #print(walk_cost_decrease, drive_cost_increase) if drive_cost_increase > 0 and best_ratio < (walk_cost_decrease / drive_cost_increase): bestStop = bstop best_ratio = walk_cost_decrease / drive_cost_increase bestTour = new_tour walk_cost = new_walk_cost drive_cost = new_drive_cost bestCost = new_walk_cost + new_drive_cost if best_ratio > 0: for b in bestStop: remain_bus_stop.remove(int(b)) tour = bestTour stops = stops + bestStop sys.stdout.write(str(bestCost) + '\n') # same as print sys.stdout.flush() else: break car_path = generate_full_path(tour, G) drop_off = find_drop_off_mapping(tour, list_of_homes, shortest) cost, _ = student_utils.cost_of_solution(G, car_path, drop_off) utils.write_data_to_file('logs/greedy_clustering_three_opt.log', [cost], separator='\n', append=True) print(len(list_of_locations), 'locations', 'greedy_clustering_three_opt:', cost) return car_path, drop_off
def greedy_clustering_two_opt(list_of_locations, list_of_homes, starting_car_location, adjacency_matrix, bus_stop_look_ahead): def findsubsets(s, n): result = [] for i in range(n): ls = [list(x) for x in list(itertools.combinations(s, i + 1))] result.extend(ls) return result G, _ = adjacency_matrix_to_graph(adjacency_matrix) shortest = dict(nx.floyd_warshall(G)) tour = [int(starting_car_location)] #stops = [int(starting_car_location)] remain_bus_stop = set([int(l) for l in list_of_locations]) remain_bus_stop.remove(int(starting_car_location)) drop_off_map = find_drop_off_mapping(tour, list_of_homes, shortest) min_walk_cost = calc_walking_cost(drop_off_map, shortest) min_drive_cost = calc_driving_cost(tour, shortest) minCost = min_walk_cost + min_drive_cost while True: bestTour = None bestStop = None bestCost = minCost bstops = findsubsets(remain_bus_stop, bus_stop_look_ahead) #print("number of stops",len(bstops), flush = True) for bstop in bstops: new_tour = tour + bstop new_drop_off_map = find_drop_off_mapping(new_tour, list_of_homes, shortest) new_tour = fast_nearest_neighbor_tour(new_tour, starting_car_location, shortest) new_tour = two_opt(new_tour, shortest) new_walk_cost = calc_walking_cost(new_drop_off_map, shortest) new_drive_cost = calc_driving_cost(new_tour, shortest) new_cost = new_walk_cost + new_drive_cost if new_cost < bestCost: bestStop = bstop bestCost = new_cost bestTour = new_tour if bestCost < minCost: for b in bestStop: remain_bus_stop.remove(b) minCost = bestCost tour = bestTour print(minCost, flush=True) #sys.stdout.write(str(minCost) + '\n') # same as print #sys.stdout.flush() else: break tour = three_opt(tour, shortest) car_path = generate_full_path(tour, G) drop_off = find_drop_off_mapping(tour, list_of_homes, shortest) cost, _ = student_utils.cost_of_solution(G, car_path, drop_off) utils.write_data_to_file('logs/greedy_clustering_three_opt.log', [cost], separator='\n', append=True) print(len(list_of_locations), 'locations', 'greedy_clustering_two_opt:', cost) return car_path, drop_off
def train(): # load data sets train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros) dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros) test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros) # Use selected tagging scheme (IOB / IOBES) update_tag_scheme(train_sentences, FLAGS.tag_schema) update_tag_scheme(test_sentences, FLAGS.tag_schema) # create maps if not exist, load data if exists maps if not os.path.isfile(FLAGS.map_file): # create dictionary for word if FLAGS.pre_emb: dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0] dico_chars, char_to_id, id_to_char = augment_with_pretrained( dico_chars_train.copy(), FLAGS.emb_file, list( itertools.chain.from_iterable([[w[0] for w in s] for s in test_sentences]))) else: _c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower) # Create a dictionary and a mapping for tags _t, tag_to_id, id_to_tag = tag_mapping(train_sentences) with open(FLAGS.map_file, "wb") as f: pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f) else: with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) # prepare data, get a collection of list containing index train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id, FLAGS.lower) dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id, FLAGS.lower) test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id, FLAGS.lower) print("%i / %i / %i sentences in train / dev / test." % (len(train_data), 0, len(test_data))) train_manager = BatchManager(train_data, FLAGS.batch_size) dev_manager = BatchManager(dev_data, 100) test_manager = BatchManager(test_data, 100) # make path for store log and model if not exist make_path(FLAGS) if os.path.isfile(FLAGS.config_file): config = load_config(FLAGS.config_file) else: config = config_model(char_to_id, tag_to_id) save_config(config, FLAGS.config_file) make_path(FLAGS) log_path = os.path.join("log", FLAGS.log_file) logger = get_logger(log_path) print_config(config, logger) # 设置训练日志目录 train_log = os.path.join(FLAGS.logdir, "train") if not os.path.exists(train_log): os.makedirs(train_log) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True steps_per_epoch = train_manager.len_data # the nums of batch data with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) # 观察所建立的计算图 train_writer = tf.summary.FileWriter(train_log, sess.graph) logger.info("start training") loss = [] dev_f1 = [] test_f1 = [] for i in range(100): for batch in train_manager.iter_batch(shuffle=True): step, batch_loss, merged = model.run_step( sess, True, batch) # step是global step # 在迭代中输出到结果 train_writer.add_summary(merged, step) loss.append(batch_loss) if step % FLAGS.steps_check == 0: iteration = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "NER loss:{:>9.6f}".format( iteration, step % steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] # use dev data to validation the model best, dev_f1_value = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger) # store the dev f1 dev_f1.append(dev_f1_value) if best: save_model(sess, model, FLAGS.ckpt_path, logger) # use current the model to test _, test_f1_value = evaluate(sess, model, "test", test_manager, id_to_tag, logger) # store the test f1 test_f1.append(test_f1_value) # write the dev_f1 and test_f1 to file f1_result = {} f1_result["dev_f1"] = dev_f1 f1_result["test_f1"] = test_f1 write_data_to_file(f1_result, "f1_result")
def matrixToInputFile(input_filename, num, kingdom_names, starting_kingdom, matrix_data, input_directory="in"): input_file = f'{input_directory}/{input_filename}' if not os.path.exists(input_directory): os.makedirs(input_directory) utils.write_data_to_file(input_file, [num], '\n') utils.write_data_to_file(input_file, kingdom_names, ' ', append=True) utils.write_data_to_file(input_file, '\n', '', append=True) utils.write_data_to_file(input_file, [starting_kingdom], '\n', append=True) for row in matrix_data: utils.write_data_to_file(input_file, row, ' ', append=True) utils.write_data_to_file(input_file, '\n', '', append=True)
def compare_by_file(input_file, compare1, compare2, output_directory, params=[]): print('Processing', input_file) global fromcomp1 global fromcomp2 global totalcost1 global totalcost2 global totalcostcombined global equallygood input_data = utils.read_file(input_file) compare_data1 = utils.read_file(compare1) compare_data2 = utils.read_file(compare2) number_of_kingdoms, list_of_kingdom_names, starting_kingdom, adjacency_matrix = data_parser(input_data) G = adjacency_matrix_to_graph(adjacency_matrix) comp1_kingdom_tour_name = compare_data1[0] comp1_conquered_kingdoms_name = compare_data1[1] comp2_kingdom_tour_name = compare_data2[0] comp2_conquered_kingdoms_name = compare_data2[1] comp1_kingdom_tour_index = convert_kingdom_names_to_indices(comp1_kingdom_tour_name, list_of_kingdom_names) comp1_conquered_kingdoms_index = convert_kingdom_names_to_indices(comp1_conquered_kingdoms_name, list_of_kingdom_names) comp2_kingdom_tour_index = convert_kingdom_names_to_indices(comp2_kingdom_tour_name, list_of_kingdom_names) comp2_conquered_kingdoms_index = convert_kingdom_names_to_indices(comp2_conquered_kingdoms_name, list_of_kingdom_names) cost1 = cost_of_solution(G, comp1_kingdom_tour_index, comp1_conquered_kingdoms_index)[0] cost2 = cost_of_solution(G, comp2_kingdom_tour_index, comp2_conquered_kingdoms_index)[0] if cost1 == 'infinite': print(compare1) raise Exception("not a valid outputs!!!") elif cost2 =='infinite': print(compare2) raise Exception("not a valid outputs!!!") basename, filename = os.path.split(input_file) output_filename = utils.input_to_output(filename) output_file = '{}/{}'.format(output_directory, output_filename) if not os.path.exists(output_directory): os.makedirs(output_directory) totalcost1 += cost1 totalcost2 += cost2 if cost1 < cost2: totalcostcombined += cost1 fromcomp1 += 1 utils.write_data_to_file(output_file, comp1_kingdom_tour_name, ' ') utils.write_to_file(output_file, '\n', append=True) utils.write_data_to_file(output_file, comp1_conquered_kingdoms_name, ' ', append=True) else : if cost1 == cost2 : equallygood +=1 else: fromcomp2 += 1 totalcostcombined += cost2 utils.write_data_to_file(output_file, comp2_kingdom_tour_name, ' ') utils.write_to_file(output_file, '\n', append=True) utils.write_data_to_file(output_file, comp2_conquered_kingdoms_name, ' ', append=True)
def store_in_cache(data, cache_path): utils.write_data_to_file(json.dumps(data), cache_path)