def solve_from_file(input_file, output_directory, params=[]):
# print('Processing', input_file)
input_data = utils.read_file(input_file)
number_of_kingdoms, list_of_kingdom_names, starting_kingdom, adjacency_matrix = data_parser(
input_data)
closed_walk, conquered_kingdoms = solve(list_of_kingdom_names,
starting_kingdom,
adjacency_matrix,
params=params)
if closed_walk == "Error":
print("Error")
else:
basename, filename = os.path.split(input_file)
output_filename = utils.input_to_output(filename)
output_file = f'{output_directory}/{output_filename}'
if not os.path.exists(output_directory):
os.makedirs(output_directory)
utils.write_data_to_file(output_file, closed_walk, ' ')
utils.write_to_file(output_file, '\n', append=True)
utils.write_data_to_file(output_file,
conquered_kingdoms,
' ',
append=True)
def naive_solver(list_of_locations, list_of_homes, starting_car_location,
adjacency_matrix):
G, _ = adjacency_matrix_to_graph(adjacency_matrix)
car_path = [int(starting_car_location)]
drop_off = {int(starting_car_location): [int(h) for h in list_of_homes]}
cost, _ = student_utils.cost_of_solution(G, car_path, drop_off)
utils.write_data_to_file('logs/naive.log', [cost],
separator='\n',
append=True)
return car_path, drop_off
def greedy_solver(list_of_locations, list_of_homes, starting_car_location,
adjacency_matrix):
G, _ = adjacency_matrix_to_graph(adjacency_matrix)
all_pairs_shortest_path = dict(nx.floyd_warshall(G))
car_path, visit_order = nearest_neighbor_tour(list_of_homes,
starting_car_location,
all_pairs_shortest_path, G)
drop_off = find_drop_off_mapping(car_path, list_of_homes,
all_pairs_shortest_path)
cost, _ = student_utils.cost_of_solution(G, car_path, drop_off)
utils.write_data_to_file('logs/greedy.log', [cost],
separator='\n',
append=True)
print(len(list_of_locations), 'locations', 'greedy:', cost)
return car_path, drop_off
def ant_colony(list_of_locations, list_of_homes, starting_car_location,
adjacency_matrix):
G, _ = adjacency_matrix_to_graph(adjacency_matrix)
all_pairs_shortest_path = dict(nx.floyd_warshall(G))
_, tour = nearest_neighbor_tour(list_of_homes, starting_car_location,
all_pairs_shortest_path, G)
tour = tour[1:]
newGraph = build_tour_graph(G, tour, all_pairs_shortest_path)
solution = ant_colony_tour(newGraph, starting_car_location)
car_path = generate_full_path(solution, G)
drop_off = find_drop_off_mapping(car_path, list_of_homes,
all_pairs_shortest_path)
cost, _ = student_utils.cost_of_solution(G, car_path, drop_off)
utils.write_data_to_file('logs/ant_colony.log', [cost],
separator='\n',
append=True)
print(len(list_of_locations), 'locations', 'ant_colony:', cost)
return car_path, drop_off
def mst_solver(list_of_locations, list_of_homes, starting_car_location,
adjacency_matrix):
G, _ = adjacency_matrix_to_graph(adjacency_matrix)
all_pairs_shortest_path = dict(nx.floyd_warshall(G))
verticies = set(list_of_homes)
verticies.add(int(starting_car_location))
verticies = list(verticies)
newGraph = build_tour_graph(G, verticies, all_pairs_shortest_path)
mst = nx.minimum_spanning_tree(newGraph)
mst_tour = list(
nx.dfs_preorder_nodes(newGraph, source=int(starting_car_location)))
mst_tour.append(int(starting_car_location))
car_path = generate_full_path(mst_tour, G)
drop_off = find_drop_off_mapping(car_path, list_of_homes,
all_pairs_shortest_path)
cost, _ = student_utils.cost_of_solution(G, car_path, drop_off)
utils.write_data_to_file('logs/mst.log', [cost],
separator='\n',
append=True)
print(len(list_of_locations), 'locations', 'mst:', cost)
return car_path, drop_off
def analyze_all(output_directory, costs_directory, params=[]):
input_files = set(utils.get_files_with_extension('inputs', 'in'))
num_outputs = len(os.listdir(output_directory))
# writing into csv table
# data = np.empty((num_outputs + 1, 5))
header = ['output_dir', 'output_filename', 'travel_cost', 'conquer_cost', 'cost']
costs_file = f'{costs_directory}/{output_directory}.csv'
if not os.path.exists(costs_directory):
os.makedirs(costs_directory)
utils.write_data_to_file(costs_file, header, ',')
for name in os.listdir(output_directory):
if name.endswith("out"):
if 'inputs/' + name.replace('.out', '.in') not in input_files:
continue
num, ext = name.split(".")
t_cost, c_cost, cost = analyze_from_file(f'inputs/{num}.in', f'{output_directory}/{name}', params=params)
if cost == -1:
continue
line_data = [output_directory, name, t_cost, c_cost, cost]
utils.write_to_file(costs_file, '\n', append=True)
utils.write_data_to_file(costs_file, line_data, ',', append=True)
def greedy_clustering_three_opt_best_ratio(list_of_locations, list_of_homes,
starting_car_location,
adjacency_matrix):
def findsubsets(s, n):
result = []
for i in range(n):
ls = [list(x) for x in list(itertools.combinations(s, i + 1))]
result.extend(ls)
return result
G, _ = adjacency_matrix_to_graph(adjacency_matrix)
shortest = dict(nx.floyd_warshall(G))
tour = [int(starting_car_location)]
stops = [int(starting_car_location)]
remain_bus_stop = set([int(l) for l in list_of_locations])
remain_bus_stop.remove(int(starting_car_location))
drop_off_map = find_drop_off_mapping(tour, list_of_homes, shortest)
walk_cost = calc_walking_cost(drop_off_map, shortest)
drive_cost = calc_driving_cost(tour, shortest)
bestCost = walk_cost + drive_cost
while True:
bestTour = None
bestStop = None
best_ratio = 0
bstops = findsubsets(remain_bus_stop, 3)
for bstop in bstops:
new_tour = stops + bstop
new_drop_off_map = find_drop_off_mapping(new_tour, list_of_homes,
shortest)
_, new_tour = nearest_neighbor_tour(new_tour,
starting_car_location,
shortest, G)
new_tour = three_opt(new_tour, shortest)
new_walk_cost = calc_walking_cost(new_drop_off_map, shortest)
walk_cost_decrease = walk_cost - new_walk_cost
new_drive_cost = calc_driving_cost(new_tour, shortest)
drive_cost_increase = new_drive_cost - drive_cost
#print(walk_cost_decrease, drive_cost_increase)
if drive_cost_increase > 0 and best_ratio < (walk_cost_decrease /
drive_cost_increase):
bestStop = bstop
best_ratio = walk_cost_decrease / drive_cost_increase
bestTour = new_tour
walk_cost = new_walk_cost
drive_cost = new_drive_cost
bestCost = new_walk_cost + new_drive_cost
if best_ratio > 0:
for b in bestStop:
remain_bus_stop.remove(int(b))
tour = bestTour
stops = stops + bestStop
sys.stdout.write(str(bestCost) + '\n') # same as print
sys.stdout.flush()
else:
break
car_path = generate_full_path(tour, G)
drop_off = find_drop_off_mapping(tour, list_of_homes, shortest)
cost, _ = student_utils.cost_of_solution(G, car_path, drop_off)
utils.write_data_to_file('logs/greedy_clustering_three_opt.log', [cost],
separator='\n',
append=True)
print(len(list_of_locations), 'locations', 'greedy_clustering_three_opt:',
cost)
return car_path, drop_off
def greedy_clustering_two_opt(list_of_locations, list_of_homes,
starting_car_location, adjacency_matrix,
bus_stop_look_ahead):
def findsubsets(s, n):
result = []
for i in range(n):
ls = [list(x) for x in list(itertools.combinations(s, i + 1))]
result.extend(ls)
return result
G, _ = adjacency_matrix_to_graph(adjacency_matrix)
shortest = dict(nx.floyd_warshall(G))
tour = [int(starting_car_location)]
#stops = [int(starting_car_location)]
remain_bus_stop = set([int(l) for l in list_of_locations])
remain_bus_stop.remove(int(starting_car_location))
drop_off_map = find_drop_off_mapping(tour, list_of_homes, shortest)
min_walk_cost = calc_walking_cost(drop_off_map, shortest)
min_drive_cost = calc_driving_cost(tour, shortest)
minCost = min_walk_cost + min_drive_cost
while True:
bestTour = None
bestStop = None
bestCost = minCost
bstops = findsubsets(remain_bus_stop, bus_stop_look_ahead)
#print("number of stops",len(bstops), flush = True)
for bstop in bstops:
new_tour = tour + bstop
new_drop_off_map = find_drop_off_mapping(new_tour, list_of_homes,
shortest)
new_tour = fast_nearest_neighbor_tour(new_tour,
starting_car_location,
shortest)
new_tour = two_opt(new_tour, shortest)
new_walk_cost = calc_walking_cost(new_drop_off_map, shortest)
new_drive_cost = calc_driving_cost(new_tour, shortest)
new_cost = new_walk_cost + new_drive_cost
if new_cost < bestCost:
bestStop = bstop
bestCost = new_cost
bestTour = new_tour
if bestCost < minCost:
for b in bestStop:
remain_bus_stop.remove(b)
minCost = bestCost
tour = bestTour
print(minCost, flush=True)
#sys.stdout.write(str(minCost) + '\n') # same as print
#sys.stdout.flush()
else:
break
tour = three_opt(tour, shortest)
car_path = generate_full_path(tour, G)
drop_off = find_drop_off_mapping(tour, list_of_homes, shortest)
cost, _ = student_utils.cost_of_solution(G, car_path, drop_off)
utils.write_data_to_file('logs/greedy_clustering_three_opt.log', [cost],
separator='\n',
append=True)
print(len(list_of_locations), 'locations', 'greedy_clustering_two_opt:',
cost)
return car_path, drop_off
def train():
# load data sets
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower,
FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist, load data if exists maps
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb:
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences,
FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
FLAGS.lower)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id,
FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.lower)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), 0, len(test_data)))
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# 设置训练日志目录
train_log = os.path.join(FLAGS.logdir, "train")
if not os.path.exists(train_log):
os.makedirs(train_log)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data # the nums of batch data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
# 观察所建立的计算图
train_writer = tf.summary.FileWriter(train_log, sess.graph)
logger.info("start training")
loss = []
dev_f1 = []
test_f1 = []
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss, merged = model.run_step(
sess, True, batch) # step是global step
# 在迭代中输出到结果
train_writer.add_summary(merged, step)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
# use dev data to validation the model
best, dev_f1_value = evaluate(sess, model, "dev", dev_manager,
id_to_tag, logger)
# store the dev f1
dev_f1.append(dev_f1_value)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
# use current the model to test
_, test_f1_value = evaluate(sess, model, "test", test_manager,
id_to_tag, logger)
# store the test f1
test_f1.append(test_f1_value)
# write the dev_f1 and test_f1 to file
f1_result = {}
f1_result["dev_f1"] = dev_f1
f1_result["test_f1"] = test_f1
write_data_to_file(f1_result, "f1_result")
def matrixToInputFile(input_filename,
num,
kingdom_names,
starting_kingdom,
matrix_data,
input_directory="in"):
input_file = f'{input_directory}/{input_filename}'
if not os.path.exists(input_directory):
os.makedirs(input_directory)
utils.write_data_to_file(input_file, [num], '\n')
utils.write_data_to_file(input_file, kingdom_names, ' ', append=True)
utils.write_data_to_file(input_file, '\n', '', append=True)
utils.write_data_to_file(input_file, [starting_kingdom], '\n', append=True)
for row in matrix_data:
utils.write_data_to_file(input_file, row, ' ', append=True)
utils.write_data_to_file(input_file, '\n', '', append=True)
def compare_by_file(input_file, compare1, compare2, output_directory, params=[]):
print('Processing', input_file)
global fromcomp1
global fromcomp2
global totalcost1
global totalcost2
global totalcostcombined
global equallygood
input_data = utils.read_file(input_file)
compare_data1 = utils.read_file(compare1)
compare_data2 = utils.read_file(compare2)
number_of_kingdoms, list_of_kingdom_names, starting_kingdom, adjacency_matrix = data_parser(input_data)
G = adjacency_matrix_to_graph(adjacency_matrix)
comp1_kingdom_tour_name = compare_data1[0]
comp1_conquered_kingdoms_name = compare_data1[1]
comp2_kingdom_tour_name = compare_data2[0]
comp2_conquered_kingdoms_name = compare_data2[1]
comp1_kingdom_tour_index = convert_kingdom_names_to_indices(comp1_kingdom_tour_name, list_of_kingdom_names)
comp1_conquered_kingdoms_index = convert_kingdom_names_to_indices(comp1_conquered_kingdoms_name, list_of_kingdom_names)
comp2_kingdom_tour_index = convert_kingdom_names_to_indices(comp2_kingdom_tour_name, list_of_kingdom_names)
comp2_conquered_kingdoms_index = convert_kingdom_names_to_indices(comp2_conquered_kingdoms_name, list_of_kingdom_names)
cost1 = cost_of_solution(G, comp1_kingdom_tour_index, comp1_conquered_kingdoms_index)[0]
cost2 = cost_of_solution(G, comp2_kingdom_tour_index, comp2_conquered_kingdoms_index)[0]
if cost1 == 'infinite':
print(compare1)
raise Exception("not a valid outputs!!!")
elif cost2 =='infinite':
print(compare2)
raise Exception("not a valid outputs!!!")
basename, filename = os.path.split(input_file)
output_filename = utils.input_to_output(filename)
output_file = '{}/{}'.format(output_directory, output_filename)
if not os.path.exists(output_directory):
os.makedirs(output_directory)
totalcost1 += cost1
totalcost2 += cost2
if cost1 < cost2:
totalcostcombined += cost1
fromcomp1 += 1
utils.write_data_to_file(output_file, comp1_kingdom_tour_name, ' ')
utils.write_to_file(output_file, '\n', append=True)
utils.write_data_to_file(output_file, comp1_conquered_kingdoms_name, ' ', append=True)
else :
if cost1 == cost2 :
equallygood +=1
else:
fromcomp2 += 1
totalcostcombined += cost2
utils.write_data_to_file(output_file, comp2_kingdom_tour_name, ' ')
utils.write_to_file(output_file, '\n', append=True)
utils.write_data_to_file(output_file, comp2_conquered_kingdoms_name, ' ', append=True)
def store_in_cache(data, cache_path): utils.write_data_to_file(json.dumps(data), cache_path)
