def main():
argv = sys.argv
if len(argv) == 3:
model_path, data_file = argv[1:]
evaluator = Evaluator(model_path, data_file)
evaluator.evaluate()
else:
print('Usage: "python evaluate.py $model_path $data_file"')
def main(argv):
# initialize evaluator and evaluate
evaluator = Evaluator(
FLAGS.experiment_path,
FLAGS.src_test_path,
FLAGS.tgt_test_path,
FLAGS.save_as_pretrained,
)
evaluator.evaluate()
def test_general(self):
ev = Evaluator()
possible_keywords = [
'family', 'food', 'outdoor', 'rest', 'indoor', 'sports', 'science',
'culture', 'history'
]
dg = DataGenerator(possible_keywords)
gen_query = dg.generate(1)[0]
gen_data = dg.generate(10)
cf1 = Type1(euclidean_distance,
combined_cosine_similarity,
0.33,
0.33,
0.33,
disable_thresholds=True)
cf2 = Type2(euclidean_distance,
combined_cosine_similarity,
0.33,
0.33,
0.33,
disable_thresholds=True)
cf3 = Type3(euclidean_distance,
combined_cosine_similarity,
0.33,
0.33,
0.33,
disable_thresholds=True)
ns1 = NaiveSolver(gen_query,
gen_data,
cf1,
result_length=10,
max_subset_size=6)
ns2 = NaiveSolver(gen_query,
gen_data,
cf2,
result_length=10,
max_subset_size=6)
ns3 = NaiveSolver(gen_query,
gen_data,
cf3,
result_length=10,
max_subset_size=6)
ev.add_solver(ns1)
ev.add_solver(ns2)
ev.add_solver(ns3)
ev.evaluate()
results = ev.get_results()
self.assertEqual(len(results), 3)
self.assertEqual(len(results[0]), 2)
self.assertEqual(len(results[1]), 2)
self.assertEqual(len(results[2]), 2)
self.assertEqual(len(results[0][0]), 10)
self.assertEqual(len(results[1][0]), 10)
self.assertEqual(len(results[2][0]), 10)
def score_embedding(embedding, groups):
evaluator = Evaluator(groups)
evaluator.evaluate(embedding)
print(" RESULTS")
print("==============")
print("OPP score: %f" % evaluator.opp)
print("Accuracy: %f" % evaluator.accuracy)
print("---------------------------------")
print("Total number of test groups: %d" % evaluator.num_total_groups)
print("Number of filtered test groups: %d (%f%%)" % (evaluator.num_filtered_groups, evaluator.percent_filtered_groups))
print("Total number of non-OOV test cases: %d" % evaluator.num_cases)
print("Number of filtered cluster entities: %d/%d (mean per %% cluster: %f%%)" % (evaluator.num_filtered_cluster_items, evaluator.num_total_cluster_items, evaluator.percent_filtered_cluster_items))
print("Number of filtered outlier entities: %d/%d (mean per %% cluster: %f%%)" % (evaluator.num_filtered_outliers, evaluator.num_total_outliers, evaluator.percent_filtered_outliers))
def evaluate(self):
logger.info('Begin evaluate of Impulso')
if self.args.experiment_id:
module = importlib.import_module(
f'experiments.{self.args.experiment_id}.src.evaluator')
Evaluator = getattr(module, 'Evaluator')
else:
from src.evaluator import Evaluator
evaluator = Evaluator(self.args.exec_type, self.hparams)
evaluator.load_data()
evaluator.evaluate()
logger.info('End evaluate of Impulso')
def run(args) -> None:
"""
This function parses the arguments provided and runs the experiment
:param args: the arguments provided by the user.
:return: None
"""
selector = GeneratorSelector()
generators = selector.getAllGenerators()
benchmark = Benchmark(benchmarkType=args.benchmark, verbose=args.verbose)
evaluator = Evaluator(benchmark=benchmark,
generators=generators,
timeLimit=benchmark.timeLimit,
verbose=args.verbose)
evaluator.evaluate()
sys.exit(0)
def search():
if request.method == 'POST':
screen_name = request.form['screen_name']
evaluator = Evaluator(app.root_path)
status, result = evaluator.evaluate(screen_name)
if status != 200:
raise Err(result, status_code=status)
return json.dumps(result)
else:
return False
help="the number of orders to parse", type=int, default=2400)
parser.add_argument("--process_num", dest="NUM_PROCESSES",
help="the number of processes", type=int, default=16)
parser.add_argument("--weight_amount", dest="WEIGHT_AMOUNT",
help="the weight of the amountCost in the objective function", type=float, default=15)
parser.add_argument("--weight_time", dest="WEIGHT_TIME",
help="the weight of the timeCost in the objective function", type=float, default=1)
parser.add_argument("--hub_cost_const", dest="HUB_BUILT_COST_CONST",
help="the constant part of cost of building a hub", type=int, default=3000)
parser.add_argument("--hub_cost_vary", dest="HUB_BUILT_COST_VARY",
help="the variable part cost of building a hub", type=float, default=2.0)
parser.add_argument("--hub_ratio", dest="HUB_UNIT_COST_RATIO",
help="the cutoff of unit price of a hub", type=float, default=0.7)
parser.add_argument("--hub_capacity", dest="HUB_CAPACITY",
help="the capacity of the hub(in prob3)", type=int, default=500)
args = parser.parse_args()
arg_dict = edict(vars(args))
merge_a_into_b(arg_dict, cfg)
print("Using the config:")
print(cfg)
solver = Solver()
evaluator = Evaluator()
solver.solve(problem_id=args.PROBLEM_ID)
# disable the tune mode but plot the distribution, see the outputs for more detail
evaluator.evaluate(prob_id=args.PROBLEM_ID, tune=False, plot=True)
rw.results_to_file(results_file_sqlm_prf, results_sqlm_prf, 'sqlm_prf')
rw.results_to_file(results_file_bm25, results_bm25, 'bm25')
rw.results_to_file(results_file_bm25_stop, results_bm25_stop, 'bm25_stop')
rw.results_to_file(results_file_bm25_stem, results_bm25_stem, 'bm25_stem')
eval_tfidf = Evaluator(results_file_tfidf, 'tfidf')
eval_tfidf_stop = Evaluator(results_file_tfidf_stop, 'tfidf_stop')
eval_sqlm = Evaluator(results_file_sqlm, 'sqlm')
eval_sqlm_stop = Evaluator(results_file_sqlm_stop, 'sqlm_stop')
eval_sqlm_prf = Evaluator(results_file_sqlm_prf, 'sqlm_prf')
eval_bm25 = Evaluator(results_file_bm25, 'bm25')
eval_bm25_stop = Evaluator(results_file_bm25_stop, 'bm25_stop')
eval_lucene = Evaluator(results_file_lucene, 'lucene')
print('Evaluating results for tf.idf...', end='')
eval_tfidf.evaluate()
eval_tfidf.eval_to_file()
print('Done')
print('Evaluating results for tf.idf with stopping...', end='')
eval_tfidf_stop.evaluate()
eval_tfidf_stop.eval_to_file()
print('Done')
print('Evaluating results for Smoothed Query Likelihood...', end='')
eval_sqlm.evaluate()
eval_sqlm.eval_to_file()
print('Done')
print('Evaluating results for Smoothed Query Likelihood with stopping...',
end='')
def main():
if not os.path.exists(config.result_dir):
os.makedirs(config.result_dir)
if not os.path.exists(config.train_log_dir):
os.makedirs(config.train_log_dir)
if not os.path.exists(config.valid_log_dir):
os.makedirs(config.valid_log_dir)
print('preparing data...')
config.word_2_id, config.id_2_word = read_dict(config.word_dict)
config.attr_2_id, config.id_2_attr = read_dict(config.attr_dict)
config.vocab_size = min(config.vocab_size, len(config.word_2_id))
config.oov_vocab_size = len(config.word_2_id) - config.vocab_size
config.attr_size = len(config.attr_2_id)
embedding_matrix = None
if args.do_train:
if os.path.exists(config.glove_file):
print('loading embedding matrix from file: {}'.format(config.glove_file))
embedding_matrix, config.word_em_size = load_glove_embedding(config.glove_file, list(config.word_2_id.keys()))
print('shape of embedding matrix: {}'.format(embedding_matrix.shape))
else:
if os.path.exists(config.glove_file):
with open(config.glove_file, 'r', encoding='utf-8') as fin:
line = fin.readline()
config.word_em_size = len(line.strip().split()) - 1
data_reader = DataReader(config)
evaluator = Evaluator('description')
print('building model...')
model = get_model(config, embedding_matrix)
saver = tf.train.Saver(max_to_keep=10)
if args.do_train:
print('loading data...')
train_data = data_reader.read_train_data()
valid_data = data_reader.read_valid_data_small()
print_title('Trainable Variables')
for v in tf.trainable_variables():
print(v)
print_title('Gradients')
for g in model.gradients:
print(g)
with tf.Session(config=sess_config) as sess:
model_file = args.model_file
if model_file is None:
model_file = tf.train.latest_checkpoint(config.result_dir)
if model_file is not None:
print('loading model from {}...'.format(model_file))
saver.restore(sess, model_file)
else:
print('initializing from scratch...')
tf.global_variables_initializer().run()
train_writer = tf.summary.FileWriter(config.train_log_dir, sess.graph)
valid_writer = tf.summary.FileWriter(config.valid_log_dir, sess.graph)
run_train(sess, model, train_data, valid_data, saver, evaluator, train_writer, valid_writer, verbose=True)
if args.do_eval:
print('loading data...')
valid_data = data_reader.read_valid_data()
with tf.Session(config=sess_config) as sess:
model_file = args.model_file
if model_file is None:
model_file = tf.train.latest_checkpoint(config.result_dir)
if model_file is not None:
print('loading model from {}...'.format(model_file))
saver.restore(sess, model_file)
predicted_ids, alignment_history, valid_loss, valid_accu = run_evaluate(sess, model, valid_data, verbose=True)
print('average valid loss: {:>.4f}, average valid accuracy: {:>.4f}'.format(valid_loss, valid_accu))
print_title('Saving Result')
save_result(predicted_ids, alignment_history, config.id_2_word, config.valid_data, config.valid_result)
evaluator.evaluate(config.valid_data, config.valid_result, config.to_lower)
else:
print('model not found!')
if args.do_test:
print('loading data...')
test_data = data_reader.read_test_data()
with tf.Session(config=sess_config) as sess:
model_file = args.model_file
if model_file is None:
model_file = tf.train.latest_checkpoint(config.result_dir)
if model_file is not None:
print('loading model from {}...'.format(model_file))
saver.restore(sess, model_file)
predicted_ids, alignment_history = run_test(sess, model, test_data, verbose=True)
print_title('Saving Result')
save_result(predicted_ids, alignment_history, config.id_2_word, config.test_data, config.test_result)
evaluator.evaluate(config.test_data, config.test_result, config.to_lower)
else:
print('model not found!')
def main(argv):
start_time = time.time()
# Evaluator, instantiate it first for logging purposes
ev = Evaluator()
query: KeywordCoordinate = load_pickle('query.pickle')
print('Query:', query)
data: dataset_type = load_pickle('dataset.pickle')
# print('Data:', dataset_comprehension(data))
# Let's filter out by user radius
# dataAux = sorted(data, key=lambda x: geographic_distance(x.coordinates, query.coordinates))
# distances = [geographic_distance(x.coordinates, query.coordinates) >= RADIUS for x in dataAux]
# print('------ Distances: ', distances)
# Load precalculated values and models
precalculated_inter_dataset_distances = load_pickle(
'precalculated_inter_dataset_distances.pickle')
precalculated_query_dataset_distances = load_pickle(
'precalculated_query_dataset_distances.pickle')
precalculated_query_dataset_keyword_similarities = load_pickle(
'precalculated_query_dataset_keyword_similarities.pickle')
# **** ONLY FOR word2vec model executions
precalculated_query_dataset_keyword_similarities_word2vec = load_pickle(
'precalculated_query_dataset_keyword_similarities_word2vec.pickle')
word2vec_model = load_word2vec_model('word2vec_model.pickle')
# ****
# Define the CostFunctions. For all possible parameters refer to the documentation.
cf1 = Type1(euclidean_distance, combined_cosine_similarity, 0.2, 0.1, 0.7)
cf2 = Type2(euclidean_distance,
word2vec_cosine_similarity,
0.2,
0.1,
0.7,
model=word2vec_model)
cf3 = Type1(
euclidean_distance,
combined_cosine_similarity,
0.2,
0.1,
0.7,
precalculated_inter_dataset_dict=precalculated_inter_dataset_distances,
precalculated_query_dataset_dict=precalculated_query_dataset_distances,
precalculated_keyword_similarity_dict=
precalculated_query_dataset_keyword_similarities)
cf4 = Type2(
euclidean_distance,
word2vec_cosine_similarity,
0,
0,
1.0,
precalculated_inter_dataset_dict=precalculated_inter_dataset_distances,
precalculated_query_dataset_dict=precalculated_query_dataset_distances,
precalculated_keyword_similarity_dict=
precalculated_query_dataset_keyword_similarities_word2vec,
model=word2vec_model)
cf5 = Type3(
euclidean_distance,
word2vec_cosine_similarity,
0.1,
0.1,
0.8,
precalculated_inter_dataset_dict=precalculated_inter_dataset_distances,
precalculated_query_dataset_dict=precalculated_query_dataset_distances,
precalculated_keyword_similarity_dict=
precalculated_query_dataset_keyword_similarities_word2vec,
model=word2vec_model)
cf6 = Type1(
euclidean_distance,
word2vec_cosine_similarity,
0.2,
0.1,
0.7,
precalculated_inter_dataset_dict=precalculated_inter_dataset_distances,
precalculated_query_dataset_dict=precalculated_query_dataset_distances,
precalculated_keyword_similarity_dict=
precalculated_query_dataset_keyword_similarities,
model=word2vec_model)
map_name = argv[0]
# map_name = 'London_mini'
# Choose which Solvers to use. For all possible parameters refer to the documentation.
max_number_of_processes = mp.cpu_count()
ns1 = NaiveSolver(
query,
data,
cf2,
result_length=5,
max_subset_size=3,
max_number_of_concurrent_processes=max_number_of_processes,
_map=map_name)
# ns2 = NaiveSolver(query, data, cf5, result_length=5, max_subset_size=3,
# max_number_of_concurrent_processes=max_number_of_processes, _map = map_name)
# ns3 = NaiveSolver(query, data, cf3, result_length=5, max_subset_size=6,
# max_number_of_concurrent_processes=max_number_of_processes)
# ns4 = NaiveSolver(query, data, cf6, result_length=5, max_subset_size=3,
# max_number_of_concurrent_processes=max_number_of_processes, _map = map_name)
# Add Solvers to Evaluator
ev.add_solver(ns1)
# ev.add_solver(ns2)
# ev.add_solver(ns4)
#Only for Debug: calculates and print physical distances between items in the dataset and the query location
#distances = [geographic_distance(x.coordinates, query.coordinates) for x in data]
# print('------ Distances: ', distances
# Run Evaluator and fetch results
ev.evaluate()
results = ev.get_results()
timings = ev.get_timings()
write_csv(map_name, results, timings)
print('*** Solution -', solution_list_comprehension(results))
# print('*** Timing -', timing_list_comprehension(timings))
initialLat = []
initialLon = []
keywords = []
gmap = gmplot.GoogleMapPlotter(query.coordinates.x, query.coordinates.y,
14)
colors = ['red', 'blue', 'green', 'purple', 'orange']
# Third dimension is the order of solution (Best: 0, Second best: 1...)
for i in range(5):
lats = []
lons = []
for kwc in results[0][0][i][1]:
lats.append(kwc.coordinates.x)
lons.append(kwc.coordinates.y)
keywords.append(kwc.keywords)
for j in range(len(lats)):
gmap.marker(lats[j], lons[j], color=colors[i])
gmap.polygon(lats, lons, color='cornflowerblue', edge_width=7)
# initialLat.append(query.coordinates.x)
# initialLon.append(query.coordinates.y)
# gmap.scatter(initialLat, initialLon, '#00FF00', size = 70, marker = False)
# gmap.scatter(lats, lons, '#FF0000',size = 50, marker = False )
# gmap.plot(lats, lons, 'cornflowerblue', edge_width = 3.0)
# gmap.polygon(lats, lons, color='cornflowerblue', edge_width=10)
# gmap.scatter(lats, lons, color='#3B0B39', size=40, marker=False)
#Your Google_API_Key
#gmap.apikey = " API_Key”
# save it to html
# gmap.scatter(lats, lons, '#FF0000', size=40, marker=True)
gmap.marker(query.coordinates.x,
query.coordinates.y,
color='cornflowerblue',
title='Query point')
gmap.draw(r"graphic_results.html")
print("--- %s seconds ---" % (time.time() - start_time))
# ns3 = NaiveSolver(query, data, cf3, result_length=5, max_subset_size=6,
# max_number_of_concurrent_processes=max_number_of_processes)
# ns4 = NaiveSolver(query, data, cf6, result_length=5, max_subset_size=3,
# max_number_of_concurrent_processes=max_number_of_processes, _map = map_name)
# Add Solvers to Evaluator
ev.add_solver(ns1)
# ev.add_solver(ns2)
# ev.add_solver(ns4)
#Only for Debug: calculates and print physical distances between items in the dataset and the query location
#distances = [geographic_distance(x.coordinates, query.coordinates) for x in data]
# print('------ Distances: ', distances
# Run Evaluator and fetch results
ev.evaluate()
results = ev.get_results()
timings = ev.get_timings()
write_csv(map_name, results, timings)
print('*** Solution -', solution_list_comprehension(results))
# print('*** Timing -', timing_list_comprehension(timings))
# initialLat = []
# initialLon = []
# keywords = []
class GeneticEngine:
def __init__(self, data):
self._evaluator = Evaluator(data)
self._item_count = self._evaluator.generator.rule_set_len
self._best_ind = None
self._best_fit_val = 0
# To assure reproductibility, the RNG seed is set prior to the items
# dict initialization. It is also seeded in main().
random.seed(64)
# # Create the item dictionary: item name is an integer, and value is
# # a weight.
# items = {}
# # Create random items and store them in the items' dictionary. (18 rule * 9)
# for i in range(NBR_ITEMS):
# items[i] = i;
creator.create("FitnessMax", base.Fitness, weights=(1.0, ))
creator.create("Individual", set, fitness=creator.FitnessMax)
self.toolbox = base.Toolbox()
# Attribute generator
# define 'attr_item' to be an attribute ('gene')
# which corresponds to integers sampled uniformly
# from the range [1,10] (i.e. 1 to 10 with equal probability)
self.toolbox.register("attr_item", random.randrange, self._item_count)
# Structure initializers
# define 'individual' to be an individual
# consisting of 10 'attr_item' elements ('genes')
self.toolbox.register("individual", tools.initRepeat,
creator.Individual, self.toolbox.attr_item,
IND_INIT_SIZE)
# define the population to be a list of individuals
self.toolbox.register("population", tools.initRepeat, list,
self.toolbox.individual)
self.toolbox.register("evaluate", self.eval_ind)
self.toolbox.register("mate", self.cx_ind)
self.toolbox.register("mutate", self.mutate_ind)
self.toolbox.register("select", tools.selNSGA2)
@property
def evaluator(self):
return self._evaluator
def eval_ind(self, ind):
"""
calculate the fitness of the individual
:param ind: the individual Chromosome object to be evaluated
:return: the fitness value
"""
print("\n:::: [genetic] individual", ind, "::::")
start = dt.now()
fit_val = self._evaluator.evaluate(ind)
if fit_val > self._best_fit_val:
self._best_ind = ind
self._best_fit_val = fit_val
print(":::: [genetic] Evaluate individual. fitness value", fit_val,
"Duration",
dt.now() - start, "::::\n")
return fit_val, None
def mutate_ind(self, ind, mu=0, sigma=4, chance_mutation=0.4):
"""
Mutate the individual by changing the Chromosome composition
:param mu:
:param sigma:
:return:
"""
if random.random() < chance_mutation:
if len(ind) > 0: # We cannot pop from an empty set
ind.remove(random.choice(sorted(tuple(ind))))
else:
ind.add(random.randrange(self._item_count))
return ind
def cx_ind(self, ind1, ind2, chance_crossover=0.7):
"""Apply a crossover operation on input sets. The first child is the
intersection of the two sets, the second child is the difference of the
two sets.
"""
if random.random() < chance_crossover:
temp = set(ind1) # Used in order to keep type
ind1 &= ind2 # Intersection (inplace)
ind2 ^= temp # Symmetric Difference (inplace)
return ind1, ind2
def verify_ind(self, ind):
"""
Verify the validity of the individual Chromosome
:param ind: the individual Chromosome to be verified
:return: True if the Chromosome is valid, False otherwise
"""
# TODO: will it be better if we define the method as the member method of Chromosome?
return True
def best_ind(self):
"""
Get the best individual after the GA calculation
:return: the best individual Chromosome
"""
random.seed(64)
NGEN = 50
MU = 50
LAMBDA = 100
CXPB = 0.7
MUTPB = 0.2
pop = self.toolbox.population(n=MU)
hof = tools.ParetoFront()
stats = tools.Statistics(lambda ind: ind.fitness.values)
stats.register("avg", numpy.mean, axis=0)
stats.register("std", numpy.std, axis=0)
stats.register("min", numpy.min, axis=0)
stats.register("max", numpy.max, axis=0)
try:
algorithms.eaMuPlusLambda(pop,
self.toolbox,
MU,
LAMBDA,
CXPB,
MUTPB,
NGEN,
stats,
halloffame=hof)
except Exception as err:
print(err)
return self._best_ind
print("The best individual is :", hof[-1])
print(len(pop))
print(len(hof))
# print("The best fitness is :", eval_ind(self, hof[-1]))
return hof[-1]
def main():
os.makedirs(config.temp_dir, exist_ok=True)
os.makedirs(config.result_dir, exist_ok=True)
os.makedirs(config.train_log_dir, exist_ok=True)
logger.setLevel(logging.INFO)
init_logger(logging.INFO, 'temp.log.txt', 'w')
logger.info('preparing data...')
config.word_2_id, config.id_2_word = read_json_dict(config.vocab_dict)
config.vocab_size = min(config.vocab_size, len(config.word_2_id))
config.oov_vocab_size = min(config.oov_vocab_size,
len(config.word_2_id) - config.vocab_size)
embedding_matrix = None
if args.do_train:
if os.path.exists(config.glove_file):
logger.info('loading embedding matrix from file: {}'.format(
config.glove_file))
embedding_matrix, config.word_em_size = load_glove_embedding(
config.glove_file, list(config.word_2_id.keys()))
logger.info('shape of embedding matrix: {}'.format(
embedding_matrix.shape))
else:
if os.path.exists(config.glove_file):
with open(config.glove_file, 'r', encoding='utf-8') as fin:
line = fin.readline()
config.word_em_size = len(line.strip().split()) - 1
data_reader = DataReader(config)
evaluator = Evaluator('tgt')
logger.info('building model...')
model = get_model(config, embedding_matrix)
saver = tf.train.Saver(max_to_keep=10)
if args.do_train:
logger.info('loading data...')
train_data = data_reader.read_train_data()
valid_data = data_reader.read_valid_data()
logger.info(log_title('Trainable Variables'))
for v in tf.trainable_variables():
logger.info(v)
logger.info(log_title('Gradients'))
for g in model.gradients:
logger.info(g)
with tf.Session(config=sess_config) as sess:
model_file = args.model_file
if model_file is None:
model_file = tf.train.latest_checkpoint(
os.path.join(config.result_dir, config.current_model))
if model_file is not None:
logger.info('loading model from {}...'.format(model_file))
saver.restore(sess, model_file)
else:
logger.info('initializing from scratch...')
tf.global_variables_initializer().run()
train_writer = tf.summary.FileWriter(config.train_log_dir,
sess.graph)
valid_log_history = run_train(sess, model, train_data, valid_data,
saver, evaluator, train_writer)
save_json(
valid_log_history,
os.path.join(config.result_dir, config.current_model,
'valid_log_history.json'))
if args.do_eval:
logger.info('loading data...')
valid_data = data_reader.read_valid_data()
with tf.Session(config=sess_config) as sess:
model_file = args.model_file
if model_file is None:
model_file = tf.train.latest_checkpoint(
os.path.join(config.result_dir, config.current_model))
if model_file is not None:
logger.info('loading model from {}...'.format(model_file))
saver.restore(sess, model_file)
predicted_ids, valid_loss, valid_accu = run_evaluate(
sess, model, valid_data)
logger.info(
'average valid loss: {:>.4f}, average valid accuracy: {:>.4f}'
.format(valid_loss, valid_accu))
logger.info(log_title('Saving Result'))
save_outputs(predicted_ids, config.id_2_word,
config.valid_data, config.valid_outputs)
results = evaluator.evaluate(config.valid_data,
config.valid_outputs,
config.to_lower)
save_json(results, config.valid_results)
else:
logger.info('model not found!')
if args.do_test:
logger.info('loading data...')
test_data = data_reader.read_test_data()
with tf.Session(config=sess_config) as sess:
model_file = args.model_file
if model_file is None:
model_file = tf.train.latest_checkpoint(
os.path.join(config.result_dir, config.current_model))
if model_file is not None:
logger.info('loading model from {}...'.format(model_file))
saver.restore(sess, model_file)
predicted_ids = run_test(sess, model, test_data)
logger.info(log_title('Saving Result'))
save_outputs(predicted_ids, config.id_2_word, config.test_data,
config.test_outputs)
results = evaluator.evaluate(config.test_data,
config.test_outputs,
config.to_lower)
save_json(results, config.test_results)
else:
logger.info('model not found!')
def main():
os.makedirs(config.temp_dir, exist_ok=True)
os.makedirs(config.result_dir, exist_ok=True)
os.makedirs(config.train_log_dir, exist_ok=True)
logger.setLevel(logging.INFO)
init_logger(logging.INFO)
logger.info('loading dict...')
config.src_2_id, config.id_2_src = read_json_dict(config.src_vocab_dict)
config.src_vocab_size = min(config.src_vocab_size, len(config.src_2_id))
config.tgt_2_id, config.id_2_tgt = read_json_dict(config.tgt_vocab_dict)
config.tgt_vocab_size = min(config.tgt_vocab_size, len(config.tgt_2_id))
data_reader = DataReader(config)
evaluator = Evaluator('tgt')
logger.info('building model...')
model = get_model(config)
saver = tf.train.Saver(max_to_keep=10)
if args.do_train:
logger.info('loading data...')
train_data = data_reader.load_train_data()
valid_data = data_reader.load_valid_data()
logger.info(log_title('Trainable Variables'))
for v in tf.trainable_variables():
logger.info(v)
logger.info(log_title('Gradients'))
for g in model.gradients:
logger.info(g)
with tf.Session(config=sess_config) as sess:
model_file = args.model_file
if model_file is None:
model_file = tf.train.latest_checkpoint(
os.path.join(config.result_dir, config.current_model))
if model_file is not None:
logger.info('loading model from {}...'.format(model_file))
saver.restore(sess, model_file)
else:
logger.info('initializing from scratch...')
tf.global_variables_initializer().run()
train_writer = tf.summary.FileWriter(config.train_log_dir,
sess.graph)
valid_log_history = run_train(sess, model, train_data, valid_data,
saver, evaluator, train_writer)
save_json(
valid_log_history,
os.path.join(config.result_dir, config.current_model,
'valid_log_history.json'))
if args.do_eval:
logger.info('loading data...')
valid_data = data_reader.load_valid_data()
with tf.Session(config=sess_config) as sess:
model_file = args.model_file
if model_file is None:
model_file = tf.train.latest_checkpoint(
os.path.join(config.result_dir, config.current_model))
if model_file is not None:
logger.info('loading model from {}...'.format(model_file))
saver.restore(sess, model_file)
predicted_ids, valid_loss, valid_accu = run_evaluate(
sess, model, valid_data)
logger.info(
'average valid loss: {:>.4f}, average valid accuracy: {:>.4f}'
.format(valid_loss, valid_accu))
logger.info(log_title('Saving Result'))
save_outputs(predicted_ids, config.id_2_tgt, config.valid_data,
config.valid_outputs)
results = evaluator.evaluate(config.valid_data,
config.valid_outputs,
config.to_lower)
save_json(results, config.valid_results)
else:
logger.info('model not found!')
if args.do_test:
logger.info('loading data...')
test_data = data_reader.load_test_data()
with tf.Session(config=sess_config) as sess:
model_file = args.model_file
if model_file is None:
model_file = tf.train.latest_checkpoint(
os.path.join(config.result_dir, config.current_model))
if model_file is not None:
logger.info('loading model from {}...'.format(model_file))
saver.restore(sess, model_file)
predicted_ids = run_test(sess, model, test_data)
logger.info(log_title('Saving Result'))
save_outputs(predicted_ids, config.id_2_tgt, config.test_data,
config.test_outputs)
results = evaluator.evaluate(config.test_data,
config.test_outputs,
config.to_lower)
save_json(results, config.test_results)
else:
logger.info('model not found!')