def train(self):
n_iter = 0
for epoch in range(self.configs["n_epoch"]):
logger.info("Epoch: {}".format(epoch))
epoch_loss = 0
for values, indicators_train, indicators_eval in tqdm(
self.iterator):
values = values.to(self.device)
indicators_train = indicators_train.to(self.device)
self.opt.zero_grad()
y, y_pred, indicators_y = self._iter(
values=values, indicators=indicators_train)
iter_loss = self.get_loss(y=y,
y_pred=y_pred,
indicators=indicators_y)
iter_loss.backward()
self.opt.step()
n_iter += 1
epoch_loss += iter_loss
logger.info("Training loss: {}".format(epoch_loss /
len(self.iterator)))
if self.is_eval:
evaluator = Evaluator(configs=None,
model=self.model,
iterator=self.iterator_eval)
eval_loss = evaluator.eval()
logger.info("Eval loss: {}".format(eval_loss))
if self.early_stopping(loss=eval_loss):
return
else:
if self.early_stopping(loss=epoch_loss / len(self.iterator)):
return
def test_store_multiple(self):
s = State()
s.append(PredicateInstance('happy', Instance('a')))
s.append(PredicateInstance('unhappy', Instance('a')))
self.assertStateContains(PredicateInstance('happy', Instance('a')),
Evaluator(state=s))
self.assertStateContains(PredicateInstance('unhappy', Instance('a')),
Evaluator(state=s))
def main():
argv = sys.argv
if len(argv) == 3:
model_path, data_file = argv[1:]
evaluator = Evaluator(model_path, data_file)
evaluator.evaluate()
else:
print('Usage: "python evaluate.py $model_path $data_file"')
def test():
assert cfg.load_path is not None
output_model_params()
loadFile = True
ifLoad, data = False, None
if loadFile:
ifLoad, data = load_file(cfg.processed_path, 'processed data',
'pickle')
if not ifLoad or not loadFile:
raise RuntimeError('cannot find pre-processed dataset')
else:
train_data_obj = data['train_data_obj']
dev_data_obj = data['dev_data_obj']
test_data_obj = data['test_data_obj']
train_data_obj.filter_data('test')
dev_data_obj.filter_data('test')
test_data_obj.filter_data('test')
emb_mat_token, emb_mat_glove = train_data_obj.emb_mat_token, train_data_obj.emb_mat_glove
with tf.variable_scope(network_type) as scope:
if network_type in model_type_set:
model = Model(emb_mat_token, emb_mat_glove,
len(train_data_obj.dicts['token']),
len(train_data_obj.dicts['char']),
train_data_obj.max_lens['token'], scope.name)
graphHandler = GraphHandler(model)
evaluator = Evaluator(model)
if cfg.gpu_mem is None:
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=cfg.gpu_mem, allow_growth=True)
graph_config = tf.ConfigProto(gpu_options=gpu_options,
allow_soft_placement=True)
else:
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=cfg.gpu_mem)
graph_config = tf.ConfigProto(gpu_options=gpu_options)
# graph_config.gpu_options.allow_growth = True
sess = tf.Session(config=graph_config)
graphHandler.initialize(sess)
# todo: test model
# ---- dev ----
dev_loss, dev_accu = evaluator.get_evaluation(sess, dev_data_obj, None)
_logger.add('==> for dev, loss: %.4f, accuracy: %.4f' %
(dev_loss, dev_accu))
# ---- test ----
test_loss, test_accu = evaluator.get_evaluation(sess, test_data_obj, None)
_logger.add('~~> for test, loss: %.4f, accuracy: %.4f' %
(test_loss, test_accu))
train_loss, train_accu = evaluator.get_evaluation(sess, train_data_obj,
None)
_logger.add('--> for test, loss: %.4f, accuracy: %.4f' %
(train_loss, train_accu))
def search():
if request.method == 'POST':
screen_name = request.form['screen_name']
evaluator = Evaluator(app.root_path)
status, result = evaluator.evaluate(screen_name)
if status != 200:
raise Err(result, status_code=status)
return json.dumps(result)
else:
return False
def main(argv):
# initialize evaluator and evaluate
evaluator = Evaluator(
FLAGS.experiment_path,
FLAGS.src_test_path,
FLAGS.tgt_test_path,
FLAGS.save_as_pretrained,
)
evaluator.evaluate()
def score_embedding(embedding, groups):
evaluator = Evaluator(groups)
evaluator.evaluate(embedding)
print(" RESULTS")
print("==============")
print("OPP score: %f" % evaluator.opp)
print("Accuracy: %f" % evaluator.accuracy)
print("---------------------------------")
print("Total number of test groups: %d" % evaluator.num_total_groups)
print("Number of filtered test groups: %d (%f%%)" % (evaluator.num_filtered_groups, evaluator.percent_filtered_groups))
print("Total number of non-OOV test cases: %d" % evaluator.num_cases)
print("Number of filtered cluster entities: %d/%d (mean per %% cluster: %f%%)" % (evaluator.num_filtered_cluster_items, evaluator.num_total_cluster_items, evaluator.percent_filtered_cluster_items))
print("Number of filtered outlier entities: %d/%d (mean per %% cluster: %f%%)" % (evaluator.num_filtered_outliers, evaluator.num_total_outliers, evaluator.percent_filtered_outliers))
def test_general(self):
ev = Evaluator()
possible_keywords = [
'family', 'food', 'outdoor', 'rest', 'indoor', 'sports', 'science',
'culture', 'history'
]
dg = DataGenerator(possible_keywords)
gen_query = dg.generate(1)[0]
gen_data = dg.generate(10)
cf1 = Type1(euclidean_distance,
combined_cosine_similarity,
0.33,
0.33,
0.33,
disable_thresholds=True)
cf2 = Type2(euclidean_distance,
combined_cosine_similarity,
0.33,
0.33,
0.33,
disable_thresholds=True)
cf3 = Type3(euclidean_distance,
combined_cosine_similarity,
0.33,
0.33,
0.33,
disable_thresholds=True)
ns1 = NaiveSolver(gen_query,
gen_data,
cf1,
result_length=10,
max_subset_size=6)
ns2 = NaiveSolver(gen_query,
gen_data,
cf2,
result_length=10,
max_subset_size=6)
ns3 = NaiveSolver(gen_query,
gen_data,
cf3,
result_length=10,
max_subset_size=6)
ev.add_solver(ns1)
ev.add_solver(ns2)
ev.add_solver(ns3)
ev.evaluate()
results = ev.get_results()
self.assertEqual(len(results), 3)
self.assertEqual(len(results[0]), 2)
self.assertEqual(len(results[1]), 2)
self.assertEqual(len(results[2]), 2)
self.assertEqual(len(results[0][0]), 10)
self.assertEqual(len(results[1][0]), 10)
self.assertEqual(len(results[2][0]), 10)
def run(args) -> None:
"""
This function parses the arguments provided and runs the experiment
:param args: the arguments provided by the user.
:return: None
"""
selector = GeneratorSelector()
generators = selector.getAllGenerators()
benchmark = Benchmark(benchmarkType=args.benchmark, verbose=args.verbose)
evaluator = Evaluator(benchmark=benchmark,
generators=generators,
timeLimit=benchmark.timeLimit,
verbose=args.verbose)
evaluator.evaluate()
sys.exit(0)
def run_one(config):
# create work dir for this run
# moses can't handle paths with colons
timestamp = datetime.datetime.now().strftime('%Y-%m-%dT%H.%M.%S')
run_work_dir = os.path.join(base_work_dir, timestamp)
assert not os.path.exists(run_work_dir)
os.makedirs(run_work_dir)
config.put('work_dir', run_work_dir)
if os.path.exists('latest'):
os.remove('latest')
os.symlink(run_work_dir, 'latest')
# set up logging
if config.run == 'debug':
logging.basicConfig(level=logging.DEBUG)
else:
log_path = os.path.join(run_work_dir, LOGFILE_NAME)
logging.basicConfig(filename=log_path, level=logging.INFO)
experiment = SMTSemparseExperiment(config)
if config.run == 'debug':
experiment.run_fold(1)
elif config.run == 'dev':
for i in range(10):
experiment.run_fold(i)
elif config.run == 'test':
experiment.run_split()
else:
assert False
if not config.nlg:
logging.info('evaluating')
Evaluator(config).run()
def set_local_parameters(self, random_seed, losses, metrics,
loss_weights, checkpoint_every, print_every):
self.random_seed = random_seed
if random_seed is not None:
random.seed(random_seed)
torch.manual_seed(random_seed)
self.losses = losses
self.metrics = metrics
self.loss_weights = loss_weights or len(losses)*[1.]
self.evaluator = Evaluator(loss=self.losses, metrics=self.metrics)
self.optimizer = None
self.checkpoint_every = checkpoint_every
self.print_every = print_every
self.logger = logging.getLogger(__name__)
self._stop_training = False
def test():
assert cfg.load_path is not None
output_model_params()
loadFile = True
ifLoad, data = False, None
if loadFile:
ifLoad, data = load_file(cfg.processed_path, 'processed data',
'pickle')
if not ifLoad or not loadFile:
raise RuntimeError('cannot find pre-processed dataset')
else:
train_data_obj = data['train_data_obj']
dev_data_obj = data['dev_data_obj']
test_data_obj = data['test_data_obj']
train_data_obj.filter_data()
dev_data_obj.filter_data()
test_data_obj.filter_data()
emb_mat_token, emb_mat_glove = train_data_obj.emb_mat_token, train_data_obj.emb_mat_glove
with tf.variable_scope(network_type) as scope:
model = Model(emb_mat_token, emb_mat_glove,
len(train_data_obj.dicts['token']),
len(train_data_obj.dicts['char']),
train_data_obj.max_lens['token'], scope.name)
graphHandler = GraphHandler(model)
evaluator = Evaluator(model)
sess = tf.Session(config=sess_config_gene(cfg.gpu_mem))
graphHandler.initialize(sess)
# test model
# ---- dev ----
dev_loss, dev_accu = evaluator.get_evaluation(sess, dev_data_obj, None)
_logger.add('==> for dev, loss: %.4f, accuracy: %.4f' %
(dev_loss, dev_accu))
# ---- test ----
test_loss, test_accu = evaluator.get_evaluation(sess, test_data_obj, None)
_logger.add('~~> for test, loss: %.4f, accuracy: %.4f' %
(test_loss, test_accu))
train_loss, train_accu = evaluator.get_evaluation(sess, train_data_obj,
None)
_logger.add('--> for test, loss: %.4f, accuracy: %.4f' %
(train_loss, train_accu))
def inference_time():
assert cfg.load_path is not None
output_model_params()
loadFile = True
ifLoad, data = False, None
if loadFile:
ifLoad, data = load_file(cfg.processed_path, 'processed data',
'pickle')
if not ifLoad or not loadFile:
raise RuntimeError('cannot find pre-processed dataset')
else:
train_data_obj = data['train_data_obj']
dev_data_obj = data['dev_data_obj']
test_data_obj = data['test_data_obj']
train_data_obj.filter_data()
dev_data_obj.filter_data()
test_data_obj.filter_data()
emb_mat_token, emb_mat_glove = train_data_obj.emb_mat_token, train_data_obj.emb_mat_glove
with tf.variable_scope(network_type) as scope:
model = Model(emb_mat_token, emb_mat_glove,
len(train_data_obj.dicts['token']),
len(train_data_obj.dicts['char']),
train_data_obj.max_lens['token'], scope.name)
graphHandler = GraphHandler(model)
evaluator = Evaluator(model)
sess = tf.Session(config=sess_config_gene(cfg.gpu_mem))
graphHandler.initialize(sess)
repeat_num = 10
time_counter = TimeCounter()
for i in range(5):
evaluator.get_evaluation(sess, dev_data_obj, None)
for t in range(repeat_num):
dev_loss, dev_accu = evaluator.get_evaluation(
sess, dev_data_obj, None, time_counter=time_counter)
# _logger.add('~~> for test, loss: %.4f, accuracy: %.4f' %
# (test_loss, test_accu))
time_rec = time_counter.update_data_round(t + 1)
print('this: ', time_rec[0], '; all: ', time_rec[1])
def test_store_and_access(self):
s = State()
p = PredicateInstance('happy', Instance('a'))
s.append(p)
self.assertEqual([p], s.fetch('happy', [Instance('a')]))
self.assertStateContains(PredicateInstance('happy', Instance('a')),
Evaluator(state=s))
def __init__(self, data):
self._evaluator = Evaluator(data)
self._item_count = self._evaluator.generator.rule_set_len
self._best_ind = None
self._best_fit_val = 0
# To assure reproductibility, the RNG seed is set prior to the items
# dict initialization. It is also seeded in main().
random.seed(64)
# # Create the item dictionary: item name is an integer, and value is
# # a weight.
# items = {}
# # Create random items and store them in the items' dictionary. (18 rule * 9)
# for i in range(NBR_ITEMS):
# items[i] = i;
creator.create("FitnessMax", base.Fitness, weights=(1.0, ))
creator.create("Individual", set, fitness=creator.FitnessMax)
self.toolbox = base.Toolbox()
# Attribute generator
# define 'attr_item' to be an attribute ('gene')
# which corresponds to integers sampled uniformly
# from the range [1,10] (i.e. 1 to 10 with equal probability)
self.toolbox.register("attr_item", random.randrange, self._item_count)
# Structure initializers
# define 'individual' to be an individual
# consisting of 10 'attr_item' elements ('genes')
self.toolbox.register("individual", tools.initRepeat,
creator.Individual, self.toolbox.attr_item,
IND_INIT_SIZE)
# define the population to be a list of individuals
self.toolbox.register("population", tools.initRepeat, list,
self.toolbox.individual)
self.toolbox.register("evaluate", self.eval_ind)
self.toolbox.register("mate", self.cx_ind)
self.toolbox.register("mutate", self.mutate_ind)
self.toolbox.register("select", tools.selNSGA2)
def evaluate(self):
logger.info('Begin evaluate of Impulso')
if self.args.experiment_id:
module = importlib.import_module(
f'experiments.{self.args.experiment_id}.src.evaluator')
Evaluator = getattr(module, 'Evaluator')
else:
from src.evaluator import Evaluator
evaluator = Evaluator(self.args.exec_type, self.hparams)
evaluator.load_data()
evaluator.evaluate()
logger.info('End evaluate of Impulso')
def test_random_distribution(self, mocked_randint):
c = Character()
simulation = Simulation(
Evaluator(actors=[c],
rules=[
Rule('rule1', [], [Predicate('p1', 0)], prob=1),
Rule('rule2', [], [Predicate('p2', 0)], prob=4)
]))
mocked_randint.return_value = 1
simulation.step()
self.assertStateContains(PredicateInstance('p1', c),
simulation.evaluator)
self.assertStateDoesNotContain(PredicateInstance('p2', c),
simulation.evaluator)
mocked_randint.return_value = 2
simulation.step()
self.assertStateContains(PredicateInstance('p1', c),
simulation.evaluator)
self.assertStateContains(PredicateInstance('p2', c),
simulation.evaluator)
def main():
energy_list = []
garbage_list = []
f = open(map_path)
z = env(f)
for i in range(3):
zone = copy.deepcopy(z)
zone.maze = clean_maze(zone.maze)
bond = Agent(zone)
current_time = 0
bond.agent_map[bond.posX][bond.posY] = 1
calculus = Evaluator()
while current_time < life_time:
run(zone, bond, calculus, current_time, i)
current_time += 1
print("Cleaned dirty: " + str(calculus.cleaned_dirty))
print("consumed energy: " + str(calculus.consumed_energy))
print("Avg dirty: " + str(calculus.total_dirty / life_time))
energy_list.append(calculus.consumed_energy)
garbage_list.append(calculus.total_dirty / life_time)
labels = ['random', 'reflex', 'model-based']
show(labels, garbage_list, energy_list)
def train(cfg):
# ======= data ===========
loaded_data = load_file(cfg['processed_path'], 'processed_datasets', mode='pickle')
if loaded_data is None and cfg['load_preproc']:
train_data_obj = Dataset(cfg['train_data_path'], 'train', bpe_path=cfg['bpe_data_dir'])
dev_data_obj = Dataset(cfg['dev_data_path'], 'dev', train_data_obj.bpe_enc)
test_data_obj = Dataset(cfg['test_data_path'], 'test', train_data_obj.bpe_enc)
dev_data_obj.sort_example()
test_data_obj.sort_example()
save_file({'train_data_obj': train_data_obj, 'dev_data_obj': dev_data_obj, 'test_data_obj': test_data_obj},
cfg['processed_path'])
else:
train_data_obj = loaded_data['train_data_obj']
dev_data_obj = loaded_data['dev_data_obj']
test_data_obj = loaded_data['test_data_obj']
# 1. delete too long sentence, max len: 50
train_data_obj.filter_example(cfg['max_sent_len'])
# ========= build network ======
if cfg['model_class'] is None:
print('Did not find the model, please check '
'1) module name 2) class name and 3) implementation of get_default_model_paramters(). exit!')
exit()
with tf.variable_scope("model") as scp:
model = cfg['model_class'](
cfg, train_data_obj.bpe_enc.n_vocab, 3,
train_data_obj.bpe_enc.get_idx_from_token(Dataset.CLASSIFY_TOKEN),
scp.name
)
# ======= build session =======
graph_handler = GraphHandler(model, cfg)
evaluator = Evaluator(model, cfg)
performance_record = PerformanceRecoder(cfg['ckpt_dir'], cfg['save_model'], cfg['save_num'])
sess = graph_handler.initialize()
model.load_openai_pretrained_transformer_model(
sess, cfg['pretrained_transformer_dir'])
# ======== begin to train =======
loss_task_ma, loss_lm_ma, accuracy_ma = MovingAverage(), MovingAverage(), MovingAverage()
for example_batch, batch_num, data_round, idx_b in train_data_obj.generate_batch_iter(
cfg['train_batch_size'], cfg['n_steps']):
global_step_val = sess.run(model.global_step) + 1
step_out = model.step(sess, example_batch, cfg['summary_period'])
loss_task_ma(step_out['task_loss']), accuracy_ma(step_out['accuracy']), loss_lm_ma(step_out['lm_loss'])
graph_handler.add_summary(step_out['summary'], global_step_val)
if global_step_val % 100 == 0:
log.info('data round: %d: %d/%d, global step:%d -- loss: %.4f, accu: %.4f' %
(data_round, idx_b, batch_num, global_step_val,
loss_task_ma.value, accuracy_ma.value))
if 'lm_loss' in step_out:
log.info('\tauxiliary language model perplexity: %.4f' % math.exp(loss_lm_ma.value))
# eval
if global_step_val % cfg['eval_period'] == 0:
dev_res = evaluator.get_evaluation(sess, dev_data_obj, global_step_val)
log.info('==> for dev, loss: %.4f, accuracy: %.4f' %
(dev_res['loss'], dev_res['accuracy']))
criterion_metric = dev_res['accuracy']
if not test_data_obj.no_label:
test_res = evaluator.get_evaluation(sess, test_data_obj, global_step_val)
log.info('~~> for test, loss: %.4f, accuracy: %.4f' %
(test_res['loss'], test_res['accuracy']))
is_in_top, deleted_step = performance_record.update_top_list(global_step_val, criterion_metric, sess)
if is_in_top: # get prediction for non-labeled test data
if test_data_obj.no_label and global_step_val > 0.4 * cfg['n_steps']:
evaluator.get_evaluation(sess, test_data_obj, global_step_val, write_predict=True)
# todo: time count
log.info(str(performance_record.top_list))
help="the number of orders to parse", type=int, default=2400)
parser.add_argument("--process_num", dest="NUM_PROCESSES",
help="the number of processes", type=int, default=16)
parser.add_argument("--weight_amount", dest="WEIGHT_AMOUNT",
help="the weight of the amountCost in the objective function", type=float, default=15)
parser.add_argument("--weight_time", dest="WEIGHT_TIME",
help="the weight of the timeCost in the objective function", type=float, default=1)
parser.add_argument("--hub_cost_const", dest="HUB_BUILT_COST_CONST",
help="the constant part of cost of building a hub", type=int, default=3000)
parser.add_argument("--hub_cost_vary", dest="HUB_BUILT_COST_VARY",
help="the variable part cost of building a hub", type=float, default=2.0)
parser.add_argument("--hub_ratio", dest="HUB_UNIT_COST_RATIO",
help="the cutoff of unit price of a hub", type=float, default=0.7)
parser.add_argument("--hub_capacity", dest="HUB_CAPACITY",
help="the capacity of the hub(in prob3)", type=int, default=500)
args = parser.parse_args()
arg_dict = edict(vars(args))
merge_a_into_b(arg_dict, cfg)
print("Using the config:")
print(cfg)
solver = Solver()
evaluator = Evaluator()
solver.solve(problem_id=args.PROBLEM_ID)
# disable the tune mode but plot the distribution, see the outputs for more detail
evaluator.evaluate(prob_id=args.PROBLEM_ID, tune=False, plot=True)
def train():
output_model_params()
loadFile = True
ifLoad, data = False, None
if loadFile:
ifLoad, data = load_file(cfg.processed_path, 'processed data',
'pickle')
if not ifLoad or not loadFile:
train_data_obj = Dataset(cfg.train_data_path, 'train')
dev_data_obj = Dataset(cfg.dev_data_path, 'dev', train_data_obj.dicts)
save_file(
{
'train_data_obj': train_data_obj,
'dev_data_obj': dev_data_obj
}, cfg.processed_path)
else:
train_data_obj = data['train_data_obj']
dev_data_obj = data['dev_data_obj']
emb_mat_token, emb_mat_glove = train_data_obj.emb_mat_token, train_data_obj.emb_mat_glove
output_cls_num = len(
train_data_obj.dicts['sub_cls']) if cfg.fine_grained else len(
train_data_obj.dicts['cls'])
with tf.variable_scope(network_type) as scope:
if network_type in model_set:
model = Model(emb_mat_token,
emb_mat_glove,
len(train_data_obj.dicts['token']),
len(train_data_obj.dicts['char']),
train_data_obj.max_lens['token'],
output_cls_num,
scope=scope.name)
graphHandler = GraphHandler(model)
evaluator = Evaluator(model)
performRecoder = PerformRecoder(3)
if cfg.gpu_mem is None:
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=cfg.gpu_mem, allow_growth=True)
graph_config = tf.ConfigProto(gpu_options=gpu_options,
allow_soft_placement=True)
else:
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=cfg.gpu_mem)
graph_config = tf.ConfigProto(gpu_options=gpu_options)
# graph_config.gpu_options.allow_growth = True
sess = tf.Session(config=graph_config)
graphHandler.initialize(sess)
# begin training
steps_per_epoch = int(
math.ceil(1.0 * train_data_obj.sample_num / cfg.train_batch_size))
num_steps = cfg.num_steps or steps_per_epoch * cfg.max_epoch
global_step = 0
for sample_batch, batch_num, data_round, idx_b in train_data_obj.generate_batch_sample_iter(
num_steps):
global_step = sess.run(model.global_step) + 1
if_get_summary = global_step % (cfg.log_period or steps_per_epoch) == 0
loss, summary, train_op = model.step(sess,
sample_batch,
get_summary=if_get_summary)
if global_step % 100 == 0:
_logger.add('data round: %d: %d/%d, global step:%d -- loss: %.4f' %
(data_round, idx_b, batch_num, global_step, loss))
if if_get_summary:
graphHandler.add_summary(summary, global_step)
# Occasional evaluation
if global_step % (cfg.eval_period or steps_per_epoch) == 0:
# ---- dev ----
dev_loss, dev_accu = evaluator.get_evaluation(
sess, dev_data_obj, global_step)
_logger.add('==> for dev, loss: %.4f, accuracy: %.4f' %
(dev_loss, dev_accu))
is_in_top, deleted_step = performRecoder.update_top_list(
global_step, dev_accu, sess)
this_epoch_time, mean_epoch_time = cfg.time_counter.update_data_round(
data_round)
# if this_epoch_time is not None and mean_epoch_time is not None:
# _logger.add('##> this epoch time: %f, mean epoch time: %f' % (this_epoch_time, mean_epoch_time))
do_analyse_qc(_logger.path)
class SupervisedTrainer(object):
""" The SupervisedTrainer class helps in setting up a training framework in a
supervised setting.
Args:
expt_dir (optional, str): experiment Directory to store details of the experiment,
by default it makes a folder in the current directory to store the details (default: `experiment`).
"""
def __init__(self, expt_dir='experiment'):
self._trainer = "Simple Trainer"
if not os.path.isabs(expt_dir):
expt_dir = os.path.join(os.getcwd(), expt_dir)
self.expt_dir = expt_dir
if not os.path.exists(self.expt_dir):
os.makedirs(self.expt_dir)
def set_local_parameters(self, random_seed, losses, metrics,
loss_weights, checkpoint_every, print_every):
self.random_seed = random_seed
if random_seed is not None:
random.seed(random_seed)
torch.manual_seed(random_seed)
self.losses = losses
self.metrics = metrics
self.loss_weights = loss_weights or len(losses)*[1.]
self.evaluator = Evaluator(loss=self.losses, metrics=self.metrics)
self.optimizer = None
self.checkpoint_every = checkpoint_every
self.print_every = print_every
self.logger = logging.getLogger(__name__)
self._stop_training = False
def _train_batch(self, ins_variable, src_variable, ins_len_variable, ins_position_variables, position_variables,
tgt_variable):
# Forward propagation
predicted_softmax, attn = self.model(ins_variable, src_variable, ins_len_variable,
ins_position_variables, position_variables)
losses = self.evaluator.compute_batch_loss(predicted_softmax, tgt_variable)
# Backward propagation
for i, loss in enumerate(losses, 0):
loss.scale_loss(self.loss_weights[i])
loss.backward(retain_graph=True)
self.optimizer.step()
self.model.zero_grad()
return losses
def _train_epoches(self, data, n_epochs,
start_epoch, start_step,
callbacks,
dev_data, monitor_data=[]):
steps_per_epoch = len(data)
total_steps = steps_per_epoch * n_epochs
callbacks.set_info(start_step, start_epoch,
steps_per_epoch,
total_steps)
# set data as attribute to trainer
self.train_data = data
self.val_data = dev_data
self.monitor_data = monitor_data
# ########################################
# This is used to resume training from same place in dataset
# after loading from checkpoint
s = start_step
if start_epoch > 1:
s -= (start_epoch - 1) * steps_per_epoch
########################################
callbacks.on_train_begin()
for epoch in range(start_epoch, n_epochs + 1):
callbacks.on_epoch_begin(epoch)
self.model.train()
for batch in data:
# Skip over the batches that are below start step
if epoch == start_epoch and s > 0:
s -= 1
continue
callbacks.on_batch_begin(batch)
ins_variables, src_variables, tgt_variable = self.get_batch_data(batch)
ins_position_variables = self.get_position_ids(ins_variables[1])
position_variables = self.get_position_ids(src_variables[1])
self.batch_losses = self._train_batch(ins_variables[0], src_variables[0], ins_variables[1],
ins_position_variables, position_variables, tgt_variable)
callbacks.on_batch_end(batch)
callbacks.on_epoch_end(epoch)
# Stop training early if flag _stop_training is True
if self._stop_training:
break
logs = callbacks.on_train_end()
return logs
def train(self, model, data,
dev_data,
num_epochs=5,
resume_training=False,
monitor_data={},
optimizer=None,
custom_callbacks=[],
learning_rate=0.001,
checkpoint_path=None,
top_k=5,
losses=[NLLLoss()],
loss_weights=None,
metrics=[],
random_seed=None,
checkpoint_every=100,
print_every=100):
""" Run training for a given model.
Args:
model (machine.models): model to run training on, if `resume=True`, it would be
overwritten by the model loaded from the latest checkpoint.
data (torchtext.data.Iterator: torchtext iterator object to train on
num_epochs (int, optional): number of epochs to run (default 5)
resume_training(bool, optional): resume training with the latest checkpoint up until the number of epochs (default False)
dev_data (torchtext.data.Iterator): dev/validation set iterator
Note: must not pass in the train iterator here as this gets evaluated during training (in between batches)
If you want to evaluate on the full train during training then make two iterators and pass the second one here
monitor_data (list of torchtext.data.Iterator, optional): list of iterators to test on (default None)
Note: must not pass in the train iterator here as this gets evaluated during training (in between batches)
If you want to evaluate on the full train during training then make two iterators and pass the second one here
optimizer (machine.optim.Optimizer, optional): optimizer for training
(default: Optimizer(pytorch.optim.Adam, max_grad_norm=5))
custom_callbacks (list, optional): list of custom call backs (see utils.callbacks.callback for base class)
learing_rate (float, optional): learning rate used by the optimizer (default 0.001)
checkpoint_path (str, optional): path to load checkpoint from in case training should be resumed
top_k (int): how many models should be stored during training
loss (list, optional): list of machine.loss.Loss objects for training (default: [machine.loss.NLLLoss])
metrics (list, optional): list of machine.metric.metric objects to be computed during evaluation
checkpoint_every (int, optional): number of epochs to checkpoint after, (default: 100)
print_every (int, optional): number of iterations to print after, (default: 100)
Returns:
model (machine.models): trained model.
"""
self.set_local_parameters(random_seed, losses, metrics,
loss_weights, checkpoint_every, print_every)
# If training is set to resume
if resume_training:
resume_checkpoint = Checkpoint.load(checkpoint_path)
model = resume_checkpoint.model
self.model = model
self.optimizer = resume_checkpoint.optimizer
# A walk around to set optimizing parameters properly
resume_optim = self.optimizer.optimizer
defaults = resume_optim.param_groups[0]
defaults.pop('params', None)
defaults.pop('initial_lr', None)
self.optimizer.optimizer = resume_optim.__class__(
self.model.parameters(), **defaults)
start_epoch = resume_checkpoint.epoch
step = resume_checkpoint.step
else:
start_epoch = 1
step = 0
self.model = model
def get_optim(optim_name):
optims = {'adam': optim.Adam, 'adagrad': optim.Adagrad,
'adadelta': optim.Adadelta, 'adamax': optim.Adamax,
'rmsprop': optim.RMSprop, 'sgd': optim.SGD,
None: optim.Adam}
return optims[optim_name]
self.optimizer = Optimizer(get_optim(optimizer)(self.model.parameters(),
lr=learning_rate),
max_grad_norm=5)
self.logger.info("Optimizer: %s, Scheduler: %s" %
(self.optimizer.optimizer, self.optimizer.scheduler))
callbacks = CallbackContainer(self,
[Logger(),
ModelCheckpoint(top_k=top_k),
History()] + custom_callbacks)
logs = self._train_epoches(data, num_epochs, start_epoch, step, dev_data=dev_data,
monitor_data=monitor_data,
callbacks=callbacks)
return self.model, logs
@staticmethod
def get_batch_data(batch):
ins_variables = getattr(batch, 'ins')
src_variables = getattr(batch, 'src')
tgt_variable = getattr(batch, 'tgt')[0]
return ins_variables, src_variables, tgt_variable
@staticmethod
def get_position_ids(length_list):
batch_size = len(length_list)
length = max(length_list).item()
pos_tensor = torch.zeros(batch_size, length).long()
for i in range(batch_size):
pos_tensor[i] = torch.LongTensor(range(0, length))
return Variable(pos_tensor)
def main(params):
# initialize the multi-GPU / multi-node training
init_distributed_mode(params)
# initialize the experiment / load data
logger = initialize_exp(params)
# Seed
torch.manual_seed(params.seed)
torch.cuda.manual_seed_all(params.seed)
# initialize SLURM signal handler for time limit / pre-emption
if params.is_slurm_job:
init_signal_handler()
# data loaders / samplers
populate_dataset(params)
train_data_loader, train_sampler, _ = get_data_loader(
img_size=params.img_size,
crop_size=params.crop_size,
shuffle=True,
batch_size=params.batch_size,
num_classes=params.num_classes,
nb_workers=params.nb_workers,
distributed_sampler=params.multi_gpu,
dataset=params.dataset,
data_path=params.train_path,
transform=params.train_transform,
split='valid' if params.debug_train else 'train',
seed=params.seed)
valid_data_loader, _, _ = get_data_loader(img_size=params.img_size,
crop_size=params.crop_size,
shuffle=False,
batch_size=params.batch_size,
num_classes=params.num_classes,
nb_workers=params.nb_workers,
distributed_sampler=False,
dataset=params.dataset,
transform='center',
split='valid',
seed=params.seed)
# build model / cuda
logger.info("Building %s model ..." % params.architecture)
ftmodel = build_model(params)
ftmodel.fc = nn.Sequential()
ftmodel.eval().cuda()
linearmodel = nn.Linear(EMBEDDING_SIZE[params.architecture],
params.num_classes).cuda()
if params.from_ckpt != "":
ckpt = torch.load(params.from_ckpt)
state_dict = {
k.replace("module.", ""): v
for k, v in ckpt['model'].items()
}
del state_dict["fc.weight"]
if "fc.bias" in state_dict:
del state_dict["fc.bias"]
missing_keys, unexcepted_keys = ftmodel.load_state_dict(state_dict,
strict=False)
print("Missing keys: ", missing_keys)
print("Unexcepted keys: ", unexcepted_keys)
# distributed # TODO: check this https://github.com/NVIDIA/apex/blob/master/examples/imagenet/main.py#L142
if params.multi_gpu:
logger.info("Using nn.parallel.DistributedDataParallel ...")
linearmodel = nn.parallel.DistributedDataParallel(
linearmodel,
device_ids=[params.local_rank],
output_device=params.local_rank,
broadcast_buffers=True)
# build trainer / reload potential checkpoints / build evaluator
trainer = Trainer(model=linearmodel, params=params, ftmodel=ftmodel)
trainer.reload_checkpoint()
evaluator = Evaluator(trainer, params)
# evaluation
if params.eval_only:
scores = evaluator.run_all_evals(trainer,
evals=['classif'],
data_loader=valid_data_loader)
for k, v in scores.items():
logger.info('%s -> %.6f' % (k, v))
logger.info("__log__:%s" % json.dumps(scores))
exit()
# training
for epoch in range(trainer.epoch, params.epochs):
# update epoch / sampler / learning rate
trainer.epoch = epoch
logger.info("============ Starting epoch %i ... ============" %
trainer.epoch)
if params.multi_gpu:
train_sampler.set_epoch(epoch)
# update learning rate
trainer.update_learning_rate()
# train
for i, (images, targets) in enumerate(train_data_loader):
trainer.classif_step(images, targets)
trainer.iter()
logger.info("============ End of epoch %i ============" %
trainer.epoch)
# evaluate classification accuracy
scores = evaluator.run_all_evals(trainer,
evals=['classif'],
data_loader=valid_data_loader)
for name, val in trainer.get_scores().items():
scores[name] = val
# print / JSON log
for k, v in scores.items():
logger.info('%s -> %.6f' % (k, v))
if params.is_master:
logger.info("__log__:%s" % json.dumps(scores))
# end of epoch
trainer.save_best_model(scores)
trainer.save_periodic()
trainer.end_epoch(scores)
def train():
output_model_params()
loadFile = True
ifLoad, data = False, None
if loadFile:
ifLoad, data = load_file(cfg.processed_path, 'processed data',
'pickle')
if not ifLoad or not loadFile:
train_data_obj = Dataset(cfg.train_data_path, 'train')
dev_data_obj = Dataset(cfg.dev_data_path,
'dev',
dicts=train_data_obj.dicts)
test_data_obj = Dataset(cfg.test_data_path,
'test',
dicts=train_data_obj.dicts)
save_file(
{
'train_data_obj': train_data_obj,
'dev_data_obj': dev_data_obj,
'test_data_obj': test_data_obj
}, cfg.processed_path)
train_data_obj.save_dict(cfg.dict_path)
else:
train_data_obj = data['train_data_obj']
dev_data_obj = data['dev_data_obj']
test_data_obj = data['test_data_obj']
train_data_obj.filter_data()
dev_data_obj.filter_data()
test_data_obj.filter_data()
emb_mat_token, emb_mat_glove = train_data_obj.emb_mat_token, train_data_obj.emb_mat_glove
with tf.variable_scope(cfg.base_name) as scope:
model = Model(emb_mat_token, emb_mat_glove,
len(train_data_obj.dicts['token']),
len(train_data_obj.dicts['char']),
train_data_obj.max_lens['token'], scope.name)
graphHandler = GraphHandler(model)
evaluator = Evaluator(model)
performRecoder = PerformRecoder(cfg.save_num)
if cfg.gpu_mem is None:
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=cfg.gpu_mem, allow_growth=True)
graph_config = tf.ConfigProto(gpu_options=gpu_options,
allow_soft_placement=True)
elif cfg.gpu_mem < 1.:
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=cfg.gpu_mem)
graph_config = tf.ConfigProto(gpu_options=gpu_options)
else:
gpu_options = tf.GPUOptions()
graph_config = tf.ConfigProto(gpu_options=gpu_options)
sess = tf.Session(config=graph_config)
graphHandler.initialize(sess)
# begin training
steps_per_epoch = int(
math.ceil(1.0 * train_data_obj.sample_num / cfg.train_batch_size))
num_steps = cfg.num_steps or steps_per_epoch * cfg.max_epoch
global_step = 0
for sample_batch, batch_num, data_round, idx_b in train_data_obj.generate_batch_sample_iter(
num_steps):
global_step = sess.run(model.global_step) + 1
if_get_summary = global_step % (cfg.log_period or steps_per_epoch) == 0
loss, summary = model.step(sess,
sample_batch,
get_summary=if_get_summary,
global_step_value=global_step)
if global_step % 100 == 0:
_logger.add(
'data round: %d: %d/%d, global step:%d -- loss_sl: %.4f, loss_rl: %.4f'
%
(data_round, idx_b, batch_num, global_step, loss[0], loss[1]))
if if_get_summary:
graphHandler.add_summary(summary, global_step)
# Occasional evaluation
evaluation = False
if cfg.model_dir_suffix == 'test':
if global_step % (cfg.eval_period or steps_per_epoch) == 0:
evaluation = True
elif is_base_training:
if global_step > cfg.num_steps - 350000 and (
global_step % (cfg.eval_period or steps_per_epoch) == 0):
evaluation = True
else:
if global_step % (cfg.eval_period or steps_per_epoch) == 0:
if cfg.load_model:
evaluation = True
else:
if global_step > 250000:
evaluation = True
if evaluation:
# ---- dev ----
dev_loss, dev_accu, dev_perc = evaluator.get_evaluation(
sess, dev_data_obj, global_step)
_logger.add(
'==> for dev, loss: %.4f %.4f, perc: %.4f, accuracy: %.4f' %
(dev_loss[0], dev_loss[1], dev_perc, dev_accu))
# ---- test ----
test_loss, test_accu, test_perc = evaluator.get_evaluation(
sess, test_data_obj, global_step)
_logger.add(
'~~> for test, loss: %.4f %.4f, perc: %.4f, accuracy: %.4f' %
(test_loss[0], test_loss[1], test_perc, test_accu))
is_in_top, deleted_step = performRecoder.update_top_list(
global_step, dev_accu, sess)
this_epoch_time, mean_epoch_time = cfg.time_counter.update_data_round(
data_round)
if this_epoch_time is not None and mean_epoch_time is not None:
_logger.add('##> this epoch time: %f, mean epoch time: %f' %
(this_epoch_time, mean_epoch_time))
if is_base_training and global_step >= 200000 and global_step % 50000 == 0 and cfg.save_model:
graphHandler.save(sess, global_step)
_logger.writeToFile()
do_analyse_snli_rl(_logger.path)
def multi_test():
assert cfg.load_path is not None
output_model_params()
loadFile = True
ifLoad, data = False, None
if loadFile:
ifLoad, data = load_file(cfg.processed_path, 'processed data',
'pickle')
if not ifLoad or not loadFile:
train_data_obj = Dataset(cfg.train_data_path, 'train')
dev_data_obj = Dataset(cfg.dev_data_path,
'dev',
dicts=train_data_obj.dicts)
test_data_obj = Dataset(cfg.test_data_path,
'test',
dicts=train_data_obj.dicts)
save_file(
{
'train_data_obj': train_data_obj,
'dev_data_obj': dev_data_obj,
'test_data_obj': test_data_obj
}, cfg.processed_path)
train_data_obj.save_dict(cfg.dict_path)
else:
train_data_obj = data['train_data_obj']
dev_data_obj = data['dev_data_obj']
test_data_obj = data['test_data_obj']
train_data_obj.filter_data()
dev_data_obj.filter_data()
test_data_obj.filter_data()
emb_mat_token, emb_mat_glove = train_data_obj.emb_mat_token, train_data_obj.emb_mat_glove
with tf.variable_scope(cfg.base_name) as scope:
model = Model(emb_mat_token, emb_mat_glove,
len(train_data_obj.dicts['token']),
len(train_data_obj.dicts['char']),
train_data_obj.max_lens['token'], scope.name)
graphHandler = GraphHandler(model)
evaluator = Evaluator(model)
if cfg.gpu_mem is None:
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=cfg.gpu_mem, allow_growth=True)
graph_config = tf.ConfigProto(gpu_options=gpu_options,
allow_soft_placement=True)
elif cfg.gpu_mem < 1.:
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=cfg.gpu_mem)
graph_config = tf.ConfigProto(gpu_options=gpu_options)
else:
gpu_options = tf.GPUOptions()
graph_config = tf.ConfigProto(gpu_options=gpu_options)
sess = tf.Session(config=graph_config)
graphHandler.initialize(sess)
repeat_num = 10
time_counter = TimeCounter()
for t in range(repeat_num):
# ---- dev ----
test_loss, test_accu, test_perc = evaluator.get_evaluation(
sess, test_data_obj, None, time_counter=time_counter)
_logger.add(
'==> for test, loss: %.4f %.4f, perc: %.4f, accuracy: %.4f' %
(test_loss[0], test_loss[1], test_perc, test_accu))
print(time_counter.update_data_round(t + 1))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("model_dir",
help="Directory containing the trained model")
parser.add_argument("test_dir", help="Directory containing test files")
parser.add_argument("result_dir", help="Directory to write result files")
parser.add_argument("--max_distance",
default=15,
type=int,
help="Set the max entity distance to build data")
parser.add_argument("--triad",
action='store_true',
default=False,
help="use triads")
parser.add_argument("--keras",
action='store_true',
default=False,
help="Use keras model")
parser.add_argument("--clustering_only",
action='store_true',
default=False,
help="Use saved linkage files to perform clustering")
parser.add_argument("--compute_linkage",
action='store_true',
default=False,
help="compute linkage, for clustering_only option")
args = parser.parse_args()
with open(os.path.join(args.model_dir, 'word_indexes.pkl'), 'rb') as f:
word_indexes = pickle.load(f)
with open(os.path.join(args.model_dir, 'pos_tags.pkl'), 'rb') as f:
pos_tags = pickle.load(f)
df = build_dataFrame(args.test_dir, threads=1, suffix='auto_conll')
if args.clustering_only:
model = None
elif args.keras:
from keras.models import load_model
model = load_model(os.path.join(args.model_dir, 'model.h5'))
else:
import torch
model = torch.load(os.path.join(args.model_dir, 'model.pt'))
model.eval()
print("Loaded model")
if args.triad:
n_files = len(df.doc_id.unique())
test_gen = DataGen(df, word_indexes, pos_tags)
test_input_gen = test_gen.generate_triad_input(
looping=True,
test_data=True,
threads=4,
max_distance=args.max_distance)
evaluator = TriadEvaluator(model, test_input_gen)
evaluator.write_results(df,
args.result_dir,
n_iterations=n_files,
clustering_only=args.clustering_only,
compute_linkage=args.compute_linkage)
else:
test_gen = DataGen(df, word_indexes, pos_tags)
test_data_q = next(
test_gen.generate_input(looping=False, test_data=True))
print("Loaded test data")
evaluator = Evaluator(model, test_data_q)
print("Performing fast evaluation...")
print(evaluator.fast_eval())
print("Saving result files...")
evaluator.write_results(df, args.result_dir)
scorer(args.result_dir)
def train():
# output_model_params()
# need to fixed, reusability of data
loadFile = True
ifLoad, data = False, None
loaddict, dicts = load_file(cfg.dict_path, 'dict', 'pickle')
if not loaddict:
raise (ValueError, 'dict load failed')
if loadFile:
ifLoad, data = load_file(cfg.processed_path, 'processed data',
'pickle')
if not ifLoad or not loadFile:
train_data_obj = Dataset(cfg.train_data_path,
'train',
dicts=dicts,
language_type='es',
unlabeled_file_path=cfg.unlabeled_data_path,
emb_file_path=cfg.emb_es_path)
dev_data_obj = Dataset(cfg.dev_data_path, 'dev', dicts=dicts)
test_data_obj = Dataset(cfg.test_data_path, 'test', dicts=dicts)
save_file(
{
'train_data_obj': train_data_obj,
'dev_data_obj': dev_data_obj,
'test_data_obj': test_data_obj
}, cfg.processed_path)
# train_data_obj.save_dict(cfg.dict_path)
else:
train_data_obj = data['train_data_obj']
dev_data_obj = data['dev_data_obj']
test_data_obj = data['test_data_obj']
emb_mat_token = train_data_obj.emb_mat_token
with tf.variable_scope(network_type) as scope:
if network_type in model_type_set:
model = Model(emb_mat_token, len(train_data_obj.dicts['es']), 100,
scope.name)
# TODO
graphHandler = GraphHandler(model)
evaluator = Evaluator(model)
performRecorder = PerformRecorder(5)
if cfg.gpu_mem < 1:
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=cfg.gpu_mem, allow_growth=True)
else:
gpu_options = tf.GPUOptions()
graph_config = tf.ConfigProto(gpu_options=gpu_options,
allow_soft_placement=True)
sess = tf.Session(config=graph_config)
graphHandler.initialize(sess)
# load model
if cfg.load_model and cfg.load_step:
saver = tf.train.Saver()
step = cfg.load_step
model_path = os.path.join(cfg.ckpt_dir,
'top_result_saver_step_%d.ckpt' % step)
saver.restore(sess, model_path)
# begin training
steps_per_epoch = int(
math.ceil(1.0 * train_data_obj.sample_num / cfg.train_batch_size))
num_steps = cfg.num_steps or steps_per_epoch * cfg.max_epoch
global_step = 0
for sample_batch, batch_num, data_round, idx_b in train_data_obj.generate_batch_sample_iter(
num_steps):
global_step = sess.run(model.global_step) + 1
if_get_summary = global_step % (cfg.log_period or steps_per_epoch) == 0
loss, summary, train_op = model.step(sess,
sample_batch,
get_summary=if_get_summary)
if global_step % 100 == 0 or global_step == 1:
_logger.add('data round: %d: %d/%d, global_step:%d -- loss:%.4f' %
(data_round, idx_b, batch_num, global_step, loss))
if if_get_summary:
graphHandler.add_summary(summary, global_step)
# Occasional evaluation
#if global_step > int(cfg.num_steps - 100000) and (global_step % (cfg.eval_period or steps_per_epoch) == 0):
if global_step % (cfg.eval_period or steps_per_epoch) == 0: # debug
# ---- dev ----
dev_loss, dev_accu, dev_accu_0, dev_accu_1 = evaluator.get_evaluation(
sess, dev_data_obj, global_step)
_logger.add(
'==> for dev, loss: %.4f, accuracy: %.4f, accuracy_0: %.4f, accuracy_1: %.4f'
% (dev_loss, dev_accu, dev_accu_0, dev_accu_1))
# ---- test ----
test_loss, test_accu, test_accu_0, test_accu_1 = evaluator.get_evaluation(
sess, test_data_obj, global_step)
_logger.add(
'~~> for test, loss: %.4f, accuracy: %.4f, accuracy_0: %.4f, accuracy_1: %.4f'
% (test_loss, test_accu, test_accu_0, test_accu_1))
if global_step > cfg.update_lr_step:
model.update_learning_rate(dev_loss, cfg.lr_decay)
if global_step > cfg.record_model_step:
is_in_top, deleted_step = performRecorder.update_top_list(
global_step, test_accu, sess)
this_epoch_time, mean_epoch_time = cfg.time_counter.update_data_round(
data_round)
if this_epoch_time is not None and mean_epoch_time is not None:
_logger.add('##> this epoch time: %f, mean epoch time: %f' %
(this_epoch_time, mean_epoch_time))
def test():
assert cfg.load_path is not None
#TODO
loadFile = False
ifLoad, data = False, None
if loadFile:
ifLoad, data = load_file(cfg.processed_path, 'processed data',
'pickle')
if not ifLoad or not loadFile:
train_data_obj = Dataset(cfg.train_data_path,
'train',
language_type='es',
unlabeled_file_path=cfg.unlabeled_data_path,
emb_file_path=cfg.emb_es_path)
dev_data_obj = Dataset(cfg.dev_data_path,
'dev',
dicts=train_data_obj.dicts)
test_data_obj = Dataset(cfg.test_data_path,
'test',
dicts=train_data_obj.dicts)
save_file(
{
'train_data_obj': train_data_obj,
'dev_data_obj': dev_data_obj,
'test_data_obj': test_data_obj
}, cfg.processed_path)
train_data_obj.save_dict(cfg.dict_path)
else:
train_data_obj = data['train_data_obj']
dev_data_obj = data['dev_data_obj']
test_data_obj = data['test_data_obj']
emb_mat_token = train_data_obj.emb_mat_token
with tf.variable_scope(network_type) as scope:
if network_type in model_type_set:
model = Model(emb_mat_token, len(train_data_obj.dicts['es']), 100,
scope.name)
# TODO
graphHandler = GraphHandler(model)
evaluator = Evaluator(model)
graph_config = tf.ConfigProto()
sess = tf.Session(config=graph_config)
graphHandler.initialize(sess)
# ---- dev ----
dev_loss, dev_accu = evaluator.get_evaluation(sess, dev_data_obj, None)
_logger.add('==> for dev, loss: %.4f, accuracy: %.4f' %
(dev_loss, dev_accu))
# ---- test ----
test_loss, test_accu = evaluator.get_evaluation(sess, test_data_obj, None)
_logger.add('~~> for test, loss: %.4f, accuracy: %.4f' %
(test_loss, test_accu))
# ---- train ----
train_loss, train_accu = evaluator.get_evaluation(sess, train_data_obj,
None)
_logger.add('--> for train, loss: %.4f, accuracy: %.4f' %
(train_loss, train_accu))
def main(params):
# initialize the multi-GPU / multi-node training
# initialize experiment / SLURM signal handler for time limit / pre-emption
init_distributed_mode(params)
logger = initialize_exp(params)
init_signal_handler()
# CPU / CUDA
if params.cpu:
assert not params.multi_gpu
else:
assert torch.cuda.is_available()
src.utils.CUDA = not params.cpu
# build environment / modules / trainer / evaluator
env = build_env(params)
modules = build_modules(env, params)
trainer = Trainer(modules, env, params)
evaluator = Evaluator(trainer)
# evaluation
if params.eval_only:
scores = evaluator.run_all_evals()
for k, v in scores.items():
logger.info("%s -> %.6f" % (k, v))
logger.info("__log__:%s" % json.dumps(scores))
exit()
# training
for _ in range(params.max_epoch):
logger.info("============ Starting epoch %i ... ============" % trainer.epoch)
trainer.n_equations = 0
while trainer.n_equations < trainer.epoch_size:
# training steps
for task_id in np.random.permutation(len(params.tasks)):
task = params.tasks[task_id]
if params.export_data:
trainer.export_data(task)
else:
trainer.enc_dec_step(task)
trainer.iter()
logger.info("============ End of epoch %i ============" % trainer.epoch)
# evaluate perplexity
scores = evaluator.run_all_evals()
# print / JSON log
for k, v in scores.items():
logger.info("%s -> %.6f" % (k, v))
if params.is_master:
logger.info("__log__:%s" % json.dumps(scores))
# end of epoch
trainer.save_best_model(scores)
trainer.save_periodic()
trainer.end_epoch(scores)
def test_eval(self):
evaluator = Evaluator({})
env = Environment()
self.exp1.eval(evaluator, env)
self.assertEqual(env.get_variable("x"), 65)