def _copy_checkpoint(from_model, to_path, step):
""" Copy checkpoint files of a specific training step. """
from_path = build.get_model_path(from_model)
log.info('Copy step %d checkpoint files from: %s' % (step, from_path))
files = glob.glob('{}/model-{}.*'.format(from_path, step))
for f in files:
shutil.copy(f, to_path)
def _make_optim_manager(optim_manager_type, loss_op, clip_norm,
train_regex_list, kwargs):
om = getattr(optim, optim_manager_type)(
loss_op,
clip_norm,
var_list=_search_var_list(train_regex_list), # None for all vars.
**_select_kwargs_regex(kwargs,
regex=r'^optim_manager(?!_type)',
start=14))
log.info(str(om))
idx = 1
if _select_kwargs_regex(kwargs, regex=r'^optim_(?!manager)'):
# Construct optimizer with 'optim_*' arguments
optim_kwargs = _select_kwargs_regex(kwargs,
regex=r'^optim_(?!manager)',
start=6)
om.add_optimizer(**optim_kwargs)
else:
while True:
# Construct optimizer with 'optim{idx}_*' arguments
optim_kwargs = _select_kwargs_regex(kwargs,
regex=r'^optim%d_' % idx,
start=6 + len(str(idx)))
if not optim_kwargs:
break
om.add_optimizer(**optim_kwargs)
idx += 1
return om
def __init__(self, path, dim, binary):
super(PretrainedEmbedding, self).__init__(dim)
self.path = path
self.__binary = binary
self.__model = None
log.info('Read pretrained %s embedding from file: %s' %
(self.__class__.__name__, self.path))
self.__model = gensim.models.KeyedVectors.load_word2vec_format(
self.path, binary=self.__binary)
def _search_var_list(var_regex_list: t.Union[t.List[str], str]):
if isinstance(var_regex_list, str):
var_regex_list = [var_regex_list]
if var_regex_list:
pattern_list = [re.compile(regex) for regex in var_regex_list]
var_list = [
var for var in tf.trainable_variables()
if any(map(lambda p: p.match(var.name), pattern_list))
]
log.info('Partical updation on parameters: \n\n\t%s\n' %
'\n\t'.join(map(lambda v: v.name, var_list)))
return var_list
def update(self, step_acc, global_step) -> None:
if step_acc < self._max_acc + self.min_delta:
self._no_update_cnt += 1
log.debug('No update for %d consequtive times.' %
self._no_update_cnt)
if self._no_update_cnt >= self.patience:
self._lr_val[self._idx] /= 2
log.info('Half the current learning rate to : %f' %
self._lr_val[self._idx])
self._no_update_cnt = 0
else:
self._no_update_cnt = 0
self._max_acc = max(self._max_acc, step_acc)
def next(self):
""" Set the next available optimizer for training. """
if self.has_next():
self._idx += 1
log.info('Train model using %s Optimizer' % self.optim.get_name())
grads_tvars = self.optim.compute_gradients(**self._compute_kwargs)
if self.clip_norm:
log.info('Apply global gradient clipping with norm %f' %
self.clip_norm)
grads = [grad for grad, tvar in grads_tvars]
tvars = [tvar for grad, tvar in grads_tvars]
grads, _ = tf.clip_by_global_norm(grads, self.clip_norm)
grads_tvars = zip(grads, tvars)
self.optim_op = self.optim.apply_gradients(grads_tvars)
def __init__(self,
glove_path='/home/jmei/data/glove.840B.300d.txt',
path='/home/jmei/data/glove.840B.300d.norm.txt',
dim=300,
binary=False,
**kwargs):
if not os.path.exists(path):
# Preprocess the original GloVe data to allow using the gensim API.
log.info('Generate normalized GloVe embeddings to file: %s' % path)
with open(glove_path, 'r') as in_file:
with open(path, 'w') as out_file:
for l in in_file:
fields = l.split(' ')
name = fields[:-300]
embed = list(map(float, fields[-300:]))
embed_norm = embed / np.linalg.norm(embed)
out_file.write(' '.join(name) + ' ' +
' '.join(map(str, embed_norm)) + '\n')
super(GloVeNorm, self).__init__(path, dim, binary, **kwargs)
def _profile_and_exit(session, model, optimizer, handle):
""" Profile the run metadata at the first iteration and exit the program.
"""
from tensorflow.python.client import timeline
run_metadata = tf.RunMetadata()
for i in range(5):
session.run(
[optimizer],
feed_dict={
model.handle: handle,
model.keep_prob: 1.0,
model.is_training: True
},
options=tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE),
run_metadata=run_metadata)
tl = timeline.Timeline(run_metadata.step_stats)
ctf = tl.generate_chrome_trace_format()
with open('timeline_%d.json' % i, 'w') as f:
f.write(ctf)
log.info('Profiles are created! Now exit.')
exit()
def load_embeddings(data_name: str, embedding_name: str, seed: int = None):
""" Load a indexed word embedding object.
This method will first check if the given setup have previously been
serialized, restore the object from file. Otherwise, construct a new object.
"""
# The naming convention of the serialization path.
if seed:
pkl_path = os.path.join(
build.BUILD_DIR, 'data',
'{}.{}.seed{}.pkl'.format(data_name, embedding_name, seed))
else:
pkl_path = os.path.join(build.BUILD_DIR, 'data',
'{}.{}.pkl'.format(data_name, embedding_name))
if os.path.exists(pkl_path):
log.info(
'Restore corpus-specific indexed word embedding from file: %s' %
pkl_path)
with open(pkl_path, 'rb') as pkl_file:
embeds = pickle.load(pkl_file)
else:
if seed:
log.debug('Set numpy random seed to %d' % seed)
np.random.seed(seed)
log.info('Build corpus-specific indexed word embedding.')
embeds = globals()[data_name].init_indexed_word_embedding(
embed.init(embedding_name))
# Serialize for reusing
log.info('Save corpus-specific indexed word embedding to file: %s.' %
pkl_path)
os.makedirs(os.path.normpath(os.path.join(pkl_path, os.pardir)),
exist_ok=True)
with open(pkl_path, 'wb') as pkl_file:
pickle.dump(embeds, pkl_file, 4)
return embeds
def load_dataset(data_name: str,
data_mode: str,
embedding_name: str,
seed: int = None) -> Dataset:
if seed:
pkl_path = os.path.join(
build.BUILD_DIR, 'data',
'{}-{}.{}.seed{}.pkl'.format(data_name, data_mode, embedding_name,
seed))
else:
pkl_path = os.path.join(
build.BUILD_DIR, 'data',
'{}-{}.{}.pkl'.format(data_name, data_mode, embedding_name))
# Load preprocessed data object from pkl if applicable.
if os.path.exists(pkl_path):
log.info('Restore %s %s dataset from file: %s' %
(data_name, data_mode, pkl_path))
with open(pkl_path, 'rb') as pkl_file:
dataset = pickle.load(pkl_file)
else:
log.info('Build %s %s dataset' % (data_name, data_mode))
embedding = load_embeddings(data_name, embedding_name, seed)
if seed:
log.debug('Set numpy random seed to %d' % seed)
np.random.seed(seed)
dataset = globals()[data_name](data_mode,
indexed_word_embedding=embedding)
os.makedirs(os.path.normpath(os.path.join(pkl_path, os.pardir)),
exist_ok=True)
log.info('Serialize %s %s dataset to file %s.' %
(data_mode, data_name, pkl_path))
with open(pkl_path, 'wb') as pkl_file:
pickle.dump(dataset, pkl_file, 4)
return dataset
def test(
name: str,
model_type: str,
step: int = None,
mode: str = 'test',
data_seed: int = None,
data_name: str = 'SNLI',
data_embedding: str = 'GloVe',
data_pad: bool = True,
batch_size: int = 10,
print_errors: bool = False,
print_errors_limit: int = 10,
**kwargs,
) -> None:
model_path = build.get_model_path(name)
model = getattr(nn, model_type)(embeddings=data.load_embeddings(
data_name, data_embedding, data_seed),
**kwargs)
log.info(str(model))
log.debug('Model parameters:\n\n\t' +
'\n\t'.join(graph.print_trainable_variables().split('\n')))
with tf.Session(config=_make_config()) as sess:
dataset = data.load_dataset(data_name, mode, data_embedding, data_seed)
data_iter, data_hd = _make_dataset_iterator(
type_name='initializable_iterator',
handle_name='data_handle',
dataset=dataset,
batch_size=batch_size,
shuffle=False,
pad=data_pad,
session=sess)
_restore_model(sess, model_path, step)
y_preds, y_trues = [], [] # type: ignore
sess.run(data_iter.initializer)
while True:
try:
true, pred = sess.run(
[model.y, model.prediction],
feed_dict={
model.handle: data_hd,
model.keep_prob: 1.0,
model.is_training: False
})
y_preds.extend(np.squeeze(pred).tolist())
y_trues.extend(np.squeeze(true).tolist())
except tf.errors.OutOfRangeError:
break
# print accuracy
print('Acc: %.4f' % sklearn.metrics.accuracy_score(y_trues, y_preds))
# Print confusion matrix
labels = list(
sorted(data.SNLI.LABELS.keys(), key=lambda x: data.SNLI.LABELS[x]))
cm = sklearn.metrics.confusion_matrix(y_trues,
y_preds,
labels=range(len(labels)))
tmpl = '%15s ' * (len(labels) + 2)
print(tmpl % tuple([''] + labels + ['']))
corr = 0
for i in range(len(labels)):
stats = cm[i]
prob = stats[i] / sum(stats)
corr += stats[i]
print(tmpl %
tuple([labels[i]] + list(map(str, cm[i])) + ['%.4f' % prob]))
print(tmpl % tuple(['%d / %d' % (corr, len(y_trues))] +
[''] * len(labels) + ['%.4f' % (corr / len(y_trues))]))
# Print errors
if print_errors:
tmpl = '\n%4d. Pred: %-20s True: %s\n %s\n %s'
for i, (y_pred, y_true) in enumerate(zip(y_preds, y_trues)):
if y_pred != y_true and print_errors_limit != 0:
s1 = ' '.join(dataset.x1_words[i])
s2 = ' '.join(dataset.x2_words[i])
l_pred = labels[y_pred]
l_true = labels[y_true]
print(tmpl % (i, l_pred, l_true, s1, s2))
print_errors_limit -= 1
def train(name: str,
model_type: str,
batch_size: int = 256,
epoch_num: int = 200,
keep_prob: float = 0.8,
train_regex_list: t.Union[t.List[str], str] = None,
optim_manager_type: str = 'NotChange',
data_name: str = 'SNLI',
data_embedding: str = 'GloVe',
data_argument: bool = False,
data_pad: bool = True,
data_cache: bool = False,
data_seed: int = None,
record_every: int = 64000,
validate_every: int = 640000,
save_every: int = 6400000,
restore_from: str = None,
restore_step: int = None,
profiling: bool = False,
clip_norm: int = None,
seed: int = None,
debug: bool = False,
**kwargs) -> None:
# Data preparation
model_path = build.get_model_path(name)
shutil.rmtree(model_path, ignore_errors=True) # remove previous trained
# Network setup
model = getattr(nn, model_type)(embeddings=data.load_embeddings(
data_name, data_embedding, data_seed),
**_select_kwargs_regex(kwargs,
r'^optim[0-9]*_',
invert=True))
log.info(str(model))
log.debug('Model parameters:\n\n\t' +
'\n\t'.join(graph.print_trainable_variables().split('\n')))
# Control randomization
if seed:
log.info(
'Set random seed for data shuffling and graph computation: %d' %
seed)
tf.set_random_seed(seed)
train_summary = _make_model_summary(model)
with tf.Session(config=_make_config()) as sess:
if debug:
from tensorflow.python import debug as tf_debug
sess = tf_debug.LocalCLIDebugWrapperSession(sess)
dataset_opts = {
'pad': data_pad,
'batch_size': batch_size,
'session': sess,
}
train_iter, train_hd = _make_dataset_iterator(
type_name='one_shot_iterator',
handle_name='train_handle',
dataset=data.load_dataset(data_name, 'train', data_embedding,
data_seed),
argument=data_argument,
bucket_boundaries=[20, 50],
repeat_num=epoch_num,
cache=data_cache,
seed=seed,
**dataset_opts)
valid_iter, valid_hd = _make_dataset_iterator(
type_name='initializable_iterator',
handle_name='valid_handle',
dataset=data.load_dataset(data_name, 'validation', data_embedding,
data_seed),
shuffle=False,
cache=True,
**dataset_opts)
test_iter, test_hd = _make_dataset_iterator(
type_name='initializable_iterator',
handle_name='test_handle',
dataset=data.load_dataset(data_name, 'test', data_embedding,
data_seed),
shuffle=False,
cache=True,
**dataset_opts)
om = _make_optim_manager(optim_manager_type, model.loss, clip_norm,
train_regex_list, kwargs)
test_wtr = tf.summary.FileWriter(os.path.join(model_path, 'test'))
train_wtr = tf.summary.FileWriter(os.path.join(model_path, 'train'),
sess.graph)
# Build a validation summary writer for each optimizer
valid_wtr = {}
for optim in om.optims:
valid_wtr[optim.get_name()] = tf.summary.FileWriter(
os.path.join(model_path, 'valid-%s' % optim.get_name()))
if restore_from:
_copy_checkpoint(restore_from, model_path, restore_step)
_restore_model(sess, model_path, restore_step)
# Evaluate the pretrained model
step = restore_step
_iterate_dataset(sess, model, valid_iter, valid_hd,
valid_wtr[om.optim.get_name()], step)
_iterate_dataset(sess, model, test_iter, test_hd, test_wtr, step)
else:
sess.run(tf.global_variables_initializer())
step = 0
if profiling:
_profile_and_exit(sess, model, om.optim_op, train_hd)
pbar = tqdm.tqdm(total=save_every, desc='Train', unit=' inst')
try:
while True:
feed_dict = {
model.handle: train_hd,
model.keep_prob: keep_prob,
model.is_training: True
}
if om.feed_lr:
feed_dict[om.lr_op] = om.lr_val
if step % record_every == 0:
summary, _, loss = sess.run(
[train_summary, om.optim_op, model.loss],
feed_dict=feed_dict)
pbar.set_postfix(loss='{:.3f}'.format(loss))
train_wtr.add_summary(summary, step)
else:
sess.run([om.optim_op], feed_dict=feed_dict)
if step and step % validate_every == 0:
pbar.set_description('Valid')
valid_acc = _iterate_dataset(
sess, model, valid_iter, valid_hd,
valid_wtr[om.optim.get_name()], step)
# Update upon the validation perfomance
om.update(valid_acc, step)
pbar.set_description('Test')
_iterate_dataset(sess, model, test_iter, test_hd, test_wtr,
step)
pbar.set_description('Train')
if step and step % save_every == 0:
save_path = _save_model(sess, model_path, step)
pbar.set_description(save_path)
pbar.update(batch_size)
pbar.close()
pbar = tqdm.tqdm(total=save_every,
desc='Train',
unit=' inst')
else:
pbar.update(batch_size)
step += batch_size
except tf.errors.OutOfRangeError:
save_path = _save_model(sess, model_path, step)
pbar.set_description(save_path)
log.info('Training finished!')
def _restore_model(session, model_path, step):
saved_path = build.get_saved_model(model_path, step)
log.info('Restore pre-trained model from: %s' % saved_path)
tf.train.Saver().restore(session, saved_path)