def _copy_checkpoint(from_model, to_path, step): """ Copy checkpoint files of a specific training step. """ from_path = build.get_model_path(from_model) log.info('Copy step %d checkpoint files from: %s' % (step, from_path)) files = glob.glob('{}/model-{}.*'.format(from_path, step)) for f in files: shutil.copy(f, to_path)
def _make_optim_manager(optim_manager_type, loss_op, clip_norm, train_regex_list, kwargs): om = getattr(optim, optim_manager_type)( loss_op, clip_norm, var_list=_search_var_list(train_regex_list), # None for all vars. **_select_kwargs_regex(kwargs, regex=r'^optim_manager(?!_type)', start=14)) log.info(str(om)) idx = 1 if _select_kwargs_regex(kwargs, regex=r'^optim_(?!manager)'): # Construct optimizer with 'optim_*' arguments optim_kwargs = _select_kwargs_regex(kwargs, regex=r'^optim_(?!manager)', start=6) om.add_optimizer(**optim_kwargs) else: while True: # Construct optimizer with 'optim{idx}_*' arguments optim_kwargs = _select_kwargs_regex(kwargs, regex=r'^optim%d_' % idx, start=6 + len(str(idx))) if not optim_kwargs: break om.add_optimizer(**optim_kwargs) idx += 1 return om
def __init__(self, path, dim, binary): super(PretrainedEmbedding, self).__init__(dim) self.path = path self.__binary = binary self.__model = None log.info('Read pretrained %s embedding from file: %s' % (self.__class__.__name__, self.path)) self.__model = gensim.models.KeyedVectors.load_word2vec_format( self.path, binary=self.__binary)
def _search_var_list(var_regex_list: t.Union[t.List[str], str]): if isinstance(var_regex_list, str): var_regex_list = [var_regex_list] if var_regex_list: pattern_list = [re.compile(regex) for regex in var_regex_list] var_list = [ var for var in tf.trainable_variables() if any(map(lambda p: p.match(var.name), pattern_list)) ] log.info('Partical updation on parameters: \n\n\t%s\n' % '\n\t'.join(map(lambda v: v.name, var_list))) return var_list
def update(self, step_acc, global_step) -> None: if step_acc < self._max_acc + self.min_delta: self._no_update_cnt += 1 log.debug('No update for %d consequtive times.' % self._no_update_cnt) if self._no_update_cnt >= self.patience: self._lr_val[self._idx] /= 2 log.info('Half the current learning rate to : %f' % self._lr_val[self._idx]) self._no_update_cnt = 0 else: self._no_update_cnt = 0 self._max_acc = max(self._max_acc, step_acc)
def next(self): """ Set the next available optimizer for training. """ if self.has_next(): self._idx += 1 log.info('Train model using %s Optimizer' % self.optim.get_name()) grads_tvars = self.optim.compute_gradients(**self._compute_kwargs) if self.clip_norm: log.info('Apply global gradient clipping with norm %f' % self.clip_norm) grads = [grad for grad, tvar in grads_tvars] tvars = [tvar for grad, tvar in grads_tvars] grads, _ = tf.clip_by_global_norm(grads, self.clip_norm) grads_tvars = zip(grads, tvars) self.optim_op = self.optim.apply_gradients(grads_tvars)
def __init__(self, glove_path='/home/jmei/data/glove.840B.300d.txt', path='/home/jmei/data/glove.840B.300d.norm.txt', dim=300, binary=False, **kwargs): if not os.path.exists(path): # Preprocess the original GloVe data to allow using the gensim API. log.info('Generate normalized GloVe embeddings to file: %s' % path) with open(glove_path, 'r') as in_file: with open(path, 'w') as out_file: for l in in_file: fields = l.split(' ') name = fields[:-300] embed = list(map(float, fields[-300:])) embed_norm = embed / np.linalg.norm(embed) out_file.write(' '.join(name) + ' ' + ' '.join(map(str, embed_norm)) + '\n') super(GloVeNorm, self).__init__(path, dim, binary, **kwargs)
def _profile_and_exit(session, model, optimizer, handle): """ Profile the run metadata at the first iteration and exit the program. """ from tensorflow.python.client import timeline run_metadata = tf.RunMetadata() for i in range(5): session.run( [optimizer], feed_dict={ model.handle: handle, model.keep_prob: 1.0, model.is_training: True }, options=tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE), run_metadata=run_metadata) tl = timeline.Timeline(run_metadata.step_stats) ctf = tl.generate_chrome_trace_format() with open('timeline_%d.json' % i, 'w') as f: f.write(ctf) log.info('Profiles are created! Now exit.') exit()
def load_embeddings(data_name: str, embedding_name: str, seed: int = None): """ Load a indexed word embedding object. This method will first check if the given setup have previously been serialized, restore the object from file. Otherwise, construct a new object. """ # The naming convention of the serialization path. if seed: pkl_path = os.path.join( build.BUILD_DIR, 'data', '{}.{}.seed{}.pkl'.format(data_name, embedding_name, seed)) else: pkl_path = os.path.join(build.BUILD_DIR, 'data', '{}.{}.pkl'.format(data_name, embedding_name)) if os.path.exists(pkl_path): log.info( 'Restore corpus-specific indexed word embedding from file: %s' % pkl_path) with open(pkl_path, 'rb') as pkl_file: embeds = pickle.load(pkl_file) else: if seed: log.debug('Set numpy random seed to %d' % seed) np.random.seed(seed) log.info('Build corpus-specific indexed word embedding.') embeds = globals()[data_name].init_indexed_word_embedding( embed.init(embedding_name)) # Serialize for reusing log.info('Save corpus-specific indexed word embedding to file: %s.' % pkl_path) os.makedirs(os.path.normpath(os.path.join(pkl_path, os.pardir)), exist_ok=True) with open(pkl_path, 'wb') as pkl_file: pickle.dump(embeds, pkl_file, 4) return embeds
def load_dataset(data_name: str, data_mode: str, embedding_name: str, seed: int = None) -> Dataset: if seed: pkl_path = os.path.join( build.BUILD_DIR, 'data', '{}-{}.{}.seed{}.pkl'.format(data_name, data_mode, embedding_name, seed)) else: pkl_path = os.path.join( build.BUILD_DIR, 'data', '{}-{}.{}.pkl'.format(data_name, data_mode, embedding_name)) # Load preprocessed data object from pkl if applicable. if os.path.exists(pkl_path): log.info('Restore %s %s dataset from file: %s' % (data_name, data_mode, pkl_path)) with open(pkl_path, 'rb') as pkl_file: dataset = pickle.load(pkl_file) else: log.info('Build %s %s dataset' % (data_name, data_mode)) embedding = load_embeddings(data_name, embedding_name, seed) if seed: log.debug('Set numpy random seed to %d' % seed) np.random.seed(seed) dataset = globals()[data_name](data_mode, indexed_word_embedding=embedding) os.makedirs(os.path.normpath(os.path.join(pkl_path, os.pardir)), exist_ok=True) log.info('Serialize %s %s dataset to file %s.' % (data_mode, data_name, pkl_path)) with open(pkl_path, 'wb') as pkl_file: pickle.dump(dataset, pkl_file, 4) return dataset
def test( name: str, model_type: str, step: int = None, mode: str = 'test', data_seed: int = None, data_name: str = 'SNLI', data_embedding: str = 'GloVe', data_pad: bool = True, batch_size: int = 10, print_errors: bool = False, print_errors_limit: int = 10, **kwargs, ) -> None: model_path = build.get_model_path(name) model = getattr(nn, model_type)(embeddings=data.load_embeddings( data_name, data_embedding, data_seed), **kwargs) log.info(str(model)) log.debug('Model parameters:\n\n\t' + '\n\t'.join(graph.print_trainable_variables().split('\n'))) with tf.Session(config=_make_config()) as sess: dataset = data.load_dataset(data_name, mode, data_embedding, data_seed) data_iter, data_hd = _make_dataset_iterator( type_name='initializable_iterator', handle_name='data_handle', dataset=dataset, batch_size=batch_size, shuffle=False, pad=data_pad, session=sess) _restore_model(sess, model_path, step) y_preds, y_trues = [], [] # type: ignore sess.run(data_iter.initializer) while True: try: true, pred = sess.run( [model.y, model.prediction], feed_dict={ model.handle: data_hd, model.keep_prob: 1.0, model.is_training: False }) y_preds.extend(np.squeeze(pred).tolist()) y_trues.extend(np.squeeze(true).tolist()) except tf.errors.OutOfRangeError: break # print accuracy print('Acc: %.4f' % sklearn.metrics.accuracy_score(y_trues, y_preds)) # Print confusion matrix labels = list( sorted(data.SNLI.LABELS.keys(), key=lambda x: data.SNLI.LABELS[x])) cm = sklearn.metrics.confusion_matrix(y_trues, y_preds, labels=range(len(labels))) tmpl = '%15s ' * (len(labels) + 2) print(tmpl % tuple([''] + labels + [''])) corr = 0 for i in range(len(labels)): stats = cm[i] prob = stats[i] / sum(stats) corr += stats[i] print(tmpl % tuple([labels[i]] + list(map(str, cm[i])) + ['%.4f' % prob])) print(tmpl % tuple(['%d / %d' % (corr, len(y_trues))] + [''] * len(labels) + ['%.4f' % (corr / len(y_trues))])) # Print errors if print_errors: tmpl = '\n%4d. Pred: %-20s True: %s\n %s\n %s' for i, (y_pred, y_true) in enumerate(zip(y_preds, y_trues)): if y_pred != y_true and print_errors_limit != 0: s1 = ' '.join(dataset.x1_words[i]) s2 = ' '.join(dataset.x2_words[i]) l_pred = labels[y_pred] l_true = labels[y_true] print(tmpl % (i, l_pred, l_true, s1, s2)) print_errors_limit -= 1
def train(name: str, model_type: str, batch_size: int = 256, epoch_num: int = 200, keep_prob: float = 0.8, train_regex_list: t.Union[t.List[str], str] = None, optim_manager_type: str = 'NotChange', data_name: str = 'SNLI', data_embedding: str = 'GloVe', data_argument: bool = False, data_pad: bool = True, data_cache: bool = False, data_seed: int = None, record_every: int = 64000, validate_every: int = 640000, save_every: int = 6400000, restore_from: str = None, restore_step: int = None, profiling: bool = False, clip_norm: int = None, seed: int = None, debug: bool = False, **kwargs) -> None: # Data preparation model_path = build.get_model_path(name) shutil.rmtree(model_path, ignore_errors=True) # remove previous trained # Network setup model = getattr(nn, model_type)(embeddings=data.load_embeddings( data_name, data_embedding, data_seed), **_select_kwargs_regex(kwargs, r'^optim[0-9]*_', invert=True)) log.info(str(model)) log.debug('Model parameters:\n\n\t' + '\n\t'.join(graph.print_trainable_variables().split('\n'))) # Control randomization if seed: log.info( 'Set random seed for data shuffling and graph computation: %d' % seed) tf.set_random_seed(seed) train_summary = _make_model_summary(model) with tf.Session(config=_make_config()) as sess: if debug: from tensorflow.python import debug as tf_debug sess = tf_debug.LocalCLIDebugWrapperSession(sess) dataset_opts = { 'pad': data_pad, 'batch_size': batch_size, 'session': sess, } train_iter, train_hd = _make_dataset_iterator( type_name='one_shot_iterator', handle_name='train_handle', dataset=data.load_dataset(data_name, 'train', data_embedding, data_seed), argument=data_argument, bucket_boundaries=[20, 50], repeat_num=epoch_num, cache=data_cache, seed=seed, **dataset_opts) valid_iter, valid_hd = _make_dataset_iterator( type_name='initializable_iterator', handle_name='valid_handle', dataset=data.load_dataset(data_name, 'validation', data_embedding, data_seed), shuffle=False, cache=True, **dataset_opts) test_iter, test_hd = _make_dataset_iterator( type_name='initializable_iterator', handle_name='test_handle', dataset=data.load_dataset(data_name, 'test', data_embedding, data_seed), shuffle=False, cache=True, **dataset_opts) om = _make_optim_manager(optim_manager_type, model.loss, clip_norm, train_regex_list, kwargs) test_wtr = tf.summary.FileWriter(os.path.join(model_path, 'test')) train_wtr = tf.summary.FileWriter(os.path.join(model_path, 'train'), sess.graph) # Build a validation summary writer for each optimizer valid_wtr = {} for optim in om.optims: valid_wtr[optim.get_name()] = tf.summary.FileWriter( os.path.join(model_path, 'valid-%s' % optim.get_name())) if restore_from: _copy_checkpoint(restore_from, model_path, restore_step) _restore_model(sess, model_path, restore_step) # Evaluate the pretrained model step = restore_step _iterate_dataset(sess, model, valid_iter, valid_hd, valid_wtr[om.optim.get_name()], step) _iterate_dataset(sess, model, test_iter, test_hd, test_wtr, step) else: sess.run(tf.global_variables_initializer()) step = 0 if profiling: _profile_and_exit(sess, model, om.optim_op, train_hd) pbar = tqdm.tqdm(total=save_every, desc='Train', unit=' inst') try: while True: feed_dict = { model.handle: train_hd, model.keep_prob: keep_prob, model.is_training: True } if om.feed_lr: feed_dict[om.lr_op] = om.lr_val if step % record_every == 0: summary, _, loss = sess.run( [train_summary, om.optim_op, model.loss], feed_dict=feed_dict) pbar.set_postfix(loss='{:.3f}'.format(loss)) train_wtr.add_summary(summary, step) else: sess.run([om.optim_op], feed_dict=feed_dict) if step and step % validate_every == 0: pbar.set_description('Valid') valid_acc = _iterate_dataset( sess, model, valid_iter, valid_hd, valid_wtr[om.optim.get_name()], step) # Update upon the validation perfomance om.update(valid_acc, step) pbar.set_description('Test') _iterate_dataset(sess, model, test_iter, test_hd, test_wtr, step) pbar.set_description('Train') if step and step % save_every == 0: save_path = _save_model(sess, model_path, step) pbar.set_description(save_path) pbar.update(batch_size) pbar.close() pbar = tqdm.tqdm(total=save_every, desc='Train', unit=' inst') else: pbar.update(batch_size) step += batch_size except tf.errors.OutOfRangeError: save_path = _save_model(sess, model_path, step) pbar.set_description(save_path) log.info('Training finished!')
def _restore_model(session, model_path, step): saved_path = build.get_saved_model(model_path, step) log.info('Restore pre-trained model from: %s' % saved_path) tf.train.Saver().restore(session, saved_path)