def run_model(self, train_config, eval_config): with tf.Graph().as_default() as g: train_model = base_model(params=train_config, mode="train", hvd=None) train_model.compile() eval_model = base_model(params=eval_config, mode="eval", hvd=None) eval_model.compile(force_var_reuse=True) train(train_model, eval_model) saver = tf.train.Saver() checkpoint = tf.train.latest_checkpoint( train_model.params['logdir']) with self.test_session(g, use_gpu=True) as sess: saver.restore(sess, checkpoint) sess.run([ train_model.get_data_layer(i).iterator.initializer for i in range(train_model.num_gpus) ]) sess.run([ eval_model.get_data_layer(i).iterator.initializer for i in range(eval_model.num_gpus) ]) weights = sess.run(tf.trainable_variables()) loss = sess.run(train_model.loss) eval_losses = sess.run(eval_model.eval_losses) eval_loss = np.mean(eval_losses) weights_new = sess.run(tf.trainable_variables()) # checking that the weights has not changed from just computing the loss for w, w_new in zip(weights, weights_new): npt.assert_allclose(w, w_new) eval_dict = evaluate(eval_model, checkpoint) return loss, eval_loss, eval_dict
def run_model(self, train_config, eval_config): with tf.Graph().as_default() as g: train_model = base_model(params=train_config, mode="train", hvd=None) train_model.compile() eval_model = base_model(params=eval_config, mode="eval", hvd=None) eval_model.compile(force_var_reuse=True) train(train_model, eval_model) saver = tf.train.Saver() checkpoint = tf.train.latest_checkpoint(train_model.params['logdir']) with self.test_session(g, use_gpu=True) as sess: saver.restore(sess, checkpoint) sess.run([train_model.get_data_layer(i).iterator.initializer for i in range(train_model.num_gpus)]) sess.run([eval_model.get_data_layer(i).iterator.initializer for i in range(eval_model.num_gpus)]) weights = sess.run(tf.trainable_variables()) loss = sess.run(train_model.loss) eval_losses = sess.run(eval_model.eval_losses) eval_loss = np.mean(eval_losses) weights_new = sess.run(tf.trainable_variables()) # checking that the weights has not changed from just computing the loss for w, w_new in zip(weights, weights_new): npt.assert_allclose(w, w_new) eval_dict = evaluate(eval_model, checkpoint) return loss, eval_loss, eval_dict
def test_get_batches_for_epoch(self): # this will take all gpu memory, but that's probably fine for tests gpus = get_available_gpus() length_list = [] for num_gpus in [1, 2, 3]: if num_gpus > len(gpus): continue for bs in [1, 2, 3, 5, 7]: if bs * num_gpus > 10: continue with tf.Graph().as_default() as g: self.eval_config['batch_size_per_gpu'] = bs self.eval_config['num_gpus'] = num_gpus model = base_model(params=self.eval_config, mode="eval", hvd=None) model.compile() model.evaluate = lambda inputs, outputs: inputs model.finalize_evaluation = lambda results: results with self.test_session(g, use_gpu=True) as sess: sess.run(tf.global_variables_initializer()) inputs_per_batch = get_results_for_epoch( model, sess, False, "eval") length_list.append( np.hstack([ inp['source_tensors'][1] for inp in inputs_per_batch ])) for i in range(len(length_list) - 1): npt.assert_allclose(length_list[i], length_list[i + 1])
def test_get_batches_for_epoch(self): # this will take all gpu memory, but that's probably fine for tests gpus = get_available_gpus() length_list = [] for num_gpus in [1, 2, 3]: if num_gpus > len(gpus): continue for bs in [1, 2, 3, 5, 7]: if bs * num_gpus > 10: continue with tf.Graph().as_default() as g: self.eval_config['batch_size_per_gpu'] = bs self.eval_config['num_gpus'] = num_gpus model = base_model(params=self.eval_config, mode="eval", hvd=None) model.compile() model.evaluate = lambda inputs, outputs: inputs model.finalize_evaluation = lambda results: results with self.test_session(g, use_gpu=True) as sess: sess.run(tf.global_variables_initializer()) inputs_per_batch = get_results_for_epoch(model, sess, False, "eval") length_list.append(np.hstack([inp['source_tensors'][1] for inp in inputs_per_batch])) for i in range(len(length_list) - 1): npt.assert_allclose(length_list[i], length_list[i + 1])
def test_infer(self): train_config, infer_config = self.prepare_config() train_config['num_epochs'] = 200 infer_config['batch_size_per_gpu'] = 4 with tf.Graph().as_default() as g: with self.test_session(g, use_gpu=True) as sess: gpus = get_available_gpus() if len(gpus) > 1: infer_config['num_gpus'] = 2 else: infer_config['num_gpus'] = 1 with tf.Graph().as_default(): train_model = base_model(params=train_config, mode="train", hvd=None) train_model.compile() train(train_model, None) with tf.Graph().as_default(): infer_model = base_model(params=infer_config, mode="infer", hvd=None) infer_model.compile() print(train_model.params['logdir']) output_file = os.path.join(train_model.params['logdir'], 'infer_out.csv') infer( infer_model, tf.train.latest_checkpoint(train_model.params['logdir']), output_file, ) pred_csv = pd.read_csv(output_file) true_csv = pd.read_csv( 'open_seq2seq/test_utils/toy_speech_data/toy_data.csv', ) for pred_row, true_row in zip(pred_csv.as_matrix(), true_csv.as_matrix()): # checking file name self.assertEqual(pred_row[0], true_row[0]) # checking prediction self.assertEqual(pred_row[-1], true_row[-1])
def test_mp_collection(self): train_config, eval_config = self.prepare_config() train_config['dtype'] = 'mixed' with tf.Graph().as_default(): model = base_model(params=train_config, mode="train", hvd=None) model.compile() self.assertEqual(len(tf.trainable_variables()), 14) self.assertEqual( len(tf.get_collection('FP32_MASTER_COPIES')), 7, # minus batch norm beta and gamma and row_conv vars )
def test_mp_collection(self): train_config, eval_config = self.prepare_config() train_config['dtype'] = 'mixed' with tf.Graph().as_default(): model = base_model(params=train_config, mode="train", hvd=None) model.compile() self.assertEqual(len(tf.trainable_variables()), 14) self.assertEqual( len(tf.get_collection('FP32_MASTER_COPIES')), 7, # minus batch norm beta and gamma and row_conv vars )
def test_infer(self): train_config, infer_config = self.prepare_config() train_config['num_epochs'] = 200 infer_config['batch_size_per_gpu'] = 4 with tf.Graph().as_default() as g: with self.test_session(g, use_gpu=True) as sess: gpus = get_available_gpus() if len(gpus) > 1: infer_config['num_gpus'] = 2 else: infer_config['num_gpus'] = 1 with tf.Graph().as_default(): train_model = base_model(params=train_config, mode="train", hvd=None) train_model.compile() train(train_model, None) with tf.Graph().as_default(): infer_model = base_model(params=infer_config, mode="infer", hvd=None) infer_model.compile() print(train_model.params['logdir']) output_file = os.path.join(train_model.params['logdir'], 'infer_out.csv') infer( infer_model, tf.train.latest_checkpoint(train_model.params['logdir']), output_file, ) pred_csv = pd.read_csv(output_file) true_csv = pd.read_csv( 'open_seq2seq/test_utils/toy_speech_data/toy_data.csv', ) for pred_row, true_row in zip(pred_csv.as_matrix(), true_csv.as_matrix()): # checking file name self.assertEqual(pred_row[0], true_row[0]) # checking prediction self.assertEqual(pred_row[-1], true_row[-1])
def test_maybe_functions(self): train_config, eval_config = self.prepare_config() with tf.Graph().as_default(): model = base_model(params=train_config, mode="train", hvd=None) model.compile() model._gpu_ids = range(5) model.params['batch_size_per_gpu'] = 2 char2idx = model.data_layer.params['char2idx'] inputs = [ ['this is a great day', 'london is the capital of great britain'], ['ooo', 'lll'], ['a b c\' asdf', 'blah blah bblah'], ['this is great day', 'london capital gret britain'], ['aaaaaaaasdfdasdf', 'df d sdf asd fd f sdf df blah\' blah'], ] outputs = [ ['this is great a day', 'london capital gret britain'], ['ooo', 'lll'], ['aaaaaaaasdfdasdf', 'df d sdf asd fd f sdf df blah blah'], ['this is a great day', 'london is the capital of great britain'], ['a b c\' asdf', 'blah blah\' bblah'], ] y = [None] * len(inputs) len_y = [None] * len(inputs) indices, values, dense_shape = [], [], [] num_gpus = len(inputs) for gpu_id in range(num_gpus): num_samples = len(inputs[gpu_id]) max_len = np.max(list(map(len, inputs[gpu_id]))) y[gpu_id] = np.zeros((num_samples, max_len), dtype=np.int) len_y[gpu_id] = np.zeros(num_samples, dtype=np.int) for sample_id in range(num_samples): num_letters = len(inputs[gpu_id][sample_id]) len_y[gpu_id][sample_id] = num_letters for letter_id in range(num_letters): y[gpu_id][sample_id, letter_id] = char2idx[ inputs[gpu_id][sample_id][letter_id]] num_gpus = len(outputs) for gpu_id in range(num_gpus): num_samples = len(outputs[gpu_id]) max_len = np.max(list(map(len, outputs[gpu_id]))) dense_shape.append(np.array((num_samples, max_len))) values.append([]) indices.append([]) for sample_id in range(num_samples): num_letters = len(outputs[gpu_id][sample_id]) for letter_id in range(num_letters): values[gpu_id].append( char2idx[outputs[gpu_id][sample_id][letter_id]]) indices[gpu_id].append(np.array([sample_id, letter_id])) values[gpu_id] = np.array(values[gpu_id], dtype=np.int) indices[gpu_id] = np.array(indices[gpu_id], dtype=np.int) input_values = [None, None, y, len_y] output_values = [ tf.SparseTensorValue(indices[i], values[i], dense_shape[i]) for i in range(num_gpus) ] output_dict = model.maybe_evaluate([input_values, input_values], [output_values, output_values]) w_lev = 0.0 w_len = 0.0 for batch_id in range(len(inputs)): for sample_id in range(len(inputs[batch_id])): input_sample = inputs[batch_id][sample_id] output_sample = outputs[batch_id][sample_id] w_lev += levenshtein(input_sample.split(), output_sample.split()) w_len += len(input_sample.split()) self.assertEqual(output_dict['Eval WER'], w_lev / w_len) self.assertEqual(output_dict['Eval WER'], 37 / 40.0) output_dict = model.maybe_print_logs(input_values, output_values) self.assertEqual(output_dict['Sample WER'], 0.4)
def test_maybe_functions(self): train_config, eval_config = self.prepare_config() with tf.Graph().as_default(): model = base_model(params=train_config, mode="train", hvd=None) model.compile() model._gpu_ids = range(5) model.params['batch_size_per_gpu'] = 2 char2idx = model.get_data_layer().params['char2idx'] inputs = [ ['this is a great day', 'london is the capital of great britain'], ['ooo', 'lll'], ['a b c\' asdf', 'blah blah bblah'], ['this is great day', 'london capital gret britain'], ['aaaaaaaasdfdasdf', 'df d sdf asd fd f sdf df blah\' blah'], ] outputs = [ ['this is great a day', 'london capital gret britain'], ['ooo', 'lll'], ['aaaaaaaasdfdasdf', 'df d sdf asd fd f sdf df blah blah'], ['this is a great day', 'london is the capital of great britain'], ['a b c\' asdf', 'blah blah\' bblah'], ] y = [None] * len(inputs) len_y = [None] * len(inputs) indices, values, dense_shape = [], [], [] num_gpus = len(inputs) for gpu_id in range(num_gpus): num_samples = len(inputs[gpu_id]) max_len = np.max(list(map(len, inputs[gpu_id]))) y[gpu_id] = np.zeros((num_samples, max_len), dtype=np.int) len_y[gpu_id] = np.zeros(num_samples, dtype=np.int) for sample_id in range(num_samples): num_letters = len(inputs[gpu_id][sample_id]) len_y[gpu_id][sample_id] = num_letters for letter_id in range(num_letters): y[gpu_id][sample_id, letter_id] = char2idx[ inputs[gpu_id][sample_id][letter_id] ] num_gpus = len(outputs) for gpu_id in range(num_gpus): num_samples = len(outputs[gpu_id]) max_len = np.max(list(map(len, outputs[gpu_id]))) dense_shape.append(np.array((num_samples, max_len))) values.append([]) indices.append([]) for sample_id in range(num_samples): num_letters = len(outputs[gpu_id][sample_id]) for letter_id in range(num_letters): values[gpu_id].append( char2idx[outputs[gpu_id][sample_id][letter_id]] ) indices[gpu_id].append(np.array([sample_id, letter_id])) values[gpu_id] = np.array(values[gpu_id], dtype=np.int) indices[gpu_id] = np.array(indices[gpu_id], dtype=np.int) x = [np.empty(2)] * len(y) len_x = [None] * len(y) input_values = list(zip(x, len_x, y, len_y)) output_values = [ [tf.SparseTensorValue(indices[i], values[i], dense_shape[i])] for i in range(num_gpus) ] results = [] for inp, out in zip(input_values, output_values): inp_dict = {'source_tensors': [inp[0], inp[1]], 'target_tensors': [inp[2], inp[3]]} results.append(model.evaluate(inp_dict, out)) for inp, out in zip(input_values, output_values): inp_dict = {'source_tensors': [inp[0], inp[1]], 'target_tensors': [inp[2], inp[3]]} results.append(model.evaluate(inp_dict, out)) output_dict = model.finalize_evaluation(results) w_lev = 0.0 w_len = 0.0 for batch_id in range(len(inputs)): for sample_id in range(len(inputs[batch_id])): input_sample = inputs[batch_id][sample_id] output_sample = outputs[batch_id][sample_id] w_lev += levenshtein(input_sample.split(), output_sample.split()) w_len += len(input_sample.split()) self.assertEqual(output_dict['Eval WER'], w_lev / w_len) self.assertEqual(output_dict['Eval WER'], 37 / 40.0) inp_dict = {'source_tensors': [input_values[0][0], input_values[0][1]], 'target_tensors': [input_values[0][2], input_values[0][3]]} output_dict = model.maybe_print_logs(inp_dict, output_values[0]) self.assertEqual(output_dict['Sample WER'], 0.4)