def train(self):
train_set = SNLIDataloaderPairs('data/snli_1.0/snli_1.0_train.jsonl')
train_set.load_vocab('./data/snli_vocab.dat', self.config.vocab_size)
train_set.set_preprocess_fn(preprocess_fn)
train_set.set_output_fn(output_fn)
# test_set = Dataloader(self.config, 'data/test_stories.csv', testing_data=True)
# test_set.load_dataset('data/test.bin')
# test_set.load_vocab('./data/default.voc', self.config.vocab_size)
# test_set.set_preprocess_fn(preprocess_fn)
# test_set.set_output_fn(output_fn_test)
generator_training = train_set.get_batch(1, 1)
print(next(generator_training))
def main(config):
import sent2vec
assert config.sent2vec.model is not None, "Please add sent2vec_model config value."
sent2vec_model = sent2vec.Sent2vecModel()
sent2vec_model.load_model(config.sent2vec.model)
preprocess_fn = Preprocess(sent2vec_model)
output_fn_test = OutputFnTest(sent2vec_model, config)
train_set = SNLIDataloaderPairs('data/snli_1.0/snli_1.0_train.jsonl')
train_set.set_preprocess_fn(preprocess_fn)
train_set.set_output_fn(output_fn)
test_set = Dataloader(config, 'data/test_stories.csv', testing_data=True)
test_set.load_dataset('data/test.bin')
test_set.load_vocab('./data/default.voc', config.vocab_size)
test_set.set_output_fn(output_fn_test)
# dev_set = SNLIDataloader('data/snli_1.0/snli_1.0_dev.jsonl')
# dev_set.set_preprocess_fn(preprocess_fn)
# dev_set.set_output_fn(output_fn)
# test_set = SNLIDataloader('data/snli_1.0/snli_1.0_test.jsonl')
generator_training = train_set.get_batch(config.batch_size,
config.n_epochs)
generator_dev = test_set.get_batch(config.batch_size, config.n_epochs)
keras_model = model(config)
verbose = 0 if not config.debug else 1
timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# Callbacks
tensorboard = keras.callbacks.TensorBoard(log_dir='./logs/' + timestamp +
'-entailmentv5/',
histogram_freq=0,
batch_size=config.batch_size,
write_graph=False,
write_grads=True)
model_path = os.path.abspath(
os.path.join(os.curdir, './builds/' + timestamp))
model_path += '-entailmentv5_checkpoint_epoch-{epoch:02d}.hdf5'
saver = keras.callbacks.ModelCheckpoint(model_path,
monitor='val_loss',
verbose=verbose,
save_best_only=True)
keras_model.fit_generator(generator_training,
steps_per_epoch=300,
epochs=config.n_epochs,
verbose=verbose,
validation_data=generator_dev,
validation_steps=len(test_set) /
config.batch_size,
callbacks=[tensorboard, saver])
def eval(self):
# Initialize tensorflow session
sess = tf.Session()
K.set_session(sess) # Set to keras backend
if self.config.debug:
print('Importing Elmo module...')
if self.config.hub.is_set("cache_dir"):
os.environ['TFHUB_CACHE_DIR'] = self.config.hub.cache_dir
elmo_model = hub.Module("https://tfhub.dev/google/elmo/1",
trainable=True)
if self.config.debug:
print('Imported.')
sess.run(tf.global_variables_initializer())
sess.run(tf.tables_initializer())
graph = tf.get_default_graph()
elmo_emb_fn = ElmoEmbedding(elmo_model)
elmo_model_emb = get_elmo_embedding(elmo_emb_fn)
output_fn = OutputFN(elmo_model_emb, graph)
test_set = SNLIDataloaderPairs('data/snli_1.0/snli_1.0_test.jsonl')
test_set.set_preprocess_fn(preprocess_fn)
test_set.set_output_fn(output_fn)
generator_test = test_set.get_batch(self.config.batch_size,
self.config.n_epochs)
# keras_model = model(elmo_emb_fn)
verbose = 0 if not self.config.debug else 1
keras_model = keras.models.load_model(
'./builds/leonhard/2018-05-30 15:22:53-type-translation_checkpoint_epoch-77.hdf5',
{'elmo_embeddings': elmo_emb_fn})
loss = keras_model.evaluate_generator(generator_test,
steps=len(test_set) /
self.config.batch_size,
verbose=verbose)
print(loss)
def main(config):
# Initialize tensorflow session
sess = tf.Session()
K.set_session(sess) # Set to keras backend
if config.debug:
print('Importing Elmo module...')
if config.hub.is_set("cache_dir"):
os.environ['TFHUB_CACHE_DIR'] = config.hub.cache_dir
elmo_model = hub.Module("https://tfhub.dev/google/elmo/1", trainable=True)
if config.debug:
print('Imported.')
sess.run(tf.global_variables_initializer())
sess.run(tf.tables_initializer())
graph = tf.get_default_graph()
elmo_emb_fn = ElmoEmbedding(elmo_model)
elmo_model_emb = get_elmo_embedding(elmo_emb_fn)
output_fn = OutputFN(elmo_model_emb, graph)
train_set = SNLIDataloaderPairs('data/snli_1.0/snli_1.0_train.jsonl')
train_set.load_vocab('./data/snli_vocab.dat', config.vocab_size)
train_set.set_preprocess_fn(preprocess_fn)
train_set.set_output_fn(output_fn)
dev_set = SNLIDataloaderPairs('data/snli_1.0/snli_1.0_dev.jsonl')
dev_set.load_vocab('./data/snli_vocab.dat', config.vocab_size)
dev_set.set_preprocess_fn(preprocess_fn)
dev_set.set_output_fn(output_fn)
# test_set = SNLIDataloader('data/snli_1.0/snli_1.0_test.jsonl')
generator_training = train_set.get_batch(config.batch_size, config.n_epochs)
generator_dev = dev_set.get_batch(config.batch_size, config.n_epochs)
# Models
g_model = generator_model(config) # Generator model
d_model = discriminator_model() # Discriminator model
c_model = combined_model(g_model, d_model) # Combined model
# Training
timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
writer = tf.summary.FileWriter('./logs/' + timestamp + '-type-translation-gan/', graph)
model_path = os.path.abspath(
os.path.join(os.curdir, './builds/' + timestamp))
model_path += '-type-translation-gan-g-model_checkpoint_step-'
last_created_file = None
d_loss, g_loss = None, None
d_acc, g_acc = None, None
min_g_loss = None
# train_G, train_D = 1, 1
for k, ((ref_sent, neutral_sent), real_neg_sentence) in enumerate(generator_training):
# We train the discriminator and generator one time step after the other
if k % 2:
# Generator training
g_loss, g_acc = c_model.train_on_batch([ref_sent, neutral_sent],
np.ones(config.batch_size)) # We want the d_model to fail
else:
# Discriminator training
len_half_batch = config.batch_size//2
ref_beg, ref_end = ref_sent[:len_half_batch], ref_sent[len_half_batch:]
neutral_beg, neutral_end = neutral_sent[:len_half_batch], neutral_sent[len_half_batch:]
neg_end = real_neg_sentence[len_half_batch:]
fake_neg_sentence = g_model.predict([ref_beg, neutral_beg], batch_size=len_half_batch)
d_loss_real, d_acc_real = d_model.train_on_batch([ref_end, neg_end],
np.ones(len_half_batch)) # Real negative endings
d_loss_fakes, d_acc_fakes = d_model.train_on_batch([ref_beg, fake_neg_sentence],
np.zeros(
len_half_batch)) # Generated negative endings
d_loss = 0.5 * np.add(d_loss_real, d_loss_fakes)
d_acc = 0.5 * np.add(d_acc_real, d_acc_fakes)
if k > 0 and not k % config.test_and_save_every:
# Testing and saving to tensorboard.
g_loss_dev, d_loss_dev = [], []
g_acc_dev, d_acc_dev = [], []
g_real_acc = []
for j, ((ref_sent_test, neutral_sent_test), real_neg_sent_test) in enumerate(generator_dev):
if j >= 100:
break
# Discriminator training
fake_neg_sentence_test = g_model.predict([ref_sent_test, neutral_sent_test],
batch_size=config.batch_size)
d_loss_real, d_acc_real = d_model.test_on_batch([ref_sent_test, real_neg_sent_test],
np.ones(config.batch_size)) # Real negative endings
d_loss_fakes, d_acc_fakes = d_model.test_on_batch([ref_sent_test, fake_neg_sentence_test],
np.zeros(
config.batch_size)) # Generated negative endings
d_loss_dev.append(0.5 * np.add(d_loss_real, d_loss_fakes))
d_acc_dev.append(0.5 * np.add(d_acc_real, d_acc_fakes))
# Generator training
g_loss_dev_one, g_acc_dev_one = c_model.test_on_batch([ref_sent, neutral_sent],
np.ones(config.batch_size)
) # We want the d_model to fail
_, g_acc_real_one = g_model.evaluate([ref_sent_test, neutral_sent_test], real_neg_sent_test, verbose=0)
g_real_acc.append(g_acc_real_one)
g_loss_dev.append(g_loss_dev_one)
g_acc_dev.append(g_acc_dev_one)
g_mean_loss, d_mean_loss = np.mean(g_loss_dev), np.mean(d_loss_dev)
g_mean_acc, d_mean_acc = np.mean(g_acc_dev), np.mean(d_acc_dev)
g_mean_acc_real = np.mean(g_real_acc)
# Save value to tensorboard
accuracy_summary = tf.Summary()
accuracy_summary.value.add(tag='train_loss_discriminator', simple_value=d_loss)
accuracy_summary.value.add(tag='train_loss_generator', simple_value=g_loss)
accuracy_summary.value.add(tag='train_acc_generator', simple_value=d_acc)
accuracy_summary.value.add(tag='train_acc_discriminator', simple_value=d_acc)
accuracy_summary.value.add(tag='test_loss_discriminator', simple_value=d_mean_loss)
accuracy_summary.value.add(tag='test_loss_generator', simple_value=g_mean_loss)
accuracy_summary.value.add(tag='test_acc_generator', simple_value=g_mean_acc)
accuracy_summary.value.add(tag='test_acc_discriminator', simple_value=d_mean_loss)
accuracy_summary.value.add(tag='test_acc_embedding', simple_value=g_mean_acc_real)
writer.add_summary(accuracy_summary, k)
# We save the model is loss is better for generator
# We only want to save the generator model
if min_g_loss is None or g_mean_loss < min_g_loss:
g_model.save(model_path + str(k) + ".hdf5")
if last_created_file is not None:
os.remove(last_created_file) # Only keep the best one
last_created_file = model_path + str(k) + ".hdf5"
def main(config):
# Initialize tensorflow session
sess = tf.Session()
K.set_session(sess) # Set to keras backend
if config.debug:
print('Importing Elmo module...')
if config.hub.is_set("cache_dir"):
os.environ['TFHUB_CACHE_DIR'] = config.hub.cache_dir
elmo_model = hub.Module("https://tfhub.dev/google/elmo/1", trainable=True)
if config.debug:
print('Imported.')
sess.run(tf.global_variables_initializer())
sess.run(tf.tables_initializer())
graph = tf.get_default_graph()
elmo_emb_fn = ElmoEmbedding(elmo_model)
elmo_model_emb = get_elmo_embedding(elmo_emb_fn)
output_fn = OutputFN(elmo_model_emb, graph)
output_fn_test = OutputFNTest(elmo_model_emb, graph)
train_set = Dataloader(config, 'data/test_stories.csv', testing_data=True)
# train_set.set_preprocess_fn(preprocess_fn)
train_set.load_dataset('data/test.bin')
train_set.load_vocab('./data/default.voc', config.vocab_size)
train_set.set_output_fn(output_fn)
dev_set = SNLIDataloaderPairs('data/snli_1.0/snli_1.0_dev.jsonl')
dev_set.load_vocab('./data/snli_vocab.dat', config.vocab_size)
dev_set.set_preprocess_fn(preprocess_fn)
dev_set.set_output_fn(output_fn_test)
# test_set = SNLIDataloader('data/snli_1.0/snli_1.0_test.jsonl')
generator_training = train_set.get_batch(config.batch_size,
config.n_epochs)
generator_dev = dev_set.get_batch(config.batch_size, config.n_epochs)
keras_model = keras.models.load_model(config.type_translation_model,
{'elmo_embeddings': elmo_emb_fn})
verbose = 0 if not config.debug else 1
timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# Callbacks
tensorboard = keras.callbacks.TensorBoard(log_dir='./logs/' + timestamp +
'-type-translation/',
histogram_freq=0,
batch_size=config.batch_size,
write_graph=False,
write_grads=True)
model_path = os.path.abspath(
os.path.join(os.curdir, './builds/' + timestamp))
model_path += '-type-translation_checkpoint_epoch-{epoch:02d}.hdf5'
saver = keras.callbacks.ModelCheckpoint(model_path,
monitor='val_loss',
verbose=verbose,
save_best_only=True)
keras_model.fit_generator(generator_training,
steps_per_epoch=5,
epochs=config.n_epochs,
verbose=verbose,
validation_data=generator_dev,
validation_steps=5,
callbacks=[tensorboard, saver])
def train(self):
# Initialize tensorflow session
sess = tf.Session()
K.set_session(sess) # Set to keras backend
if self.config.debug:
print('Importing Elmo module...')
if self.config.hub.is_set("cache_dir"):
os.environ['TFHUB_CACHE_DIR'] = self.config.hub.cache_dir
elmo_model = hub.Module("https://tfhub.dev/google/elmo/1",
trainable=True)
if self.config.debug:
print('Imported.')
# If we gave the models to the encoder decodes...
self.use_pretrained_models = self.config.alignment.is_set(
'decoder_target_model') and self.config.alignment.is_set(
'decoder_src_model') and self.config.alignment.is_set(
'encoder_target_model') and self.config.alignment.is_set(
'encoder_src_model')
sess.run(tf.global_variables_initializer())
sess.run(tf.tables_initializer())
self.graph = tf.get_default_graph()
elmo_emb_fn = ElmoEmbedding(elmo_model)
elmo_embeddings = keras.layers.Lambda(elmo_emb_fn,
output_shape=(1024, ))
sentence = keras.layers.Input(shape=(1, ), dtype="string")
sentence_emb = elmo_embeddings(sentence)
self.elmo_model = keras.models.Model(inputs=sentence,
outputs=sentence_emb)
train_set = SNLIDataloaderPairs('data/snli_1.0/snli_1.0_train.jsonl')
train_set.set_preprocess_fn(preprocess_fn)
train_set.load_vocab('./data/snli_vocab.dat', self.config.vocab_size)
train_set.set_output_fn(self.output_fn)
test_set = Dataloader(self.config,
'data/test_stories.csv',
testing_data=True)
test_set.load_dataset('data/test.bin')
test_set.load_vocab('./data/default.voc', self.config.vocab_size)
test_set.set_output_fn(self.output_fn_test)
generator_training = train_set.get_batch(self.config.batch_size,
self.config.n_epochs)
generator_dev = test_set.get_batch(self.config.batch_size,
self.config.n_epochs)
self.define_models()
model = self.build_graph()
frozen_model = self.build_frozen_graph()
timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
writer = tf.summary.FileWriter('./logs/' + timestamp + '-alignment/',
self.graph)
model_path = os.path.abspath(
os.path.join(os.curdir, './builds/' + timestamp))
model_path += '-alignment-model_checkpoint_step-'
last_created_file = None
self.use_frozen = False
min_source_loss = None
for k, (inputs, labels) in enumerate(generator_training):
# We train the frozen model and the unfrozen model jointly
if self.use_frozen:
# Generator training
metrics = frozen_model.train_on_batch(inputs, labels)
if not k % self.config.print_train_every:
print_on_tensorboard(writer, frozen_model.metrics_names,
metrics, k, 'train_f')
else:
metrics = model.train_on_batch(inputs, labels)
if not k % self.config.print_train_every:
print_on_tensorboard(writer, model.metrics_names, metrics,
k, 'train_uf')
self.use_frozen = not self.use_frozen
if k > 0 and not k % self.config.test_and_save_every:
test_metrics = []
for j, (inputs_val, labels_val) in enumerate(generator_dev):
test_metrics.append(
frozen_model.test_on_batch(inputs_val, labels_val))
test_metrics = np.mean(test_metrics, axis=0)
# Save value to tensorboard
print_on_tensorboard(writer, frozen_model.metrics_names,
test_metrics, k, 'test')
test_metrics_dict = get_dict_from_lists(
frozen_model.metrics_names, test_metrics)
# We save the model is loss is better for generator
# We only want to save the generator model
if min_source_loss is None or test_metrics_dict[
'disrc_src_loss'] < min_source_loss:
frozen_model.save(model_path + str(k) + ".hdf5")
if last_created_file is not None:
os.remove(last_created_file) # Only keep the best one
last_created_file = model_path + str(k) + ".hdf5"