def main(args):
try:
opts, args = getopt.getopt(args, "c:s", ["config="])
except getopt.GetoptError:
print('usage: -c config.json')
sys.exit(2)
start_from_model = False
for opt, arg in opts:
if opt in ("-c", "--config"):
config_fname = os.path.join('configurations', arg)
elif opt == '-s':
start_from_model = True
if start_from_model:
filemode = 'a'
else:
filemode = 'w'
log_path = 'logging/vae_nlg_{}'.format(int(round(time.time() * 1000)))
os.mkdir(log_path)
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO,
filename='{}/evolution.log'.format(log_path),
filemode=filemode)
with open(config_fname, 'r') as json_data:
config_data = json.load(json_data)
batch_size = config_data['batch_size']
epochs = config_data['nb_epochs']
discriminator_iterations = config_data['discriminator_iterations']
tweets_path = config_data['tweets_path']
vocab_path = config_data['vocab_path']
vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb'))
#== == == == == == =
# Load all the Data
#== == == == == == =
delimiter = ''
noutputs = 11
logging.info('Load Training Data')
train_input, train_output, train_weights, train_lex = load_text_gen_data(
join(tweets_path, 'trainset.csv'),
config_data,
vocab,
noutputs,
word_based=False)
logging.info('Load Validation Data')
valid_input, valid_output, _, valid_lex = load_text_gen_data(
join(tweets_path, 'devset.csv'),
config_data,
vocab,
noutputs,
word_based=False)
logging.info('Load Output Validation Data')
valid_dev_input, valid_dev_output, _, valid_dev_lex = load_text_gen_data(
join(tweets_path, 'devset_reduced.csv'),
config_data,
vocab,
noutputs,
random_output=False,
word_based=False)
step = K.variable(1., name='step_varialbe')
steps_per_epoch = ceil(train_output[0].shape[0] /
config_data['batch_size'])
# == == == == == == == == == == =
# Define and load the CNN model
# == == == == == == == == == == =
vae_model_train, vae_model_test, vae_vanilla_train_model, vae_vanilla_test_model, discriminator_model, decoder_test, discriminator = get_vae_gan_model(
config_data, vocab, step)
with open(os.path.join(log_path, 'models.txt'), 'wt') as fh:
fh.write('VAE Model Train\n')
fh.write('---------\n')
vae_model_train.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('VAE Model Test\n')
fh.write('--------------\n')
vae_model_test.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('VAE Model Pretrain\n')
fh.write('---------------------------\n')
vae_vanilla_train_model.summary(
print_fn=lambda x: fh.write(x + '\n'))
fh.write('VAE Model Pretrain Test\n')
fh.write('---------------------------\n')
vae_vanilla_test_model.summary(
print_fn=lambda x: fh.write(x + '\n'))
fh.write('Decoder Test\n')
fh.write('-------------------\n')
decoder_test.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('Discriminator Models\n')
fh.write('-------------------\n')
discriminator_model.summary(print_fn=lambda x: fh.write(x + '\n'))
terminate_on_nan = TerminateOnNaN()
output_callback = LexOutputCallback(
vae_vanilla_test_model,
valid_dev_input,
valid_dev_lex,
1,
vocab,
delimiter,
fname='{}/test_output'.format(log_path))
#output_callback_full = LexOutputCallback(vae_vanilla_test_model, valid_dev_input, valid_dev_lex, 1, vocab, delimiter, fname='{}/test_output'.format(log_path))
#
# vae_vanilla_train_model.fit_generator(
# generator=generate_data_stream(config_data['pretrain_path'], config_data, vocab, config_data['batch_size'], noutputs=3),
# steps_per_epoch=steps_per_epoch,
# epochs=ceil(config_data['pretrain_samples']/config_data['pretrain_samples_per_epoch']),
# callbacks=[output_callback, terminate_on_nan],
# validation_data=(valid_input, valid_output[:3]),
# )
vae_vanilla_train_model.fit(
x=train_input,
y=train_output[:2],
epochs=config_data['pretrain_epochs'],
batch_size=batch_size,
validation_data=(valid_input, valid_output[:2]),
sample_weight=train_weights[:2],
callbacks=[output_callback, terminate_on_nan])
terminate_on_nan = TerminateOnNaN()
model_checkpoint = ModelCheckpoint(
'models/vae_model/weights.{epoch:02d}.hdf5',
period=10,
save_weights_only=True)
out_labels = [
'enc_' + s for s in vae_model_train._get_deduped_metrics_names()
]
out_labels += [
'dis_' + s
for s in discriminator_model._get_deduped_metrics_names()
]
callback_metrics = out_labels + ['val_' + n for n in out_labels]
tensorboard = TensorBoard(log_dir='logging/tensorboard',
histogram_freq=0,
write_grads=True,
write_images=True)
step_callback = StepCallback(step, steps_per_epoch)
output_callback = LexOutputCallback(
vae_vanilla_test_model,
valid_dev_input,
valid_dev_lex,
1,
vocab,
delimiter,
fname='{}/test_output'.format(log_path))
output_callback_full = LexOutputCallback(
vae_vanilla_test_model,
valid_input,
valid_lex,
5,
vocab,
delimiter,
fname='{}/test_valid_output'.format(log_path))
callbacks = CallbackList([
BaseLogger(),
ProgbarLogger(count_mode='steps'), step_callback, tensorboard,
output_callback, output_callback_full, model_checkpoint,
terminate_on_nan
])
callbacks.set_model(vae_model_train)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': True,
'do_validation': True,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
initial_epoch = 0
num_train_samples = train_input[0].shape[0]
index_array = np.arange(num_train_samples)
steps = 0
epoch = initial_epoch
while epoch < epochs:
epoch_logs = {}
callbacks.on_epoch_begin(epoch)
index_array = _batch_shuffle(index_array, batch_size)
steps_done = 0
batches = _make_batches(num_train_samples, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_logs = {}
batch_ids = index_array[batch_start:batch_end]
X = _slice_arrays(train_input, batch_ids)
y = _slice_arrays(train_output, batch_ids)
sample_weights = _slice_arrays(train_weights, batch_ids)
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
set_trainability(discriminator, trainable=False)
enc_outs = vae_model_train.train_on_batch(
x=X, y=y, sample_weight=sample_weights)
set_trainability(discriminator, trainable=True)
list_disc_loss_real = []
if steps < 25 or steps % 500 == 0:
disc_iterations = 25
else:
disc_iterations = discriminator_iterations
for disc_it in range(disc_iterations):
real_idx = np.random.choice(train_input[0].shape[0],
len(batch_ids),
replace=False)
disX_train = train_input[-1][
real_idx] #take input 8 as train input and the rest as targets
disy_train = [x[real_idx] for x in train_input[:8]
] #take input 1-7 as targets
#train on real data
dis_outs_real = discriminator_model.train_on_batch(
disX_train, disy_train)
list_disc_loss_real.append(dis_outs_real)
loss_d_real = np.mean(list_disc_loss_real, axis=0)
outs = np.concatenate((enc_outs, loss_d_real))
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
epoch_logs = {}
batch_index += 1
steps_done += 1
steps += 1
# Epoch finished.
if steps_done >= steps_per_epoch:
valid_len = valid_output[0].shape[0]
enc_val_outs = vae_model_train.evaluate(valid_input,
valid_output,
verbose=False)
dis_val_outs = discriminator_model.evaluate(
valid_input[-1], valid_input[:8], verbose=False)
val_outs = enc_val_outs + dis_val_outs
#val_outs = full_model.evaluate(valid_input, valid_output, verbose=False)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
callbacks.on_train_end()
def main(args):
try:
opts, args = getopt.getopt(args, "c:s", ["config="])
except getopt.GetoptError:
print('usage: -c config.json')
sys.exit(2)
start_from_model = False
for opt, arg in opts:
if opt in ("-c", "--config"):
config_fname = os.path.join('configurations', arg)
elif opt == '-s':
start_from_model = True
if start_from_model:
filemode = 'a'
else:
filemode = 'w'
log_path = 'logging/vae_nlg_{}'.format(int(round(time.time() * 1000)))
os.mkdir(log_path)
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO,
filename='{}/evolution.log'.format(log_path),
filemode=filemode)
with open(config_fname, 'r') as json_data:
config_data = json.load(json_data)
batch_size = config_data['batch_size']
epochs = config_data['nb_epochs']
discriminator_iterations = config_data['discriminator_iterations']
tweets_path = config_data['tweets_path']
vocab_path = config_data['vocab_path']
vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb'))
model_path = config_data['output_path']
if not os.path.exists(model_path):
os.makedirs(model_path)
#== == == == == == =
# Load all the Data
#== == == == == == =
delimiter = ''
noutputs = 9 #+ get_number_outputs(config_data)
logging.info('Load Training Data')
train_input, train_output, train_weights, train_lex = load_text_gen_data(
join(tweets_path, 'trainset.csv'),
join(tweets_path, 'train_lex_features.csv'),
join(tweets_path, 'train_lex_features_tree.json'),
config_data,
vocab,
noutputs,
word_based=False)
logging.info('Load Validation Data')
valid_input, valid_output, _, valid_lex = load_text_gen_data(
join(tweets_path, 'devset.csv'),
join(tweets_path, 'devset_lex_features.csv'),
join(tweets_path, 'dev_lex_features_tree.json'),
config_data,
vocab,
noutputs,
word_based=False)
logging.info('Load Output Validation Data')
valid_dev_input, valid_dev_output, _, valid_dev_lex = load_text_gen_data(
join(tweets_path, 'devset_reduced.csv'),
join(tweets_path, 'devset_lex_features.csv'),
join(tweets_path, 'dev_lex_features_tree.json'),
config_data,
vocab,
noutputs,
random_output=True,
word_based=False,
random_first_word=True,
)
valid_dev_input3, valid_dev_output3, _, valid_dev_lex3 = load_text_gen_data(
join(tweets_path, 'test_e2e.csv'),
join(tweets_path, 'devset_lex_features.csv'),
join(tweets_path, 'dev_lex_features_tree.json'),
config_data,
vocab,
noutputs,
random_output=True,
word_based=False,
random_first_word=True)
step = K.variable(1., name='step_varialbe')
steps_per_epoch = ceil(train_output[0].shape[0] /
config_data['batch_size'])
# == == == == == == == == == == =
# Define and load the CNN model
# == == == == == == == == == == =
train_model, test_model, discriminator_models = vae_model(
config_data, vocab, step)
with open(os.path.join(log_path, 'models.txt'), 'wt') as fh:
fh.write('VAE Model Train\n')
fh.write('---------\n')
train_model.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('VAE Model Test\n')
fh.write('--------------\n')
test_model.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('Discriminator Models\n')
for discriminator in discriminator_models:
fh.write('{}\n'.format(discriminator.name))
fh.write('---------------------------\n')
discriminator.summary(print_fn=lambda x: fh.write(x + '\n'))
optimizer = Adadelta(lr=1,
epsilon=1e-8,
rho=0.95,
decay=0.0001,
clipnorm=10)
train_model.compile(
optimizer=optimizer,
loss=lambda y_true, y_pred: y_pred,
)
disX_train = train_input[
-1] # take input 8 as train input and the rest as targets
disy_train = train_input[:-1] # take input 1-7 as targets
if config_data.get('pretrain_dirscr', 1) == 1:
for i, discriminator in enumerate(discriminator_models):
discriminator.compile(
optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy']
) #output of the discriminator model are the outputs -> specifiy cross_entropy as loss
if discriminator.name == 'nsent':
last_ones_idx = last_one(disy_train[i])
y_train = np.zeros_like(disy_train[i])
for j in range(y_train.shape[0]):
lidx = last_ones_idx[j]
y_train[j][lidx] = 1.0
last_ones_idx = last_one(valid_input[i])
y_dev = np.zeros_like(valid_input[i])
for j in range(y_dev.shape[0]):
lidx = last_ones_idx[j]
y_dev[j][lidx] = 1.0
else:
y_train = disy_train[i]
y_dev = valid_input[i]
# == == == == == == == == =
# Pretrain Discriminators
# == == == == == == == == =
early_stopping = EarlyStopping(monitor='val_loss',
patience=25,
mode='min',
verbose=1,
min_delta=1e-6)
model_checkpoint = ModelCheckpoint(join(
model_path,
'discr_weights_{}.hdf5'.format(discriminator.name)),
save_weights_only=True,
save_best_only=True,
monitor='val_loss',
mode='min',
verbose=1)
logging.info('Pretrain the {} Discriminator'.format(
discriminator.name))
history = discriminator.fit(
x=disX_train,
y=y_train,
validation_data=(valid_input[-1], y_dev),
epochs=1,
batch_size=1024,
callbacks=[early_stopping, model_checkpoint])
losses = history.history['loss']
val_losses = history.history['val_loss']
val_accs = history.history['val_acc']
for i, (loss, val_loss,
val_acc) in enumerate(zip(losses, val_losses,
val_accs)):
logging.info(
'Epoch: {} Loss: {} Val Loss: {} Val Acc: {}'.format(
i, loss, val_loss, val_acc))
for i, discriminator in enumerate(discriminator_models):
logging.info('Loading the {} Discriminator'.format(
discriminator.name))
model_weights = join(
model_path, 'discr_weights_{}.hdf5'.format(discriminator.name))
discriminator.load_weights(model_weights)
terminate_on_nan = TerminateOnNaN()
model_checkpoint = ModelCheckpoint(join(model_path,
'weights.{epoch:02d}.hdf5'),
period=15,
save_weights_only=True)
tensorboard = TensorBoard(log_dir='logging/tensorboard',
histogram_freq=0,
write_grads=True,
write_images=True)
step_callback = StepCallback(step, steps_per_epoch)
lex_output = LexOutputCallbackGAN(
test_model,
valid_dev_input,
valid_dev_lex,
1,
vocab,
delimiter,
fname='{}/test_output'.format(log_path))
#lex_output2 = LexOutputCallbackGAN(test_model, valid_dev_input2, valid_dev_lex2, 1, vocab, delimiter, fname='{}/test_output_random'.format(log_path))
lex_output3 = LexOutputCallbackGAN(
test_model,
valid_dev_input3,
valid_dev_lex3,
1,
vocab,
delimiter,
fname='{}/final_test_output_random'.format(log_path))
callbacks = [
step_callback, tensorboard, lex_output, lex_output3,
model_checkpoint, terminate_on_nan
]
for i, discriminator in enumerate(discriminator_models):
set_trainability(discriminator, trainable=False)
logging.info('Train the Model')
train_model.fit(x=train_input,
y=train_output,
epochs=epochs,
batch_size=batch_size,
validation_data=(valid_input, valid_output),
sample_weight=train_weights,
callbacks=callbacks)
def main(args):
try:
opts, args = getopt.getopt(args, "c:s", ["config="])
except getopt.GetoptError:
print('usage: -c config.json')
sys.exit(2)
start_from_model = False
for opt, arg in opts:
if opt in ("-c", "--config"):
config_fname = os.path.join('configurations', arg)
elif opt == '-s':
start_from_model = True
if start_from_model:
filemode = 'a'
else:
filemode = 'w'
log_path = 'logging/vae_nlg_{}'.format(int(round(time.time() * 1000)))
os.mkdir(log_path)
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO,
filename='{}/evolution.log'.format(log_path),
filemode=filemode)
with open(config_fname, 'r') as json_data:
config_data = json.load(json_data)
tweets_path = config_data['tweets_path']
vocab_path = config_data['vocab_path']
vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb'))
#== == == == == == =
# Load all the Data
#== == == == == == =
delimiter = ''
noutputs = 2
logging.info('Load Training Data')
train_input, train_output = load_text_gen_data(
join(tweets_path, 'trainset.csv'), config_data, vocab, noutputs)
logging.info('Load Validation Data')
valid_input, valid_output = load_text_gen_data(
join(tweets_path, 'devset.csv'), config_data, vocab, noutputs)
step = K.variable(1.)
# == == == == == == == == == == =
# Define and load the CNN model
# == == == == == == == == == == =
cnn_model, test_model = vae_model(config_data, vocab, step)
cnn_model.save_weights(config_data['base_model_path'])
cnn_model.summary()
model_path = 'models/vae_model/'
steps_per_epoch = ceil(train_output[0].shape[0] /
config_data['batch_size'])
terminate_on_nan = TerminateOnNaN()
model_checkpoint = ModelCheckpoint(
'models/vae_model/weights.{epoch:02d}.hdf5',
period=10,
save_weights_only=True)
reduce_callback = ReduceLROnPlateau(monitor='val_loss',
factor=0.995,
patience=100,
min_lr=0.001,
cooldown=50)
optimizer = Nadam(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-8,
schedule_decay=0.001,
clipnorm=10)
#optimizer = Adadelta(lr=1.0, epsilon=1e-8, rho=0.95, decay=0.0001, clipnorm=10)
cnn_model.compile(optimizer=optimizer,
loss=lambda y_true, y_pred: y_pred)
cnn_model.fit(
x=train_input,
y=train_output,
epochs=1000,
batch_size=config_data['batch_size'],
validation_data=(valid_input, valid_output),
callbacks=[
StepCallback(step, steps_per_epoch),
OutputCallback(test_model,
valid_input[0],
15,
vocab,
delimiter,
fname='{}/test_output'.format(log_path)),
terminate_on_nan, model_checkpoint, reduce_callback
],
)
test_model.summary()
cnn_out_path = join(
config_data['output_path'],
'trained_deconv_vae_{}_model'.format(config_data['model_type']))
cnn_model.save_weights(cnn_out_path)
cnn_out_path = join(
config_data['output_path'],
'trained_deconv_vae_{}_model_test'.format(
config_data['model_type']))
test_model.save_weights(cnn_out_path)
output_text(test_model, valid_input, vocab)
def main(args):
try:
opts, args = getopt.getopt(args, "c:s", ["config="])
except getopt.GetoptError:
print('usage: -c config.json')
sys.exit(2)
start_from_model = False
for opt, arg in opts:
if opt in ("-c", "--config"):
config_fname = os.path.join('configurations', arg)
elif opt == '-s':
start_from_model = True
if start_from_model:
filemode = 'a'
else:
filemode = 'w'
curr_time = time.time()
log_path = 'logging/vae_nlg_{}'.format(int(round(curr_time * 1000)))
os.mkdir(log_path)
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO,
filename='{}/evolution.log'.format(log_path),
filemode=filemode)
with open(config_fname, 'r') as json_data:
config_data = json.load(json_data)
tweets_path = config_data['tweets_path']
vocab_path = config_data['vocab_path']
vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb'))
#== == == == == == =
# Load all the Data
#== == == == == == =
if config_data['model_type'] == 'recurrent':
from vae_architectures.vae_deconv_recurrent import vae_model
from data_loaders.data_loader_charlevel import load_data, generate_data_stream
delimiter = ''
noutputs = 3
elif config_data['model_type'] == 'baseline_word':
from vae_architectures.vae_deconv_baseline_word import vae_model
from data_loaders.data_loader_wordlevel import load_data, generate_data_stream
vocab = {k: v[0] for k, v in vocab.items()}
noutputs = 2
delimiter = ' '
elif config_data['model_type'] == 'recurrent_word':
from vae_architectures.vae_deconv_recurrent_word import vae_model
from data_loaders.data_loader_wordlevel import load_data, generate_data_stream
vocab = {k: v[0] for k, v in vocab.items()}
noutputs = 3
delimiter = ' '
else:
from vae_architectures.vae_deconv_baseline import vae_model
from data_loaders.data_loader_charlevel import load_data, generate_data_stream
noutputs = 1
delimiter = ''
logging.info('Load Training Data')
train_input, train_output = load_data(
join(tweets_path, 'en_train.tsv'), config_data, vocab, noutputs)
logging.info('Load Validation Data')
valid_input, valid_output = load_data(
join(tweets_path, 'en_valid15.tsv'), config_data, vocab, noutputs)
step = K.variable(1.)
model_path = 'models/vae_model_{}'.format(curr_time)
os.mkdir(model_path)
# == == == == == == == == == == =
# Define and load the CNN model
# == == == == == == == == == == =
cnn_model, test_model = vae_model(config_data, vocab, step)
cnn_model.save_weights(os.path.join(model_path, 'base_cnn_model.h5'))
cnn_model.summary()
steps_per_epoch = ceil(config_data['samples_per_epoch'] /
config_data['batch_size'])
initial_epoch = 0
terminate_on_nan = TerminateOnNaN()
model_checkpoint = ModelCheckpoint(os.path.join(
model_path, 'weights.{epoch:02d}.hdf5'),
period=10,
save_weights_only=True)
reduce_callback = ReduceLROnPlateau(monitor='val_loss',
factor=0.995,
patience=10,
min_lr=0.001,
cooldown=50)
#optimizer = Nadam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-8, schedule_decay=0.001, clipnorm=10)
optimizer = Adadelta(lr=1.0,
epsilon=1e-8,
rho=0.95,
decay=0.0001,
clipnorm=10)
cnn_model.compile(optimizer=optimizer,
loss=lambda y_true, y_pred: y_pred)
cnn_model.fit_generator(
generator=generate_data_stream(config_data['training_path'],
config_data,
vocab,
config_data['batch_size'],
noutputs=noutputs),
steps_per_epoch=steps_per_epoch,
epochs=ceil(
config_data['nb_epochs'] *
(config_data['nsamples'] / config_data['samples_per_epoch'])),
callbacks=[
StepCallback(step, steps_per_epoch),
OutputCallback(test_model,
valid_input[0],
5,
vocab,
delimiter,
fname='{}/test_output'.format(log_path)),
terminate_on_nan, model_checkpoint, reduce_callback
],
validation_data=(valid_input, valid_output),
initial_epoch=initial_epoch)
test_model.summary()
cnn_out_path = join(
config_data['output_path'],
'trained_deconv_vae_{}_model'.format(config_data['model_type']))
cnn_model.save_weights(cnn_out_path)
cnn_out_path = join(
config_data['output_path'],
'trained_deconv_vae_{}_model_test'.format(
config_data['model_type']))
test_model.save_weights(cnn_out_path)
output_text(test_model, valid_input, vocab)
def main(args):
try:
opts, args = getopt.getopt(args, "c:s", ["config="])
except getopt.GetoptError:
print('usage: -c config.json')
sys.exit(2)
start_from_model = False
for opt, arg in opts:
if opt in ("-c", "--config"):
config_fname = os.path.join('configurations', arg)
elif opt == '-s':
start_from_model = True
if start_from_model:
filemode = 'a'
else:
filemode = 'w'
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO,
filename='logging/vae_gan/evolution.log',
filemode=filemode)
with open(config_fname, 'r') as json_data:
config_data = json.load(json_data)
tweets_path = config_data['tweets_path']
vocab_word_path = config_data['vocab_word_path']
vocab_char_path = config_data['vocab_char_path']
vocab_word = cPickle.load(
open(join(vocab_word_path, 'vocabulary.pkl'), 'rb'))
vocab_char = cPickle.load(
open(join(vocab_char_path, 'vocabulary.pkl'), 'rb'))
#== == == == == == =
# Load all the Data
#== == == == == == =
noutputs = 5
logging.info('Load Validation Data')
valid_input, valid_output = load_data(
join(tweets_path, 'en_valid15.tsv'), config_data, vocab_word,
vocab_char, noutputs)
step = K.variable(1.)
# == == == == == == == == == == =
# Define and load the CNN model
# == == == == == == == == == == =
vae_train_model, vae_gan_model, discriminator_model, discriminators, vae_test_model = get_vae_gan_model(
config_data, vocab_char, step)
#full_model.summary()
vae_train_model.summary()
vae_gan_model.summary()
discriminator_model.summary()
model_path = 'models/vae_model/'
steps_per_epoch = int(
ceil(config_data['samples_per_epoch'] / config_data['batch_size']))
epochs = int(
ceil(config_data['nb_epochs'] *
(config_data['nsamples'] / config_data['samples_per_epoch'])))
batch_size = config_data['batch_size']
initial_epoch = 0
skip_texts = 0
save_models = [
(vae_train_model, 'models/vae_model/vae_train_weights.hdf5'),
(vae_gan_model, 'models/vae_model/vae_gan_weights.hdf5'),
(discriminator_model,
'models/vae_model/discriminator_weights.hdf5'),
(vae_test_model, 'models/vae_modelvae_test_weights.hdf5'),
]
terminate_on_nan = TerminateOnNaN()
model_checkpoint = MultiModelCheckpoint(models=save_models, period=15)
generator = generate_data_stream(config_data['training_path'],
config_data,
vocab_char,
vocab_word,
config_data['batch_size'],
skip_data=skip_texts,
noutputs=noutputs)
enc_out_labels = [
'vae_train_' + s
for s in vae_train_model._get_deduped_metrics_names()
]
dec_out_labels = [
'vae_gan_' + s for s in vae_gan_model._get_deduped_metrics_names()
]
dis_out_labels = [
'dis_' + s
for s in discriminator_model._get_deduped_metrics_names()
]
out_labels = enc_out_labels + dec_out_labels + dis_out_labels
callback_metrics = out_labels + ['val_' + n for n in out_labels]
step_callback = StepCallback(step, steps_per_epoch)
output_callback = OutputCallback(vae_test_model, valid_input, 15,
vocab_char, '')
callbacks = CallbackList([
BaseLogger(),
ProgbarLogger(count_mode='steps'), step_callback, output_callback,
model_checkpoint
])
callbacks.set_model(vae_gan_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': True,
'do_validation': True,
'metrics': callback_metrics,
})
enqueuer = GeneratorEnqueuer(generator,
use_multiprocessing=False,
wait_time=0.01)
try:
enqueuer.start(workers=1, max_queue_size=10)
output_generator = enqueuer.get()
callbacks.on_train_begin()
epoch = initial_epoch
while epoch < epochs:
epoch_logs = {}
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
while steps_done < steps_per_epoch:
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
X, y = next(output_generator)
callbacks.on_batch_begin(batch_index, batch_logs)
set_trainability(vae_train_model, trainable=True)
for discriminator in discriminators:
set_trainability(discriminator, trainable=False)
vae_train_outs = vae_train_model.train_on_batch(X, y)
vae_gan_outs = vae_gan_model.train_on_batch(X, y[:3])
set_trainability(vae_train_model, trainable=False)
for discriminator in discriminators:
set_trainability(discriminator, trainable=True)
dis_outs = discriminator_model.train_on_batch(X, y[:3])
outs = vae_train_outs + vae_gan_outs + dis_outs
# outs = full_model.train_on_batch(X, y)
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
epoch_logs = {}
batch_index += 1
steps_done += 1
# Epoch finished.
if steps_done >= steps_per_epoch:
vae_train_val_outs = vae_train_model.evaluate(
valid_input, valid_output, verbose=False)
vae_gan_val_outs = vae_gan_model.evaluate(
valid_input, valid_output[:3], verbose=False)
dis_val_outs = discriminator_model.evaluate(
valid_input, valid_output[:3], verbose=False)
val_outs = vae_train_val_outs + vae_gan_val_outs + dis_val_outs
# val_outs = full_model.evaluate(valid_input, valid_output, verbose=False)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
finally:
if enqueuer is not None:
enqueuer.stop()
callbacks.on_train_end()
def main(args):
try:
opts, args = getopt.getopt(args, "c:s", ["config="])
except getopt.GetoptError:
print('usage: -c config.json')
sys.exit(2)
start_from_model = False
for opt, arg in opts:
if opt in ("-c", "--config"):
config_fname = os.path.join('configurations', arg)
elif opt == '-s':
start_from_model = True
if start_from_model:
filemode = 'a'
else:
filemode = 'w'
log_path = 'logging/vae_nlg_{}'.format(int(round(time.time() * 1000)))
os.mkdir(log_path)
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO,
filename='{}/evolution.log'.format(log_path),
filemode=filemode)
with open(config_fname, 'r') as json_data:
config_data = json.load(json_data)
batch_size = config_data['batch_size']
epochs = config_data['nb_epochs']
discriminator_iterations = config_data['discriminator_iterations']
tweets_path = config_data['tweets_path']
vocab_path = config_data['vocab_path']
vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb'))
#== == == == == == =
# Load all the Data
#== == == == == == =
noutputs = 5
logging.info('Load Training Data')
train_input, train_output = load_text_pairs(
join(tweets_path, 'training_set.tsv'), config_data, vocab,
noutputs)
logging.info('Load Validation Data')
valid_input, valid_output = load_text_pairs(
join(tweets_path, 'vaild_set.tsv'), config_data, vocab, noutputs)
logging.info('Load Output Validation Data')
valid_dev_input, valid_dev_output = load_text_pairs(
join(tweets_path, 'test_set.tsv'), config_data, vocab, noutputs)
#train_input = [x[:1213] for x in train_input]
#train_output = [x[:1213] for x in train_output]
noise_valid_input = np.zeros(shape=(valid_input[0].shape[0],
config_data['z_size']))
step = K.variable(1.)
steps_per_epoch = ceil(train_output[0].shape[0] /
config_data['batch_size'])
# == == == == == == == == == == =
# Define and load the CNN model
# == == == == == == == == == == =
vae_model, vae_model_test, decoder_discr_model, decoder_test_model, discriminator_model, discriminator = get_vae_gan_model(
config_data, vocab, step)
with open(os.path.join(log_path, 'models.txt'), 'wt') as fh:
fh.write('VAE Model\n')
fh.write('---------\n')
vae_model.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('VAE Model Test\n')
fh.write('--------------\n')
vae_model_test.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('Decoder Discriminator Model\n')
fh.write('---------------------------\n')
decoder_discr_model.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('Decoder Test Model\n')
fh.write('---------------------------\n')
decoder_test_model.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('Discriminator Model\n')
fh.write('-------------------\n')
discriminator_model.summary(print_fn=lambda x: fh.write(x + '\n'))
terminate_on_nan = TerminateOnNaN()
model_checkpoint = ModelCheckpoint(
'models/vae_model/weights.{epoch:02d}.hdf5',
period=10,
save_weights_only=True)
enc_out_labels = [
'enc_' + s for s in vae_model._get_deduped_metrics_names()
]
dec_out_labels = [
'dec_' + s
for s in decoder_discr_model._get_deduped_metrics_names()
]
dis_out_labels = [
'dis_' + s
for s in discriminator_model._get_deduped_metrics_names()
]
out_labels = enc_out_labels + dec_out_labels + [
'dis_real', 'dis_gen', 'dis_noise'
]
#out_labels = full_model._get_deduped_metrics_names()
callback_metrics = out_labels + ['val_' + n for n in out_labels]
step_callback = StepCallback(step, steps_per_epoch)
output_callback = GANOutputCallback(
vae_model_test,
valid_dev_input[0],
1,
vocab,
'',
fname='{}/test_output'.format(log_path))
callbacks = CallbackList([
BaseLogger(),
ProgbarLogger(count_mode='steps'), step_callback, output_callback,
model_checkpoint, terminate_on_nan
])
callbacks.set_model(vae_model_test)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': True,
'do_validation': True,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
initial_epoch = 0
num_train_samples = train_input[0].shape[0]
index_array = np.arange(num_train_samples)
steps = 0
epoch = initial_epoch
while epoch < epochs:
epoch_logs = {}
callbacks.on_epoch_begin(epoch)
index_array = _batch_shuffle(index_array, batch_size)
steps_done = 0
batches = _make_batches(num_train_samples, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_logs = {}
batch_ids = index_array[batch_start:batch_end]
X = _slice_arrays(train_input, batch_ids)
y = _slice_arrays(train_output, batch_ids)
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
set_trainability(discriminator, trainable=False)
enc_outs = vae_model.train_on_batch(x=X, y=y[:3])
set_trainability(discriminator, trainable=True)
list_disc_loss_real = []
list_disc_loss_gen = []
list_disc_loss_noise = []
if steps < 25 or steps % 500 == 0:
disc_iterations = 100
else:
disc_iterations = discriminator_iterations
noise_input = np.zeros(shape=(len(batch_ids),
config_data['z_size']))
for disc_it in range(disc_iterations):
#clip_weights(discriminator)
real_idx = np.random.choice(train_input[0].shape[0],
len(batch_ids),
replace=False)
train_real_batch = [x[real_idx] for x in train_input]
#train on real data
x_fake = vae_model_test.predict_on_batch(
x=train_real_batch[0])
x_noise_fake = decoder_test_model.predict_on_batch(
x=noise_input)
train_input_discr = np.concatenate(
(train_real_batch[0], train_real_batch[0],
train_real_batch[0]))
train_output_discr = np.concatenate(
(train_real_batch[1], x_fake, x_noise_fake))
labels = np.asarray(
len(batch_ids) * [1] + 2 * len(batch_ids) * [-1])
index_array_discr = np.arange(len(labels))
np.random.shuffle(index_array_discr)
discr_batch = [
train_input_discr[index_array_discr],
train_output_discr[index_array_discr]
]
discr_batch_labels = labels[index_array_discr]
dis_outs_real = discriminator_model.train_on_batch(
discr_batch, discr_batch_labels)
#dis_outs_real = discriminator_model.train_on_batch(train_real_batch, -np.ones(shape=(len(batch_ids), 1)))
#dis_outs_gen = discriminator_model.train_on_batch([train_real_batch[0], x_fake], np.ones(shape=(len(batch_ids), 1)))
#dis_outs_gen_noise = discriminator_model.train_on_batch([train_real_batch[0], x_noise_fake], np.ones(shape=(len(batch_ids), 1)))
list_disc_loss_real.append(dis_outs_real)
#list_disc_loss_gen.append(dis_outs_gen)
#list_disc_loss_noise.append(dis_outs_gen_noise)
loss_d_real = -np.mean(list_disc_loss_real)
loss_d_gen = np.mean(list_disc_loss_gen)
loss_d_noise = np.mean(list_disc_loss_noise)
set_trainability(discriminator, trainable=False)
decoder_discr_input = [X[0], noise_input]
dec_outs = decoder_discr_model.train_on_batch(
x=decoder_discr_input,
y=-np.ones(shape=(len(batch_ids), 1)))
outs = enc_outs + [dec_outs] + [loss_d_real]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
epoch_logs = {}
batch_index += 1
steps_done += 1
steps += 1
# Epoch finished.
if steps_done >= steps_per_epoch:
valid_len = valid_output[0].shape[0]
enc_val_outs = vae_model.evaluate(valid_input,
valid_output[:3],
verbose=False)
dec_val_outs = decoder_discr_model.evaluate(
[valid_input[0], noise_valid_input],
-np.ones(shape=(valid_len, 1)),
verbose=False)
dis_val_outs = discriminator_model.evaluate(
valid_input,
-np.ones(shape=(valid_len, 1)),
verbose=False)
val_outs = enc_val_outs + [dec_val_outs] + [dis_val_outs]
#val_outs = full_model.evaluate(valid_input, valid_output, verbose=False)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
callbacks.on_train_end()