def on_epoch_end(self, epoch, logs=None):
if 'roc_auc' not in self.params['metrics']:
# self.params['metrics'].append('roc_auc')
self.params['metrics'].append('val_roc_auc')
# X_train, Y_train, _, _ = self.model.training_data
X_test = list(self.model.validation_data[0:-3])
Y_test = self.model.validation_data[-3]
# train_pred = self.model.predict_proba(X_train, verbose=0)
val_pred = self.model.predict_proba(X_test, verbose=0)
# logs['roc_auc'] = roc_auc_score(Y_train[:,1], train_pred[:,1])
logs['val_roc_auc'] = roc_auc_score(Y_test[:, 1], val_pred[:, 1])
ProgbarLogger.on_epoch_end(self, epoch, logs)
def main():
os.makedirs(c.path_to_summaries, exist_ok=True)
os.makedirs(c.path_to_models, exist_ok=True)
model = get_model()
call_backs = [
# EarlyStopping('val_acc', min_delta=1e-5, patience=20),
ProgbarLogger('steps'),
ModelCheckpoint(c.path_to_models + '/model', save_best_only=False),
TensorBoard(c.path_to_summaries)
]
train_gen, test_gen = tools.get_generators(c)
# train_gen, test_gen = tools.fake_generator(c), tools.fake_generator(c)
model.fit_generator(generator=train_gen,
steps_per_epoch=50,
epochs=c.epochs,
verbose=1,
callbacks=call_backs,
validation_data=test_gen,
validation_steps=30,
max_queue_size=c.max_queue_size,
workers=c.workers)
model.save(c.path_to_models + '/model_OE')
def train(self,
documents: Iterable[str],
epochs: int,
directory: Optional[str],
progress_bar: bool = True):
from keras.callbacks import ProgbarLogger, EarlyStopping, ReduceLROnPlateau, ModelCheckpoint
self.save(directory)
vectors, labels = self.tokenizer.encoded_training_set_from_documents(
documents)
callbacks = [
EarlyStopping(monitor="loss"),
ReduceLROnPlateau(monitor="loss")
]
if directory is not None:
callbacks.append(ModelCheckpoint(self.model_path(directory)))
if progress_bar:
callbacks.append(ProgbarLogger())
if self.history.iterations < epochs:
history = self.model.fit(vectors,
labels,
epochs=epochs,
initial_epoch=self.history.iterations,
verbose=int(progress_bar),
callbacks=callbacks)
self.history += TrainingHistory.from_keras_history(history)
self.save(directory)
return self.history
def __init__(self, model_obj):
self.model_obj = model_obj
self.checkpointer = ModelCheckpoint(
filepath='/Users/rishab/Desktop/fynd/solid_app/tmp/weights.hdf5',
verbose=1,
save_best_only=True)
self.progbarLogger = ProgbarLogger(count_mode='steps')
self.early_monitor = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=0,
verbose=0,
mode='auto')
self.remote_monitor = RemoteMonitor(root='http://0.0.0.0:5001',
path='/publish',
field='data',
headers=None)
self.history = LossHistory()
self.board = TensorBoard(log_dir='./logs',
histogram_freq=0,
batch_size=32,
write_graph=True,
write_grads=False,
write_images=False,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None)
def make_train_episode(model, layers, num_img, train_gen, valid_gen):
path_to_weights = os.path.join(config.train.path_to_models,
'model{}'.format(layers))
call_backs = [
EarlyStopping('val_acc', min_delta=1e-5, patience=20),
ProgbarLogger('steps'),
ModelCheckpoint(path_to_weights,
save_best_only=True,
save_weights_only=True),
LearningRateScheduler(lambda x: tools.lr_scheduler(x, config)),
CSVLogger(config.train.path_to_log),
TensorBoard(config.train.path_to_summaries)
]
steps_per_epoch = int(
(1 - config.data.valid_size) * num_img / config.data.batch_size) // 1
validation_steps = int(
config.data.valid_size * num_img / config.data.batch_size) // 1
model.fit_generator(generator=train_gen,
steps_per_epoch=steps_per_epoch,
epochs=config.train.epochs,
verbose=1,
callbacks=call_backs,
validation_data=valid_gen,
validation_steps=validation_steps,
max_queue_size=config.train.max_queue_size,
workers=config.train.workers,
use_multiprocessing=False)
def train_regression_model():
num_batches = 1848
total_iterations = 90000
batch_size = 64
num_epochs = ceil(total_iterations / num_batches)
initial_epoch = 0
regression_model_file = 'model_%02d.hdf5' % initial_epoch
regression_model = None
if os.path.exists(regression_model_file):
print('Loading from saved file.')
regression_model = model.get_regression_model(regression_model_file)
else:
print('Start training from scratch.')
regression_model = model.create_regression_model()
regression_model.summary()
progbar = ProgbarLogger('steps')
checkpoint = ModelCheckpoint('model_{epoch:02d}.hdf5',
verbose=1,
monitor='loss')
terminate = TerminateOnNaN()
callbacks = [checkpoint, progbar, terminate]
regression_model.fit_generator(
generator=dataset.load_dataset(training_dir),
steps_per_epoch=num_batches,
epochs=num_epochs,
callbacks=callbacks,
initial_epoch=initial_epoch,
verbose=1)
def train(steps=28, epochs=1, unet=0):
model = get_model(IMAGE_H, IMAGE_W, INPUT_CHANNELS, SAVED_MODEL, unet)
callbacks = [
ProgbarLogger(count_mode='steps', stateful_metrics=None),
ModelCheckpoint('rgb_weights.h5',
monitor='val_loss',
save_best_only=False,
verbose=0),
util.MemLeakCallback()
]
print("Training network")
history = model.fit_generator( #Do training from generator
generator=variation_gen(batch_size),
steps_per_epoch=
steps, #Number of samples obtained using batch_gen or variation_gen (28 for whole dataset)
validation_data=batch_generator(batch_size),
epochs=epochs, #Number of passes over whole dataset
validation_steps=1,
verbose=1,
shuffle=False,
callbacks=callbacks)
model.save("./rgb.h5")
return
def train():
mc = ModelCheckpoint(filepath="models", verbose=1, save_best_only=True)
tb = TensorBoard("logs")
pb = ProgbarLogger()
train_gen, valid_gen = get_generators()
train_size = 1281167
valid_size = 50000
m = VGG_M()
m.compile(optimizer='adam', loss='categorical_crossentropy', metrics = ['accuracy'])
m.fit_generator(train_gen, 1000, 1000, callbacks = [tb, pb],
validation_data=valid_gen, nb_val_samples=500, nb_worker=16, pickle_safe=True)
def train(train_x, train_y, test_x, test_y, model, epochs):
progbar = ProgbarLogger()
history = model.fit(
x=train_x,
y=train_y,
epochs=epochs,
callbacks=[progbar],
#validation_freq=1,
validation_data=(test_x, test_y))
plot_model_history(history.history)
def train(X, Y):
nnmodel = model(X)
nnmodel.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
progbar = ProgbarLogger()
nnmodel.fit(X, Y, epochs=500, batch_size=64, verbose=0)
scores = nnmodel.evaluate(X, Y)
print(scores)
model_json = nnmodel.to_json()
with open("model.json", "w") as json_file:
json_file.write(model_json)
nnmodel.save_weights("model.h5")
print("Saved model to disk")
def train(train_config):
epoch = train_config.get('epoch', 10)
batch_size = train_config.get('batch', 32)
split = train_config.get('split', 0.2)
model_name = train_config.get('model', 'noname')
train_dir = train_config['train']
test_dir = train_config['test']
model_dir = os.path.join("./models", "{}".format(model_name))
model_file = open(os.path.join(model_dir, "model_log.txt"), 'w')
def print_f(val):
model_file.write("{}\n".format(val))
print(val)
model = CNN_RNN_Sequential_model(print_f)
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy', 'mae'],
)
# load training set
print('Loading data')
X, y, y_raws, label_set = load_dataset(train_dir)
X_train, X_val, y_train, y_val = train_test_split(X,
y,
test_size=split,
random_state=42)
model.fit(X_train,
y_train,
epochs=epoch,
batch_size=batch_size,
verbose=1,
validation_data=[X_val, y_val],
callbacks=[
ProgbarLogger(),
ReduceLROnPlateau(),
TensorBoard(log_dir=model_dir,
histogram_freq=1,
batch_size=100),
CSVLogger(filename=os.path.join(model_dir, "logs.log"))
])
after_train(model, model_file, model_dir, train_config, label_set,
model_name)
model_file.close()
def __init__(
self,
name,
n_input=1161, # n_frames (9) * narrow_band_window (256/2 + 1)
n_output=128, # high_band_window (512/4 + 1)
n_hidden=2048,
n_layers=3,
fe=None):
self.n_input = n_input
self.n_output = n_output
self.n_hidden = n_hidden
self.n_layers = n_layers
self.fe = fe
self.name = name
self.model = Sequential()
self.model.add(Dense(self.n_hidden, input_shape=(self.n_input, )))
self.model.add(Activation('relu'))
for idx in range(self.n_layers):
self.model.add(Dense(n_hidden))
self.model.add(Activation('relu'))
self.model.add(Dense(self.n_output))
self.model.compile(optimizer='adam', loss='mse') #, metrics=[])
self.progbar_logger = ProgbarLogger()
self.psnr_metrics = PsnrMetrics(self)
self.model_checkpoint = ModelCheckpoint(self.name,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min')
self.early_stopping = EarlyStopping(monitor='val_loss',
min_delta=0.0,
patience=8,
mode='min')
def train_model(self,
batch_size=32,
nb_epoch=50,
load_data=False,
old_weight_path=''):
print("start training model...")
if load_data:
train_data, train_labels, valid_data, valid_labels = self.load_data(
)
else:
train_data, train_labels, valid_data, valid_labels = self.prepare_train_data(
)
model = self.baseModel()
if old_weight_path != '':
print("load last epoch model to continue train")
model.load_weights(old_weight_path)
model.fit(train_data,
train_labels,
batch_size=batch_size,
epochs=nb_epoch,
validation_data=(valid_data, valid_labels),
callbacks=[
ModelCheckpoint(
"output/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='min',
period=2),
ProgbarLogger()
])
return model
def _prepare_callbacks(self, callbacks: List[Callback],
val_ins: List[numpy.array], epochs: int,
batch_size: int, num_train_samples: int,
callback_metrics: List[str], do_validation: bool,
verbose: int):
"""
Sets up Keras callbacks to perform various monitoring functions during training.
"""
self.history = History() # pylint: disable=attribute-defined-outside-init
callbacks = [BaseLogger()] + (callbacks or []) + [self.history]
if verbose:
callbacks += [ProgbarLogger()]
callbacks = CallbackList(callbacks)
# it's possible to callback a different model than self
# (used by Sequential models).
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self # pylint: disable=redefined-variable-type
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'samples': num_train_samples,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
callback_model.stop_training = False
for cbk in callbacks:
cbk.validation_data = val_ins
return callbacks, callback_model
from keras.callbacks import EarlyStopping, ProgbarLogger, ModelCheckpoint
from keras.callbacks import LearningRateScheduler, CSVLogger, TensorBoard
from keras.optimizers import RMSprop
from keras.models import load_model
import tools
from facial_recognizer import facial_recognizer
from config import config as c
os.makedirs(c.path_to_summaries, exist_ok=True)
os.makedirs(c.path_to_models, exist_ok=True)
call_backs = [
# EarlyStopping('val_acc', min_delta=1e-5, patience=20),
ProgbarLogger('steps'),
ModelCheckpoint(c.path_to_models + '/model_OE', save_best_only=True),
TensorBoard(c.path_to_summaries)
]
model = facial_recognizer(c)
model.compile(
# optimizer=SGD(lr=1e-3, momentum=0.9),
optimizer='adam',
loss={
'out_au': 'binary_crossentropy',
'out_emotion': 'categorical_crossentropy'
})
train_gen = tools.fake_generator(c)
for _ in range(10):
target_size=(img_height, img_width),
batch_size=batch_size,
class_mode="categorical",
seed=seed)
temp2 = ig2.flow_from_directory(Xs,
target_size=(img_height, img_width),
batch_size=batch_size,
class_mode="categorical",
seed=seed)
for batch in izip(temp1, temp2):
yield [batch[0][0], batch[1][0]], [batch[0][1]]
# Save the model according to the conditions
progbar = ProgbarLogger(count_mode='steps')
checkpoint = ModelCheckpoint("rgbd.{epoch:02d}-{val_loss:.2f}.hdf5",
monitor='val_acc',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto',
period=1)
early = EarlyStopping(monitor='val_acc',
min_delta=0,
patience=1,
verbose=1,
mode='auto')
# Train the model
model1.fit_generator(custom_iterator(train_data_dir_nir, train_data_dir),
def main(args):
try:
opts, args = getopt.getopt(args, "c:s", ["config="])
except getopt.GetoptError:
print('usage: -c config.json')
sys.exit(2)
start_from_model = False
for opt, arg in opts:
if opt in ("-c", "--config"):
config_fname = os.path.join('configurations', arg)
elif opt == '-s':
start_from_model = True
if start_from_model:
filemode = 'a'
else:
filemode = 'w'
log_path = 'logging/vae_nlg_{}'.format(int(round(time.time() * 1000)))
os.mkdir(log_path)
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO,
filename='{}/evolution.log'.format(log_path),
filemode=filemode)
with open(config_fname, 'r') as json_data:
config_data = json.load(json_data)
batch_size = config_data['batch_size']
epochs = config_data['nb_epochs']
discriminator_iterations = config_data['discriminator_iterations']
tweets_path = config_data['tweets_path']
vocab_path = config_data['vocab_path']
vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb'))
#== == == == == == =
# Load all the Data
#== == == == == == =
delimiter = ''
noutputs = 11
logging.info('Load Training Data')
train_input, train_output, train_weights, train_lex = load_text_gen_data(
join(tweets_path, 'trainset.csv'),
config_data,
vocab,
noutputs,
word_based=False)
logging.info('Load Validation Data')
valid_input, valid_output, _, valid_lex = load_text_gen_data(
join(tweets_path, 'devset.csv'),
config_data,
vocab,
noutputs,
word_based=False)
logging.info('Load Output Validation Data')
valid_dev_input, valid_dev_output, _, valid_dev_lex = load_text_gen_data(
join(tweets_path, 'devset_reduced.csv'),
config_data,
vocab,
noutputs,
random_output=False,
word_based=False)
step = K.variable(1., name='step_varialbe')
steps_per_epoch = ceil(train_output[0].shape[0] /
config_data['batch_size'])
# == == == == == == == == == == =
# Define and load the CNN model
# == == == == == == == == == == =
vae_model_train, vae_model_test, vae_vanilla_train_model, vae_vanilla_test_model, discriminator_model, decoder_test, discriminator = get_vae_gan_model(
config_data, vocab, step)
with open(os.path.join(log_path, 'models.txt'), 'wt') as fh:
fh.write('VAE Model Train\n')
fh.write('---------\n')
vae_model_train.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('VAE Model Test\n')
fh.write('--------------\n')
vae_model_test.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('VAE Model Pretrain\n')
fh.write('---------------------------\n')
vae_vanilla_train_model.summary(
print_fn=lambda x: fh.write(x + '\n'))
fh.write('VAE Model Pretrain Test\n')
fh.write('---------------------------\n')
vae_vanilla_test_model.summary(
print_fn=lambda x: fh.write(x + '\n'))
fh.write('Decoder Test\n')
fh.write('-------------------\n')
decoder_test.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('Discriminator Models\n')
fh.write('-------------------\n')
discriminator_model.summary(print_fn=lambda x: fh.write(x + '\n'))
terminate_on_nan = TerminateOnNaN()
output_callback = LexOutputCallback(
vae_vanilla_test_model,
valid_dev_input,
valid_dev_lex,
1,
vocab,
delimiter,
fname='{}/test_output'.format(log_path))
#output_callback_full = LexOutputCallback(vae_vanilla_test_model, valid_dev_input, valid_dev_lex, 1, vocab, delimiter, fname='{}/test_output'.format(log_path))
#
# vae_vanilla_train_model.fit_generator(
# generator=generate_data_stream(config_data['pretrain_path'], config_data, vocab, config_data['batch_size'], noutputs=3),
# steps_per_epoch=steps_per_epoch,
# epochs=ceil(config_data['pretrain_samples']/config_data['pretrain_samples_per_epoch']),
# callbacks=[output_callback, terminate_on_nan],
# validation_data=(valid_input, valid_output[:3]),
# )
vae_vanilla_train_model.fit(
x=train_input,
y=train_output[:2],
epochs=config_data['pretrain_epochs'],
batch_size=batch_size,
validation_data=(valid_input, valid_output[:2]),
sample_weight=train_weights[:2],
callbacks=[output_callback, terminate_on_nan])
terminate_on_nan = TerminateOnNaN()
model_checkpoint = ModelCheckpoint(
'models/vae_model/weights.{epoch:02d}.hdf5',
period=10,
save_weights_only=True)
out_labels = [
'enc_' + s for s in vae_model_train._get_deduped_metrics_names()
]
out_labels += [
'dis_' + s
for s in discriminator_model._get_deduped_metrics_names()
]
callback_metrics = out_labels + ['val_' + n for n in out_labels]
tensorboard = TensorBoard(log_dir='logging/tensorboard',
histogram_freq=0,
write_grads=True,
write_images=True)
step_callback = StepCallback(step, steps_per_epoch)
output_callback = LexOutputCallback(
vae_vanilla_test_model,
valid_dev_input,
valid_dev_lex,
1,
vocab,
delimiter,
fname='{}/test_output'.format(log_path))
output_callback_full = LexOutputCallback(
vae_vanilla_test_model,
valid_input,
valid_lex,
5,
vocab,
delimiter,
fname='{}/test_valid_output'.format(log_path))
callbacks = CallbackList([
BaseLogger(),
ProgbarLogger(count_mode='steps'), step_callback, tensorboard,
output_callback, output_callback_full, model_checkpoint,
terminate_on_nan
])
callbacks.set_model(vae_model_train)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': True,
'do_validation': True,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
initial_epoch = 0
num_train_samples = train_input[0].shape[0]
index_array = np.arange(num_train_samples)
steps = 0
epoch = initial_epoch
while epoch < epochs:
epoch_logs = {}
callbacks.on_epoch_begin(epoch)
index_array = _batch_shuffle(index_array, batch_size)
steps_done = 0
batches = _make_batches(num_train_samples, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_logs = {}
batch_ids = index_array[batch_start:batch_end]
X = _slice_arrays(train_input, batch_ids)
y = _slice_arrays(train_output, batch_ids)
sample_weights = _slice_arrays(train_weights, batch_ids)
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
set_trainability(discriminator, trainable=False)
enc_outs = vae_model_train.train_on_batch(
x=X, y=y, sample_weight=sample_weights)
set_trainability(discriminator, trainable=True)
list_disc_loss_real = []
if steps < 25 or steps % 500 == 0:
disc_iterations = 25
else:
disc_iterations = discriminator_iterations
for disc_it in range(disc_iterations):
real_idx = np.random.choice(train_input[0].shape[0],
len(batch_ids),
replace=False)
disX_train = train_input[-1][
real_idx] #take input 8 as train input and the rest as targets
disy_train = [x[real_idx] for x in train_input[:8]
] #take input 1-7 as targets
#train on real data
dis_outs_real = discriminator_model.train_on_batch(
disX_train, disy_train)
list_disc_loss_real.append(dis_outs_real)
loss_d_real = np.mean(list_disc_loss_real, axis=0)
outs = np.concatenate((enc_outs, loss_d_real))
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
epoch_logs = {}
batch_index += 1
steps_done += 1
steps += 1
# Epoch finished.
if steps_done >= steps_per_epoch:
valid_len = valid_output[0].shape[0]
enc_val_outs = vae_model_train.evaluate(valid_input,
valid_output,
verbose=False)
dis_val_outs = discriminator_model.evaluate(
valid_input[-1], valid_input[:8], verbose=False)
val_outs = enc_val_outs + dis_val_outs
#val_outs = full_model.evaluate(valid_input, valid_output, verbose=False)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
callbacks.on_train_end()
model_metrics = [
'accuracy',
]
model = build_model()
model.compile(optimizer='adam',
loss=model_losses,
loss_weights=model_loss_weights,
metrics=model_metrics)
model.summary()
if __debug__: print('Fitting...')
callbacks = [ProgbarLogger()]
if args.early_stopping:
callbacks.append(EarlyStopping(monitor='val_loss', patience=5))
model.fit(X_train,
y_train,
validation_data=(X_dev, y_dev),
epochs=args.epochs,
batch_size=args.bsize,
callbacks=callbacks,
verbose=args.verbose)
if __debug__:
print(args)
print('Evaluating...')
model = Model(input=sequence_one_hot, output=dense3)
model.compile(optimizer='sgd',
loss='categorical_crossentropy',
metrics=['accuracy'])
# Generates .png image with the model structure
plot_model(model, show_shapes=True, to_file='model_structure_new.png')
step_size_callback = LearningRateScheduler(step_size_decay)
# Callbacks to save and retreive the best weight configurations found during training phase
all_callbacks = [
step_size_callback,
FlushCallback(),
ModelCheckpoint('model' + '.hdf5', save_best_only=True, verbose=1),
ProgbarLogger(count_mode='samples')
]
#--------------- TRAINING ------------------#
# No vaidation split was given in the paper, so I've set it to 0.2 arbitarily
hist = model.fit(x,
y,
batch_size=mini_batch_size,
nb_epoch=max_epochs,
verbose=2,
validation_split=0.2,
callbacks=all_callbacks)
# Prints to console, the summary of losses etc through all the epochs
ch = input("\nPrint training loss and error history? (y/n)")
if ch == 'y':
def fit_dataset(self,
dataset,
steps_per_epoch=None,
batch_size=32,
epochs=1,
verbose=1,
callbacks=None,
on_sample=None,
on_scores=None):
"""Train the model on the given dataset for a given number of epochs.
Arguments
---------
dataset: Instance of `BaseDataset` that provides the data
to train on.
steps_per_epoch: int or None, number of gradient updates before
considering an epoch has passed. If None it is set
to be `len(dataset.train_data) / batch_size`.
batch_size: int, number of samples per gradient update
epochs: int, number of times to iterate `steps_per_epoch` times
verbose: {0, >0}, whether to employ the progress bar Keras
callback or not
callbacks: list of Keras callbacks to be called during training
on_sample: callable that accepts the sampler, idxs, w, scores
on_scores: callable that accepts the sampler and scores
"""
try:
if len(dataset.train_data) < batch_size:
raise ValueError(("The model cannot be trained with "
"batch_size > training set"))
except RuntimeError as e:
assert "no size" in str(e)
# Set steps_per_epoch properly
if steps_per_epoch is None:
steps_per_epoch = len(dataset.train_data) // batch_size
# Create the callbacks list
self.history = History()
callbacks = [BaseLogger()] + (callbacks or []) + [self.history]
if verbose > 0:
callbacks += [ProgbarLogger(count_mode="steps")]
callbacks = CallbackList(callbacks)
#TODO: Should we be making it possible to call back a different model
# than self.model.model?
callbacks.set_model(self.model.model)
callbacks.set_params({
"epochs":
epochs,
"steps":
steps_per_epoch,
"verbose":
verbose,
"do_validation":
len(dataset.test_data) > 0,
"metrics":
self._get_metric_names() +
["val_" + n for n in self._get_metric_names()]
})
# Create the sampler
sampler = self.sampler(dataset, batch_size, steps_per_epoch, epochs)
# Start the training loop
epoch = 0
self.model.model.stop_training = False
callbacks.on_train_begin()
while epoch < epochs:
callbacks.on_epoch_begin(epoch)
for step in range(steps_per_epoch):
batch_logs = {"batch": step, "size": batch_size}
callbacks.on_batch_begin(step, batch_logs)
# Importance sampling is done here
idxs, (x, y), w = sampler.sample(batch_size)
# Train on the sampled data
loss, metrics, scores = self.model.train_batch(x, y, w)
# Update the sampler
sampler.update(idxs, scores)
values = map(lambda x: x.mean(), [loss] + metrics)
for l, o in zip(self._get_metric_names(), values):
batch_logs[l] = o
callbacks.on_batch_end(step, batch_logs)
if on_scores is not None:
on_scores(sampler, self._latest_scores)
if on_sample is not None:
on_sample(sampler, self._latest_sample_event["idxs"],
self._latest_sample_event["w"],
self._latest_sample_event["predicted_scores"])
if self.model.model.stop_training:
break
# Evaluate now that an epoch passed
epoch_logs = {}
if len(dataset.test_data) > 0:
val = self.model.evaluate(*dataset.test_data[:],
batch_size=batch_size)
epoch_logs = {
"val_" + l: o
for l, o in zip(self._get_metric_names(), val)
}
callbacks.on_epoch_end(epoch, epoch_logs)
if self.model.model.stop_training:
break
epoch += 1
callbacks.on_train_end()
return self.history
def train(self,
epochs,
steps_per_epoch,
initial_epoch=0,
end_of_epoch_callback=None,
verbose=1):
epoch = initial_epoch
logger = ProgbarLogger(count_mode='steps')
logger.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'metrics': self.metric_names
})
logger.on_train_begin()
while epoch < epochs:
step = 0
batch = 0
logger.on_epoch_begin(epoch)
while step < steps_per_epoch:
self.batch_logs['batch'] = batch
logger.on_batch_begin(batch, self.batch_logs)
for i in range(len(self.models)):
x, y = self.output_generators[i].__next__()
if x is None:
x, y = self.output_generators[i].__next__()
outs = self.models[i].train_on_batch(x, y)
if not isinstance(outs, list):
outs = [outs]
if self.print_full_losses:
for l, o in zip(self.metric_names, outs):
self.batch_logs[l] = o
else:
self.batch_logs[self.metric_names[i]] = outs[0]
logger.on_batch_end(batch, self.batch_logs)
step += 1
batch += 1
logger.on_epoch_end(epoch)
if end_of_epoch_callback is not None:
end_of_epoch_callback(epoch)
epoch += 1
def main(args):
try:
opts, args = getopt.getopt(args, "c:s", ["config="])
except getopt.GetoptError:
print('usage: -c config.json')
sys.exit(2)
start_from_model = False
for opt, arg in opts:
if opt in ("-c", "--config"):
config_fname = os.path.join('configurations', arg)
elif opt == '-s':
start_from_model = True
if start_from_model:
filemode = 'a'
else:
filemode = 'w'
log_path = 'logging/vae_nlg_{}'.format(int(round(time.time() * 1000)))
os.mkdir(log_path)
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO,
filename='{}/evolution.log'.format(log_path),
filemode=filemode)
with open(config_fname, 'r') as json_data:
config_data = json.load(json_data)
batch_size = config_data['batch_size']
epochs = config_data['nb_epochs']
discriminator_iterations = config_data['discriminator_iterations']
tweets_path = config_data['tweets_path']
vocab_path = config_data['vocab_path']
vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb'))
#== == == == == == =
# Load all the Data
#== == == == == == =
noutputs = 5
logging.info('Load Training Data')
train_input, train_output = load_text_pairs(
join(tweets_path, 'training_set.tsv'), config_data, vocab,
noutputs)
logging.info('Load Validation Data')
valid_input, valid_output = load_text_pairs(
join(tweets_path, 'vaild_set.tsv'), config_data, vocab, noutputs)
logging.info('Load Output Validation Data')
valid_dev_input, valid_dev_output = load_text_pairs(
join(tweets_path, 'test_set.tsv'), config_data, vocab, noutputs)
#train_input = [x[:1213] for x in train_input]
#train_output = [x[:1213] for x in train_output]
noise_valid_input = np.zeros(shape=(valid_input[0].shape[0],
config_data['z_size']))
step = K.variable(1.)
steps_per_epoch = ceil(train_output[0].shape[0] /
config_data['batch_size'])
# == == == == == == == == == == =
# Define and load the CNN model
# == == == == == == == == == == =
vae_model, vae_model_test, decoder_discr_model, decoder_test_model, discriminator_model, discriminator = get_vae_gan_model(
config_data, vocab, step)
with open(os.path.join(log_path, 'models.txt'), 'wt') as fh:
fh.write('VAE Model\n')
fh.write('---------\n')
vae_model.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('VAE Model Test\n')
fh.write('--------------\n')
vae_model_test.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('Decoder Discriminator Model\n')
fh.write('---------------------------\n')
decoder_discr_model.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('Decoder Test Model\n')
fh.write('---------------------------\n')
decoder_test_model.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('Discriminator Model\n')
fh.write('-------------------\n')
discriminator_model.summary(print_fn=lambda x: fh.write(x + '\n'))
terminate_on_nan = TerminateOnNaN()
model_checkpoint = ModelCheckpoint(
'models/vae_model/weights.{epoch:02d}.hdf5',
period=10,
save_weights_only=True)
enc_out_labels = [
'enc_' + s for s in vae_model._get_deduped_metrics_names()
]
dec_out_labels = [
'dec_' + s
for s in decoder_discr_model._get_deduped_metrics_names()
]
dis_out_labels = [
'dis_' + s
for s in discriminator_model._get_deduped_metrics_names()
]
out_labels = enc_out_labels + dec_out_labels + [
'dis_real', 'dis_gen', 'dis_noise'
]
#out_labels = full_model._get_deduped_metrics_names()
callback_metrics = out_labels + ['val_' + n for n in out_labels]
step_callback = StepCallback(step, steps_per_epoch)
output_callback = GANOutputCallback(
vae_model_test,
valid_dev_input[0],
1,
vocab,
'',
fname='{}/test_output'.format(log_path))
callbacks = CallbackList([
BaseLogger(),
ProgbarLogger(count_mode='steps'), step_callback, output_callback,
model_checkpoint, terminate_on_nan
])
callbacks.set_model(vae_model_test)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': True,
'do_validation': True,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
initial_epoch = 0
num_train_samples = train_input[0].shape[0]
index_array = np.arange(num_train_samples)
steps = 0
epoch = initial_epoch
while epoch < epochs:
epoch_logs = {}
callbacks.on_epoch_begin(epoch)
index_array = _batch_shuffle(index_array, batch_size)
steps_done = 0
batches = _make_batches(num_train_samples, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_logs = {}
batch_ids = index_array[batch_start:batch_end]
X = _slice_arrays(train_input, batch_ids)
y = _slice_arrays(train_output, batch_ids)
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
set_trainability(discriminator, trainable=False)
enc_outs = vae_model.train_on_batch(x=X, y=y[:3])
set_trainability(discriminator, trainable=True)
list_disc_loss_real = []
list_disc_loss_gen = []
list_disc_loss_noise = []
if steps < 25 or steps % 500 == 0:
disc_iterations = 100
else:
disc_iterations = discriminator_iterations
noise_input = np.zeros(shape=(len(batch_ids),
config_data['z_size']))
for disc_it in range(disc_iterations):
#clip_weights(discriminator)
real_idx = np.random.choice(train_input[0].shape[0],
len(batch_ids),
replace=False)
train_real_batch = [x[real_idx] for x in train_input]
#train on real data
x_fake = vae_model_test.predict_on_batch(
x=train_real_batch[0])
x_noise_fake = decoder_test_model.predict_on_batch(
x=noise_input)
train_input_discr = np.concatenate(
(train_real_batch[0], train_real_batch[0],
train_real_batch[0]))
train_output_discr = np.concatenate(
(train_real_batch[1], x_fake, x_noise_fake))
labels = np.asarray(
len(batch_ids) * [1] + 2 * len(batch_ids) * [-1])
index_array_discr = np.arange(len(labels))
np.random.shuffle(index_array_discr)
discr_batch = [
train_input_discr[index_array_discr],
train_output_discr[index_array_discr]
]
discr_batch_labels = labels[index_array_discr]
dis_outs_real = discriminator_model.train_on_batch(
discr_batch, discr_batch_labels)
#dis_outs_real = discriminator_model.train_on_batch(train_real_batch, -np.ones(shape=(len(batch_ids), 1)))
#dis_outs_gen = discriminator_model.train_on_batch([train_real_batch[0], x_fake], np.ones(shape=(len(batch_ids), 1)))
#dis_outs_gen_noise = discriminator_model.train_on_batch([train_real_batch[0], x_noise_fake], np.ones(shape=(len(batch_ids), 1)))
list_disc_loss_real.append(dis_outs_real)
#list_disc_loss_gen.append(dis_outs_gen)
#list_disc_loss_noise.append(dis_outs_gen_noise)
loss_d_real = -np.mean(list_disc_loss_real)
loss_d_gen = np.mean(list_disc_loss_gen)
loss_d_noise = np.mean(list_disc_loss_noise)
set_trainability(discriminator, trainable=False)
decoder_discr_input = [X[0], noise_input]
dec_outs = decoder_discr_model.train_on_batch(
x=decoder_discr_input,
y=-np.ones(shape=(len(batch_ids), 1)))
outs = enc_outs + [dec_outs] + [loss_d_real]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
epoch_logs = {}
batch_index += 1
steps_done += 1
steps += 1
# Epoch finished.
if steps_done >= steps_per_epoch:
valid_len = valid_output[0].shape[0]
enc_val_outs = vae_model.evaluate(valid_input,
valid_output[:3],
verbose=False)
dec_val_outs = decoder_discr_model.evaluate(
[valid_input[0], noise_valid_input],
-np.ones(shape=(valid_len, 1)),
verbose=False)
dis_val_outs = discriminator_model.evaluate(
valid_input,
-np.ones(shape=(valid_len, 1)),
verbose=False)
val_outs = enc_val_outs + [dec_val_outs] + [dis_val_outs]
#val_outs = full_model.evaluate(valid_input, valid_output, verbose=False)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
callbacks.on_train_end()
from keras.layers import Dense
from keras.models import Sequential
from keras.callbacks import ProgbarLogger
import numpy as np
import matplotlib.pyplot as plt
n_samples = 500
def powerTwo(x):
return x*x
np.random.seed(10)
X = np.random.uniform(-2, 2, n_samples)
Y = powerTwo(X)
# tarin MLP
model = Sequential()
model.add(Dense(100, input_dim=1, activation='relu'))
model.add(Dense(1))
model.compile(loss='mae', optimizer='rmsprop', metrics=['accuracy'])
model.fit(X, Y, nb_epoch=20, batch_size=2, verbose=1, validation_split = 0.1, callbacks =[ProgbarLogger()])
#try new samples
new_samples = 300
new_X = np.random.uniform(-2, 2, new_samples)
new_Y = powerTwo(new_X)
pred_Y = model.predict(new_X)
plt.plot(new_X,new_Y,'go',label="X*X")
plt.plot(new_X,pred_Y,'yo',label="PredictedX*X")
plt.show()
print("green shows original X^2 and yellow shows predicted X^2")
def __init__(self, verbose=1):
ProgbarLogger.__init__(self)
self.verbose2 = verbose
# genenor.next()
# genenor.next()
# exit()
name_experiment = params_train['name_experiment']
# exit()
batch_size = params_train['batch_size']
mkdir(name_experiment)
filepath = './' + name_experiment + '/' + name_experiment + '_weights_{epoch:02d}_{loss:.4f}.hdf5'
checkpointer = ModelCheckpoint(filepath=filepath,
verbose=1,
monitor='loss',
mode='min',
save_best_only=True,
save_weights_only=True)
progbarLogger = ProgbarLogger()
model = get_model(params_transform, params_train)
model = initialize_model_from_vgg(model)
model.load_weights(filepath='./exp1/exp1_weights_00_1849.1228.hdf5')
"""
COCO数据集中
train2014 : 82783 个样本
val2014 : 40504 个样本
数据集生成的时候，将val2014中的前2644个样本标记位 'isValidation = 1'
所以用于训练的样本数为 82783+40504-2644 = 120643
这里设置，我们训练model的总样本数为 6000000
"""
model.fit_generator(
generator=genenor,
steps_per_epoch=params_train['steps_per_epoch'],
def _train_by_batch(self):
# batch finite generator should be loaded within epoch loop
logger.info('Start training by batch')
self.validation_xy = self.load_data('val', feed_mode='all')
do_validation = bool(self.validation_xy)
# prepare display labels in tensorboard
out_labels = self.model._get_deduped_metrics_names()
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# prepare callbacks
self.model.history = History()
callbacks = [BaseLogger()] + (self.callbacks
or []) + [self.model.history]
# callbacks = (self.callbacks or []) + [self.model.history]
if self.verbose:
callbacks += [ProgbarLogger(count_mode='samples')]
callbacks = CallbackList(callbacks)
# it's possible to callback a different model than this model
if hasattr(self.model, 'callback_model') and self.model.callback_model:
callback_model = self.model.callback_model
else:
callback_model = self.model
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': self.epochs,
'samples': self.data.nb_train,
'verbose': self.verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
for epoch in range(self.epochs):
start_e = time()
callbacks.on_epoch_begin(epoch)
xy_gen = self.load_data('train', feed_mode='batch')
logger.info('New training epoch')
for batch_index, (x, y) in enumerate(xy_gen):
# build batch logs
batch_logs = {}
if isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
outs = self.model.train_on_batch(x, y)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if (batch_index + 1) % 1000 == 0 and do_validation:
val_outs = self.model.evaluate(*self.validation_xy,
batch_size=81920,
verbose=0)
batch_logs = {}
if not isinstance(val_outs, list):
val_outs = [val_outs]
for l, o in zip(out_labels, val_outs):
batch_logs['val_' + l] = o
print(' - Eval inside: %.6f' % val_outs[0])
for cb in self.callbacks:
if cb.__class__ == tensorBoard:
cb.on_batch_end(batch_index,
batch_logs,
count=False)
epoch_logs = {}
if do_validation:
val_outs = self.model.evaluate(*self.validation_xy,
batch_size=81920,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_batch_end(epoch, epoch_logs)
callbacks.on_epoch_end(epoch, epoch_logs)
elapsed_e = timedelta(seconds=int(time() - start_e))
self.send_metric('elapsed_per_epoch', elapsed_e)
if not self.no_save and do_validation and (epoch !=
self.epochs - 1):
self.model.save(
'results/trained_models/%s_ctr_model_%.4f_epoch_%d.h5' %
(self.sess_id, val_outs[0], epoch))
callbacks.on_train_end()
return self.model.history
def predict_image(version, image_path, batch_size, overlap, data_format=None):
def current_time_millis():
return int(round(time.time() * 1000))
def offset(size, diff, overlap):
return math.floor(diff / math.ceil(diff / (size * (1 - overlap))))
def map_c(i, j, b, l):
return int(((i * b) + j) / l)
def map_r(i, j, b, l):
return ((i * b) + j) % l
if data_format is None:
data_format = K.image_data_format()
if data_format not in {'channels_first', 'channels_last'}:
raise ValueError('Unknown data_format:', data_format)
path = version.model_file.name
print(_('Loading model "%(path)s".') % {'path': path})
model = load_model(os.path.join(settings.MEDIA_ROOT, path))
if len(model.inputs) != 1:
raise RuntimeError('Models with more than one input are not'
' supported at the moment.')
inputs = []
for i in range(len(model.inputs)):
name = model.inputs[i].name
pos = min(
name.index('/') if '/' in name else len(name),
name.index(':') if ':' in name else len(name))
name = name[:pos]
inputs.append({'shape': model.inputs[i].shape.as_list(), 'name': name})
if data_format == 'channels_first':
inputs[i]['grayscale'] = inputs[i]['shape'][1] == 1
inputs[i]['r'] = inputs[i]['shape'][2]
inputs[i]['c'] = inputs[i]['shape'][3]
elif data_format == 'channels_last':
inputs[i]['r'] = inputs[i]['shape'][1]
inputs[i]['c'] = inputs[i]['shape'][2]
inputs[i]['grayscale'] = inputs[i]['shape'][3] == 1
inputs[i]['img'] = img_to_array(
load_img(image_path, inputs[i]['grayscale']))
inputs[i]['img'] *= 1. / 255
if data_format == 'channels_first':
inputs[i]['img_r'] = inputs[i]['img'].shape[1]
inputs[i]['img_c'] = inputs[i]['img'].shape[2]
elif data_format == 'channels_last':
inputs[i]['img_r'] = inputs[i]['img'].shape[0]
inputs[i]['img_c'] = inputs[i]['img'].shape[1]
inputs[i]['diff_r'] = inputs[i]['img_r'] - inputs[i]['r']
inputs[i]['diff_c'] = inputs[i]['img_c'] - inputs[i]['c']
inputs[i]['offset_r'] = offset(inputs[i]['r'], inputs[i]['diff_r'],
overlap)
inputs[i]['offset_c'] = offset(inputs[i]['c'], inputs[i]['diff_c'],
overlap)
inputs[i]['nb_r'] = math.ceil(
inputs[i]['diff_r'] / inputs[i]['offset_r']) + 1
inputs[i]['nb_c'] = math.ceil(
inputs[i]['diff_c'] / inputs[i]['offset_c']) + 1
inputs = inputs[0]
N = inputs['nb_r'] * inputs['nb_c']
steps = math.ceil(N / batch_size)
metrics = []
outputs = []
for i in range(len(model.outputs)):
tshape = model.outputs[i].shape.as_list()
name = model.outputs[i].name
pos = min(
name.index('/') if '/' in name else len(name),
name.index(':') if ':' in name else len(name))
name = name[:pos]
activation = model.get_layer(name).activation.__name__.lower()
outputs.append({'name': name, 'shape': tshape})
if len(tshape) == 2:
if activation == 'softmax':
outputs[i]['t'] = 'class'
else:
outputs[i]['t'] = 'multi'
nb_classes = tshape[1]
if nb_classes is None:
nb_classes = model.get_layer(name).output_shape[1]
nb_classes = int(nb_classes)
metrics += ['%s:%s' % (name, i) for i in range(nb_classes)]
if data_format == 'channels_first':
shape = (nb_classes, inputs['nb_r'], inputs['nb_c'])
elif data_format == 'channels_last':
shape = (inputs['nb_r'], inputs['nb_c'], nb_classes)
elif len(tshape) == 4:
if activation == 'softmax':
outputs[i]['t'] = 'class'
else:
outputs[i]['t'] = 'img'
shape = (inputs['nb_r'], inputs['nb_c']) + tuple(tshape[1:])
outputs[i]['p'] = np.zeros(shape)
history = History()
callbacks = CallbackList([BaseLogger(), history, ProgbarLogger()])
callbacks.set_model(model)
callbacks.set_params({
'batch_size': batch_size,
'epochs': 1,
'steps': steps,
'samples': N,
'verbose': 1,
'do_validation': False,
'metrics': metrics,
})
callbacks.on_train_begin()
callbacks.on_epoch_begin(0)
start_time = current_time_millis()
for b in range(steps):
current_index = (b * batch_size) % N
if N >= current_index + batch_size:
current_batch_size = batch_size
else:
current_batch_size = N - current_index
batch_logs = {'batch': b, 'size': current_batch_size}
for metric in metrics:
batch_logs[metric] = 0
callbacks.on_batch_begin(b, batch_logs)
bX = np.zeros((current_batch_size, ) + tuple(inputs['shape'][1:]))
for j in range(current_batch_size):
idx_r = map_r(b, j, batch_size, inputs['nb_r'])
idx_c = map_c(b, j, batch_size, inputs['nb_r'])
top = min(idx_r * inputs['offset_r'],
inputs['img_r'] - inputs['r'])
bottom = min(idx_r * inputs['offset_r'] + inputs['r'],
inputs['img_r'])
left = min(idx_c * inputs['offset_c'],
inputs['img_c'] - inputs['c'])
right = min(idx_c * inputs['offset_c'] + inputs['c'],
inputs['img_c'])
if data_format == 'channels_first':
bX[j] = inputs['img'][:, top:bottom, left:right]
elif data_format == 'channels_last':
bX[j] = inputs['img'][top:bottom, left:right, :]
p = model.predict_on_batch(bX)
if type(p) != list:
p = [p]
for j in range(current_batch_size):
for i in range(len(outputs)):
idx_r = map_r(b, j, batch_size, inputs['nb_r'])
idx_c = map_c(b, j, batch_size, inputs['nb_r'])
if len(outputs[i]['p'].shape) == 3:
if data_format == 'channels_first':
outputs[i]['p'][:, idx_r, idx_c] = p[i][j]
elif data_format == 'channels_last':
outputs[i]['p'][idx_r, idx_c, :] = p[i][j]
metric = metrics[p[i][j].argmax()]
batch_logs[metric] += 1. / current_batch_size
elif len(outputs[i]['p'].shape) == 5:
outputs[i]['p'][idx_r, idx_c, :, :, :] = p[i][j]
callbacks.on_batch_end(b, batch_logs)
runtime = (current_time_millis() - start_time) / 1000
callbacks.on_epoch_end(0, {'runtime': runtime})
callbacks.on_train_end()
for i in range(len(outputs)):
if len(outputs[i]['shape']) == 2:
if data_format == 'channels_first':
shape = (outputs[i]['p'].shape[0], inputs['img_r'],
inputs['img_c'])
elif data_format == 'channels_last':
shape = (inputs['img_r'], inputs['img_c'],
outputs[i]['p'].shape[2])
elif len(tshape) == 4:
if data_format == 'channels_first':
shape = (outputs[i]['p'].shape[2], inputs['img_r'],
inputs['img_c'])
elif data_format == 'channels_last':
shape = (inputs['img_r'], inputs['img_c'],
outputs[i]['p'].shape[4])
count = np.zeros(shape)
outputs[i]['img'] = np.zeros(shape)
if len(outputs[i]['p'].shape) == 3:
if data_format == 'channels_first':
nb_rows = outputs[i]['p'].shape[1]
nb_cols = outputs[i]['p'].shape[2]
elif data_format == 'channels_last':
nb_rows = outputs[i]['p'].shape[0]
nb_cols = outputs[i]['p'].shape[1]
elif len(outputs[i]['p'].shape) == 5:
nb_rows = outputs[i]['p'].shape[0]
nb_cols = outputs[i]['p'].shape[1]
for j in range(nb_rows):
for k in range(nb_cols):
top = min(j * inputs['offset_r'],
inputs['img_r'] - inputs['r'])
bottom = min(j * inputs['offset_r'] + inputs['r'],
inputs['img_r'])
left = min(k * inputs['offset_c'],
inputs['img_c'] - inputs['c'])
right = min(k * inputs['offset_c'] + inputs['c'],
inputs['img_c'])
if data_format == 'channels_first':
outputs[i]['img'][:, top:bottom, left:right] += \
outputs[i]['p'][:, j, k]
count[:, top:bottom, left:right] += 1
elif data_format == 'channels_last':
outputs[i]['img'][top:bottom, left:right, :] += \
outputs[i]['p'][j, k, :]
count[top:bottom, left:right, :] += 1
outputs[i]['img'] /= count
del outputs[i]['p']
del outputs[i]['shape']
return history.history, outputs
def add_progressbar_callback(self):
self.callbacks.append(ProgbarLogger())
def on_train_begin(self, logs=None):
ProgbarLogger.on_train_begin(self, logs)
self.verbose = self.verbose2
def main(args):
try:
opts, args = getopt.getopt(args, "c:s", ["config="])
except getopt.GetoptError:
print('usage: -c config.json')
sys.exit(2)
start_from_model = False
for opt, arg in opts:
if opt in ("-c", "--config"):
config_fname = os.path.join('configurations', arg)
elif opt == '-s':
start_from_model = True
if start_from_model:
filemode = 'a'
else:
filemode = 'w'
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO,
filename='logging/vae_gan/evolution.log',
filemode=filemode)
with open(config_fname, 'r') as json_data:
config_data = json.load(json_data)
tweets_path = config_data['tweets_path']
vocab_path = config_data['vocab_path']
vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb'))
#== == == == == == =
# Load all the Data
#== == == == == == =
noutputs = 5
logging.info('Load Training Data')
train_input, train_output = load_data(
join(tweets_path, 'en_train.tsv'), config_data, vocab, noutputs)
logging.info('Load Validation Data')
valid_input, valid_output = load_data(
join(tweets_path, 'en_valid15.tsv'), config_data, vocab, noutputs)
logging.info('Load Validation Data')
valid_input2, valid_output2 = load_data(
join(tweets_path, 'en_test16.tsv'), config_data, vocab, noutputs)
logging.info('Load Test Data')
test_input, test_output = load_data(join(tweets_path, 'en_test17.tsv'),
config_data, vocab, noutputs)
step = K.variable(1.)
# == == == == == == == == == == =
# Define and load the CNN model
# == == == == == == == == == == =
full_model, encoding_train_model, decoder_train_model, discriminator_train_model, decoder_inference, encoder, decoder, discriminator, discriminator_pretrain_model = vae_gan_model(
config_data, vocab, step)
#full_model.summary()
encoding_train_model.summary()
decoder_train_model.summary()
discriminator_train_model.summary()
decoder_inference.summary()
encoder.summary()
decoder.summary()
discriminator.summary()
#pretrain_discriminator(discriminator_pretrain_model, train_input, vocab)
model_path = 'models/vae_model/'
steps_per_epoch = int(
ceil(config_data['samples_per_epoch'] / config_data['batch_size']))
epochs = int(
ceil(config_data['nb_epochs'] *
(config_data['nsamples'] / config_data['samples_per_epoch'])))
batch_size = config_data['batch_size']
initial_epoch = 0
skip_texts = 0
terminate_on_nan = TerminateOnNaN()
model_checkpoint = ModelCheckpoint(
'models/vae_model/weights.{epoch:02d}.hdf5',
period=10,
save_weights_only=True)
generator = generate_data_stream(config_data['training_path'],
config_data,
vocab,
config_data['batch_size'],
skip_data=skip_texts,
noutputs=noutputs)
enqueuer = GeneratorEnqueuer(generator,
use_multiprocessing=False,
wait_time=0.01)
enqueuer.start(workers=1, max_queue_size=10)
output_generator = enqueuer.get()
enc_out_labels = [
'enc_' + s
for s in encoding_train_model._get_deduped_metrics_names()
]
dec_out_labels = [
'dec_' + s
for s in decoder_train_model._get_deduped_metrics_names()
]
dis_out_labels = [
'dis_' + s
for s in discriminator_train_model._get_deduped_metrics_names()
]
out_labels = enc_out_labels + dec_out_labels + dis_out_labels
#out_labels = full_model._get_deduped_metrics_names()
callback_metrics = out_labels + ['val_' + n for n in out_labels]
step_callback = NewCallback(step, steps_per_epoch)
output_callback = OutputCallback(decoder_inference, valid_input, 15,
vocab, '')
callbacks = CallbackList([
BaseLogger(),
ProgbarLogger(count_mode='steps'), step_callback, output_callback
])
callbacks.set_model(full_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': True,
'do_validation': True,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
epoch = initial_epoch
while epoch < epochs:
epoch_logs = {}
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
while steps_done < steps_per_epoch:
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
X, y = next(output_generator)
callbacks.on_batch_begin(batch_index, batch_logs)
set_trainability(encoder, trainable=True)
set_trainability(decoder, trainable=False)
set_trainability(discriminator, trainable=False)
enc_outs = encoding_train_model.train_on_batch(X, y[:3])
set_trainability(encoder, trainable=False)
set_trainability(decoder, trainable=True)
set_trainability(discriminator, trainable=False)
dec_outs = decoder_train_model.train_on_batch(X, y[:4])
set_trainability(encoder, trainable=False)
set_trainability(decoder, trainable=False)
set_trainability(discriminator, trainable=True)
dis_outs = discriminator_train_model.train_on_batch(X, y[0])
outs = enc_outs + dec_outs + [dis_outs]
#outs = full_model.train_on_batch(X, y)
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
epoch_logs = {}
batch_index += 1
steps_done += 1
# Epoch finished.
if steps_done >= steps_per_epoch:
enc_val_outs = encoding_train_model.evaluate(
valid_input, valid_output[:3], verbose=False)
dec_val_outs = decoder_train_model.evaluate(
valid_input, valid_output[:4], verbose=False)
dis_val_outs = discriminator_train_model.evaluate(
valid_input, valid_output[0], verbose=False)
val_outs = enc_val_outs + dec_val_outs + [dis_val_outs]
#val_outs = full_model.evaluate(valid_input, valid_output, verbose=False)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
callbacks.on_train_end()