def on_epoch_end(self, epoch, logs=None): if 'roc_auc' not in self.params['metrics']: # self.params['metrics'].append('roc_auc') self.params['metrics'].append('val_roc_auc') # X_train, Y_train, _, _ = self.model.training_data X_test = list(self.model.validation_data[0:-3]) Y_test = self.model.validation_data[-3] # train_pred = self.model.predict_proba(X_train, verbose=0) val_pred = self.model.predict_proba(X_test, verbose=0) # logs['roc_auc'] = roc_auc_score(Y_train[:,1], train_pred[:,1]) logs['val_roc_auc'] = roc_auc_score(Y_test[:, 1], val_pred[:, 1]) ProgbarLogger.on_epoch_end(self, epoch, logs)
def main(): os.makedirs(c.path_to_summaries, exist_ok=True) os.makedirs(c.path_to_models, exist_ok=True) model = get_model() call_backs = [ # EarlyStopping('val_acc', min_delta=1e-5, patience=20), ProgbarLogger('steps'), ModelCheckpoint(c.path_to_models + '/model', save_best_only=False), TensorBoard(c.path_to_summaries) ] train_gen, test_gen = tools.get_generators(c) # train_gen, test_gen = tools.fake_generator(c), tools.fake_generator(c) model.fit_generator(generator=train_gen, steps_per_epoch=50, epochs=c.epochs, verbose=1, callbacks=call_backs, validation_data=test_gen, validation_steps=30, max_queue_size=c.max_queue_size, workers=c.workers) model.save(c.path_to_models + '/model_OE')
def train(self, documents: Iterable[str], epochs: int, directory: Optional[str], progress_bar: bool = True): from keras.callbacks import ProgbarLogger, EarlyStopping, ReduceLROnPlateau, ModelCheckpoint self.save(directory) vectors, labels = self.tokenizer.encoded_training_set_from_documents( documents) callbacks = [ EarlyStopping(monitor="loss"), ReduceLROnPlateau(monitor="loss") ] if directory is not None: callbacks.append(ModelCheckpoint(self.model_path(directory))) if progress_bar: callbacks.append(ProgbarLogger()) if self.history.iterations < epochs: history = self.model.fit(vectors, labels, epochs=epochs, initial_epoch=self.history.iterations, verbose=int(progress_bar), callbacks=callbacks) self.history += TrainingHistory.from_keras_history(history) self.save(directory) return self.history
def __init__(self, model_obj): self.model_obj = model_obj self.checkpointer = ModelCheckpoint( filepath='/Users/rishab/Desktop/fynd/solid_app/tmp/weights.hdf5', verbose=1, save_best_only=True) self.progbarLogger = ProgbarLogger(count_mode='steps') self.early_monitor = EarlyStopping(monitor='val_loss', min_delta=0, patience=0, verbose=0, mode='auto') self.remote_monitor = RemoteMonitor(root='http://0.0.0.0:5001', path='/publish', field='data', headers=None) self.history = LossHistory() self.board = TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=32, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None)
def make_train_episode(model, layers, num_img, train_gen, valid_gen): path_to_weights = os.path.join(config.train.path_to_models, 'model{}'.format(layers)) call_backs = [ EarlyStopping('val_acc', min_delta=1e-5, patience=20), ProgbarLogger('steps'), ModelCheckpoint(path_to_weights, save_best_only=True, save_weights_only=True), LearningRateScheduler(lambda x: tools.lr_scheduler(x, config)), CSVLogger(config.train.path_to_log), TensorBoard(config.train.path_to_summaries) ] steps_per_epoch = int( (1 - config.data.valid_size) * num_img / config.data.batch_size) // 1 validation_steps = int( config.data.valid_size * num_img / config.data.batch_size) // 1 model.fit_generator(generator=train_gen, steps_per_epoch=steps_per_epoch, epochs=config.train.epochs, verbose=1, callbacks=call_backs, validation_data=valid_gen, validation_steps=validation_steps, max_queue_size=config.train.max_queue_size, workers=config.train.workers, use_multiprocessing=False)
def train_regression_model(): num_batches = 1848 total_iterations = 90000 batch_size = 64 num_epochs = ceil(total_iterations / num_batches) initial_epoch = 0 regression_model_file = 'model_%02d.hdf5' % initial_epoch regression_model = None if os.path.exists(regression_model_file): print('Loading from saved file.') regression_model = model.get_regression_model(regression_model_file) else: print('Start training from scratch.') regression_model = model.create_regression_model() regression_model.summary() progbar = ProgbarLogger('steps') checkpoint = ModelCheckpoint('model_{epoch:02d}.hdf5', verbose=1, monitor='loss') terminate = TerminateOnNaN() callbacks = [checkpoint, progbar, terminate] regression_model.fit_generator( generator=dataset.load_dataset(training_dir), steps_per_epoch=num_batches, epochs=num_epochs, callbacks=callbacks, initial_epoch=initial_epoch, verbose=1)
def train(steps=28, epochs=1, unet=0): model = get_model(IMAGE_H, IMAGE_W, INPUT_CHANNELS, SAVED_MODEL, unet) callbacks = [ ProgbarLogger(count_mode='steps', stateful_metrics=None), ModelCheckpoint('rgb_weights.h5', monitor='val_loss', save_best_only=False, verbose=0), util.MemLeakCallback() ] print("Training network") history = model.fit_generator( #Do training from generator generator=variation_gen(batch_size), steps_per_epoch= steps, #Number of samples obtained using batch_gen or variation_gen (28 for whole dataset) validation_data=batch_generator(batch_size), epochs=epochs, #Number of passes over whole dataset validation_steps=1, verbose=1, shuffle=False, callbacks=callbacks) model.save("./rgb.h5") return
def train(): mc = ModelCheckpoint(filepath="models", verbose=1, save_best_only=True) tb = TensorBoard("logs") pb = ProgbarLogger() train_gen, valid_gen = get_generators() train_size = 1281167 valid_size = 50000 m = VGG_M() m.compile(optimizer='adam', loss='categorical_crossentropy', metrics = ['accuracy']) m.fit_generator(train_gen, 1000, 1000, callbacks = [tb, pb], validation_data=valid_gen, nb_val_samples=500, nb_worker=16, pickle_safe=True)
def train(train_x, train_y, test_x, test_y, model, epochs): progbar = ProgbarLogger() history = model.fit( x=train_x, y=train_y, epochs=epochs, callbacks=[progbar], #validation_freq=1, validation_data=(test_x, test_y)) plot_model_history(history.history)
def train(X, Y): nnmodel = model(X) nnmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) progbar = ProgbarLogger() nnmodel.fit(X, Y, epochs=500, batch_size=64, verbose=0) scores = nnmodel.evaluate(X, Y) print(scores) model_json = nnmodel.to_json() with open("model.json", "w") as json_file: json_file.write(model_json) nnmodel.save_weights("model.h5") print("Saved model to disk")
def train(train_config): epoch = train_config.get('epoch', 10) batch_size = train_config.get('batch', 32) split = train_config.get('split', 0.2) model_name = train_config.get('model', 'noname') train_dir = train_config['train'] test_dir = train_config['test'] model_dir = os.path.join("./models", "{}".format(model_name)) model_file = open(os.path.join(model_dir, "model_log.txt"), 'w') def print_f(val): model_file.write("{}\n".format(val)) print(val) model = CNN_RNN_Sequential_model(print_f) model.compile( optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy', 'mae'], ) # load training set print('Loading data') X, y, y_raws, label_set = load_dataset(train_dir) X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=split, random_state=42) model.fit(X_train, y_train, epochs=epoch, batch_size=batch_size, verbose=1, validation_data=[X_val, y_val], callbacks=[ ProgbarLogger(), ReduceLROnPlateau(), TensorBoard(log_dir=model_dir, histogram_freq=1, batch_size=100), CSVLogger(filename=os.path.join(model_dir, "logs.log")) ]) after_train(model, model_file, model_dir, train_config, label_set, model_name) model_file.close()
def __init__( self, name, n_input=1161, # n_frames (9) * narrow_band_window (256/2 + 1) n_output=128, # high_band_window (512/4 + 1) n_hidden=2048, n_layers=3, fe=None): self.n_input = n_input self.n_output = n_output self.n_hidden = n_hidden self.n_layers = n_layers self.fe = fe self.name = name self.model = Sequential() self.model.add(Dense(self.n_hidden, input_shape=(self.n_input, ))) self.model.add(Activation('relu')) for idx in range(self.n_layers): self.model.add(Dense(n_hidden)) self.model.add(Activation('relu')) self.model.add(Dense(self.n_output)) self.model.compile(optimizer='adam', loss='mse') #, metrics=[]) self.progbar_logger = ProgbarLogger() self.psnr_metrics = PsnrMetrics(self) self.model_checkpoint = ModelCheckpoint(self.name, monitor='val_loss', verbose=1, save_best_only=True, mode='min') self.early_stopping = EarlyStopping(monitor='val_loss', min_delta=0.0, patience=8, mode='min')
def train_model(self, batch_size=32, nb_epoch=50, load_data=False, old_weight_path=''): print("start training model...") if load_data: train_data, train_labels, valid_data, valid_labels = self.load_data( ) else: train_data, train_labels, valid_data, valid_labels = self.prepare_train_data( ) model = self.baseModel() if old_weight_path != '': print("load last epoch model to continue train") model.load_weights(old_weight_path) model.fit(train_data, train_labels, batch_size=batch_size, epochs=nb_epoch, validation_data=(valid_data, valid_labels), callbacks=[ ModelCheckpoint( "output/weights.{epoch:02d}-{val_loss:.2f}.hdf5", monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='min', period=2), ProgbarLogger() ]) return model
def _prepare_callbacks(self, callbacks: List[Callback], val_ins: List[numpy.array], epochs: int, batch_size: int, num_train_samples: int, callback_metrics: List[str], do_validation: bool, verbose: int): """ Sets up Keras callbacks to perform various monitoring functions during training. """ self.history = History() # pylint: disable=attribute-defined-outside-init callbacks = [BaseLogger()] + (callbacks or []) + [self.history] if verbose: callbacks += [ProgbarLogger()] callbacks = CallbackList(callbacks) # it's possible to callback a different model than self # (used by Sequential models). if hasattr(self, 'callback_model') and self.callback_model: callback_model = self.callback_model else: callback_model = self # pylint: disable=redefined-variable-type callbacks.set_model(callback_model) callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, 'samples': num_train_samples, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics or [], }) callbacks.on_train_begin() callback_model.stop_training = False for cbk in callbacks: cbk.validation_data = val_ins return callbacks, callback_model
from keras.callbacks import EarlyStopping, ProgbarLogger, ModelCheckpoint from keras.callbacks import LearningRateScheduler, CSVLogger, TensorBoard from keras.optimizers import RMSprop from keras.models import load_model import tools from facial_recognizer import facial_recognizer from config import config as c os.makedirs(c.path_to_summaries, exist_ok=True) os.makedirs(c.path_to_models, exist_ok=True) call_backs = [ # EarlyStopping('val_acc', min_delta=1e-5, patience=20), ProgbarLogger('steps'), ModelCheckpoint(c.path_to_models + '/model_OE', save_best_only=True), TensorBoard(c.path_to_summaries) ] model = facial_recognizer(c) model.compile( # optimizer=SGD(lr=1e-3, momentum=0.9), optimizer='adam', loss={ 'out_au': 'binary_crossentropy', 'out_emotion': 'categorical_crossentropy' }) train_gen = tools.fake_generator(c) for _ in range(10):
target_size=(img_height, img_width), batch_size=batch_size, class_mode="categorical", seed=seed) temp2 = ig2.flow_from_directory(Xs, target_size=(img_height, img_width), batch_size=batch_size, class_mode="categorical", seed=seed) for batch in izip(temp1, temp2): yield [batch[0][0], batch[1][0]], [batch[0][1]] # Save the model according to the conditions progbar = ProgbarLogger(count_mode='steps') checkpoint = ModelCheckpoint("rgbd.{epoch:02d}-{val_loss:.2f}.hdf5", monitor='val_acc', verbose=1, save_best_only=True, save_weights_only=False, mode='auto', period=1) early = EarlyStopping(monitor='val_acc', min_delta=0, patience=1, verbose=1, mode='auto') # Train the model model1.fit_generator(custom_iterator(train_data_dir_nir, train_data_dir),
def main(args): try: opts, args = getopt.getopt(args, "c:s", ["config="]) except getopt.GetoptError: print('usage: -c config.json') sys.exit(2) start_from_model = False for opt, arg in opts: if opt in ("-c", "--config"): config_fname = os.path.join('configurations', arg) elif opt == '-s': start_from_model = True if start_from_model: filemode = 'a' else: filemode = 'w' log_path = 'logging/vae_nlg_{}'.format(int(round(time.time() * 1000))) os.mkdir(log_path) logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO, filename='{}/evolution.log'.format(log_path), filemode=filemode) with open(config_fname, 'r') as json_data: config_data = json.load(json_data) batch_size = config_data['batch_size'] epochs = config_data['nb_epochs'] discriminator_iterations = config_data['discriminator_iterations'] tweets_path = config_data['tweets_path'] vocab_path = config_data['vocab_path'] vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb')) #== == == == == == = # Load all the Data #== == == == == == = delimiter = '' noutputs = 11 logging.info('Load Training Data') train_input, train_output, train_weights, train_lex = load_text_gen_data( join(tweets_path, 'trainset.csv'), config_data, vocab, noutputs, word_based=False) logging.info('Load Validation Data') valid_input, valid_output, _, valid_lex = load_text_gen_data( join(tweets_path, 'devset.csv'), config_data, vocab, noutputs, word_based=False) logging.info('Load Output Validation Data') valid_dev_input, valid_dev_output, _, valid_dev_lex = load_text_gen_data( join(tweets_path, 'devset_reduced.csv'), config_data, vocab, noutputs, random_output=False, word_based=False) step = K.variable(1., name='step_varialbe') steps_per_epoch = ceil(train_output[0].shape[0] / config_data['batch_size']) # == == == == == == == == == == = # Define and load the CNN model # == == == == == == == == == == = vae_model_train, vae_model_test, vae_vanilla_train_model, vae_vanilla_test_model, discriminator_model, decoder_test, discriminator = get_vae_gan_model( config_data, vocab, step) with open(os.path.join(log_path, 'models.txt'), 'wt') as fh: fh.write('VAE Model Train\n') fh.write('---------\n') vae_model_train.summary(print_fn=lambda x: fh.write(x + '\n')) fh.write('VAE Model Test\n') fh.write('--------------\n') vae_model_test.summary(print_fn=lambda x: fh.write(x + '\n')) fh.write('VAE Model Pretrain\n') fh.write('---------------------------\n') vae_vanilla_train_model.summary( print_fn=lambda x: fh.write(x + '\n')) fh.write('VAE Model Pretrain Test\n') fh.write('---------------------------\n') vae_vanilla_test_model.summary( print_fn=lambda x: fh.write(x + '\n')) fh.write('Decoder Test\n') fh.write('-------------------\n') decoder_test.summary(print_fn=lambda x: fh.write(x + '\n')) fh.write('Discriminator Models\n') fh.write('-------------------\n') discriminator_model.summary(print_fn=lambda x: fh.write(x + '\n')) terminate_on_nan = TerminateOnNaN() output_callback = LexOutputCallback( vae_vanilla_test_model, valid_dev_input, valid_dev_lex, 1, vocab, delimiter, fname='{}/test_output'.format(log_path)) #output_callback_full = LexOutputCallback(vae_vanilla_test_model, valid_dev_input, valid_dev_lex, 1, vocab, delimiter, fname='{}/test_output'.format(log_path)) # # vae_vanilla_train_model.fit_generator( # generator=generate_data_stream(config_data['pretrain_path'], config_data, vocab, config_data['batch_size'], noutputs=3), # steps_per_epoch=steps_per_epoch, # epochs=ceil(config_data['pretrain_samples']/config_data['pretrain_samples_per_epoch']), # callbacks=[output_callback, terminate_on_nan], # validation_data=(valid_input, valid_output[:3]), # ) vae_vanilla_train_model.fit( x=train_input, y=train_output[:2], epochs=config_data['pretrain_epochs'], batch_size=batch_size, validation_data=(valid_input, valid_output[:2]), sample_weight=train_weights[:2], callbacks=[output_callback, terminate_on_nan]) terminate_on_nan = TerminateOnNaN() model_checkpoint = ModelCheckpoint( 'models/vae_model/weights.{epoch:02d}.hdf5', period=10, save_weights_only=True) out_labels = [ 'enc_' + s for s in vae_model_train._get_deduped_metrics_names() ] out_labels += [ 'dis_' + s for s in discriminator_model._get_deduped_metrics_names() ] callback_metrics = out_labels + ['val_' + n for n in out_labels] tensorboard = TensorBoard(log_dir='logging/tensorboard', histogram_freq=0, write_grads=True, write_images=True) step_callback = StepCallback(step, steps_per_epoch) output_callback = LexOutputCallback( vae_vanilla_test_model, valid_dev_input, valid_dev_lex, 1, vocab, delimiter, fname='{}/test_output'.format(log_path)) output_callback_full = LexOutputCallback( vae_vanilla_test_model, valid_input, valid_lex, 5, vocab, delimiter, fname='{}/test_valid_output'.format(log_path)) callbacks = CallbackList([ BaseLogger(), ProgbarLogger(count_mode='steps'), step_callback, tensorboard, output_callback, output_callback_full, model_checkpoint, terminate_on_nan ]) callbacks.set_model(vae_model_train) callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, 'steps': steps_per_epoch, 'verbose': True, 'do_validation': True, 'metrics': callback_metrics or [], }) callbacks.on_train_begin() initial_epoch = 0 num_train_samples = train_input[0].shape[0] index_array = np.arange(num_train_samples) steps = 0 epoch = initial_epoch while epoch < epochs: epoch_logs = {} callbacks.on_epoch_begin(epoch) index_array = _batch_shuffle(index_array, batch_size) steps_done = 0 batches = _make_batches(num_train_samples, batch_size) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_logs = {} batch_ids = index_array[batch_start:batch_end] X = _slice_arrays(train_input, batch_ids) y = _slice_arrays(train_output, batch_ids) sample_weights = _slice_arrays(train_weights, batch_ids) batch_logs['batch'] = batch_index batch_logs['size'] = batch_size callbacks.on_batch_begin(batch_index, batch_logs) set_trainability(discriminator, trainable=False) enc_outs = vae_model_train.train_on_batch( x=X, y=y, sample_weight=sample_weights) set_trainability(discriminator, trainable=True) list_disc_loss_real = [] if steps < 25 or steps % 500 == 0: disc_iterations = 25 else: disc_iterations = discriminator_iterations for disc_it in range(disc_iterations): real_idx = np.random.choice(train_input[0].shape[0], len(batch_ids), replace=False) disX_train = train_input[-1][ real_idx] #take input 8 as train input and the rest as targets disy_train = [x[real_idx] for x in train_input[:8] ] #take input 1-7 as targets #train on real data dis_outs_real = discriminator_model.train_on_batch( disX_train, disy_train) list_disc_loss_real.append(dis_outs_real) loss_d_real = np.mean(list_disc_loss_real, axis=0) outs = np.concatenate((enc_outs, loss_d_real)) for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) epoch_logs = {} batch_index += 1 steps_done += 1 steps += 1 # Epoch finished. if steps_done >= steps_per_epoch: valid_len = valid_output[0].shape[0] enc_val_outs = vae_model_train.evaluate(valid_input, valid_output, verbose=False) dis_val_outs = discriminator_model.evaluate( valid_input[-1], valid_input[:8], verbose=False) val_outs = enc_val_outs + dis_val_outs #val_outs = full_model.evaluate(valid_input, valid_output, verbose=False) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) epoch += 1 callbacks.on_train_end()
model_metrics = [ 'accuracy', ] model = build_model() model.compile(optimizer='adam', loss=model_losses, loss_weights=model_loss_weights, metrics=model_metrics) model.summary() if __debug__: print('Fitting...') callbacks = [ProgbarLogger()] if args.early_stopping: callbacks.append(EarlyStopping(monitor='val_loss', patience=5)) model.fit(X_train, y_train, validation_data=(X_dev, y_dev), epochs=args.epochs, batch_size=args.bsize, callbacks=callbacks, verbose=args.verbose) if __debug__: print(args) print('Evaluating...')
model = Model(input=sequence_one_hot, output=dense3) model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy']) # Generates .png image with the model structure plot_model(model, show_shapes=True, to_file='model_structure_new.png') step_size_callback = LearningRateScheduler(step_size_decay) # Callbacks to save and retreive the best weight configurations found during training phase all_callbacks = [ step_size_callback, FlushCallback(), ModelCheckpoint('model' + '.hdf5', save_best_only=True, verbose=1), ProgbarLogger(count_mode='samples') ] #--------------- TRAINING ------------------# # No vaidation split was given in the paper, so I've set it to 0.2 arbitarily hist = model.fit(x, y, batch_size=mini_batch_size, nb_epoch=max_epochs, verbose=2, validation_split=0.2, callbacks=all_callbacks) # Prints to console, the summary of losses etc through all the epochs ch = input("\nPrint training loss and error history? (y/n)") if ch == 'y':
def fit_dataset(self, dataset, steps_per_epoch=None, batch_size=32, epochs=1, verbose=1, callbacks=None, on_sample=None, on_scores=None): """Train the model on the given dataset for a given number of epochs. Arguments --------- dataset: Instance of `BaseDataset` that provides the data to train on. steps_per_epoch: int or None, number of gradient updates before considering an epoch has passed. If None it is set to be `len(dataset.train_data) / batch_size`. batch_size: int, number of samples per gradient update epochs: int, number of times to iterate `steps_per_epoch` times verbose: {0, >0}, whether to employ the progress bar Keras callback or not callbacks: list of Keras callbacks to be called during training on_sample: callable that accepts the sampler, idxs, w, scores on_scores: callable that accepts the sampler and scores """ try: if len(dataset.train_data) < batch_size: raise ValueError(("The model cannot be trained with " "batch_size > training set")) except RuntimeError as e: assert "no size" in str(e) # Set steps_per_epoch properly if steps_per_epoch is None: steps_per_epoch = len(dataset.train_data) // batch_size # Create the callbacks list self.history = History() callbacks = [BaseLogger()] + (callbacks or []) + [self.history] if verbose > 0: callbacks += [ProgbarLogger(count_mode="steps")] callbacks = CallbackList(callbacks) #TODO: Should we be making it possible to call back a different model # than self.model.model? callbacks.set_model(self.model.model) callbacks.set_params({ "epochs": epochs, "steps": steps_per_epoch, "verbose": verbose, "do_validation": len(dataset.test_data) > 0, "metrics": self._get_metric_names() + ["val_" + n for n in self._get_metric_names()] }) # Create the sampler sampler = self.sampler(dataset, batch_size, steps_per_epoch, epochs) # Start the training loop epoch = 0 self.model.model.stop_training = False callbacks.on_train_begin() while epoch < epochs: callbacks.on_epoch_begin(epoch) for step in range(steps_per_epoch): batch_logs = {"batch": step, "size": batch_size} callbacks.on_batch_begin(step, batch_logs) # Importance sampling is done here idxs, (x, y), w = sampler.sample(batch_size) # Train on the sampled data loss, metrics, scores = self.model.train_batch(x, y, w) # Update the sampler sampler.update(idxs, scores) values = map(lambda x: x.mean(), [loss] + metrics) for l, o in zip(self._get_metric_names(), values): batch_logs[l] = o callbacks.on_batch_end(step, batch_logs) if on_scores is not None: on_scores(sampler, self._latest_scores) if on_sample is not None: on_sample(sampler, self._latest_sample_event["idxs"], self._latest_sample_event["w"], self._latest_sample_event["predicted_scores"]) if self.model.model.stop_training: break # Evaluate now that an epoch passed epoch_logs = {} if len(dataset.test_data) > 0: val = self.model.evaluate(*dataset.test_data[:], batch_size=batch_size) epoch_logs = { "val_" + l: o for l, o in zip(self._get_metric_names(), val) } callbacks.on_epoch_end(epoch, epoch_logs) if self.model.model.stop_training: break epoch += 1 callbacks.on_train_end() return self.history
def train(self, epochs, steps_per_epoch, initial_epoch=0, end_of_epoch_callback=None, verbose=1): epoch = initial_epoch logger = ProgbarLogger(count_mode='steps') logger.set_params({ 'epochs': epochs, 'steps': steps_per_epoch, 'verbose': verbose, 'metrics': self.metric_names }) logger.on_train_begin() while epoch < epochs: step = 0 batch = 0 logger.on_epoch_begin(epoch) while step < steps_per_epoch: self.batch_logs['batch'] = batch logger.on_batch_begin(batch, self.batch_logs) for i in range(len(self.models)): x, y = self.output_generators[i].__next__() if x is None: x, y = self.output_generators[i].__next__() outs = self.models[i].train_on_batch(x, y) if not isinstance(outs, list): outs = [outs] if self.print_full_losses: for l, o in zip(self.metric_names, outs): self.batch_logs[l] = o else: self.batch_logs[self.metric_names[i]] = outs[0] logger.on_batch_end(batch, self.batch_logs) step += 1 batch += 1 logger.on_epoch_end(epoch) if end_of_epoch_callback is not None: end_of_epoch_callback(epoch) epoch += 1
def main(args): try: opts, args = getopt.getopt(args, "c:s", ["config="]) except getopt.GetoptError: print('usage: -c config.json') sys.exit(2) start_from_model = False for opt, arg in opts: if opt in ("-c", "--config"): config_fname = os.path.join('configurations', arg) elif opt == '-s': start_from_model = True if start_from_model: filemode = 'a' else: filemode = 'w' log_path = 'logging/vae_nlg_{}'.format(int(round(time.time() * 1000))) os.mkdir(log_path) logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO, filename='{}/evolution.log'.format(log_path), filemode=filemode) with open(config_fname, 'r') as json_data: config_data = json.load(json_data) batch_size = config_data['batch_size'] epochs = config_data['nb_epochs'] discriminator_iterations = config_data['discriminator_iterations'] tweets_path = config_data['tweets_path'] vocab_path = config_data['vocab_path'] vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb')) #== == == == == == = # Load all the Data #== == == == == == = noutputs = 5 logging.info('Load Training Data') train_input, train_output = load_text_pairs( join(tweets_path, 'training_set.tsv'), config_data, vocab, noutputs) logging.info('Load Validation Data') valid_input, valid_output = load_text_pairs( join(tweets_path, 'vaild_set.tsv'), config_data, vocab, noutputs) logging.info('Load Output Validation Data') valid_dev_input, valid_dev_output = load_text_pairs( join(tweets_path, 'test_set.tsv'), config_data, vocab, noutputs) #train_input = [x[:1213] for x in train_input] #train_output = [x[:1213] for x in train_output] noise_valid_input = np.zeros(shape=(valid_input[0].shape[0], config_data['z_size'])) step = K.variable(1.) steps_per_epoch = ceil(train_output[0].shape[0] / config_data['batch_size']) # == == == == == == == == == == = # Define and load the CNN model # == == == == == == == == == == = vae_model, vae_model_test, decoder_discr_model, decoder_test_model, discriminator_model, discriminator = get_vae_gan_model( config_data, vocab, step) with open(os.path.join(log_path, 'models.txt'), 'wt') as fh: fh.write('VAE Model\n') fh.write('---------\n') vae_model.summary(print_fn=lambda x: fh.write(x + '\n')) fh.write('VAE Model Test\n') fh.write('--------------\n') vae_model_test.summary(print_fn=lambda x: fh.write(x + '\n')) fh.write('Decoder Discriminator Model\n') fh.write('---------------------------\n') decoder_discr_model.summary(print_fn=lambda x: fh.write(x + '\n')) fh.write('Decoder Test Model\n') fh.write('---------------------------\n') decoder_test_model.summary(print_fn=lambda x: fh.write(x + '\n')) fh.write('Discriminator Model\n') fh.write('-------------------\n') discriminator_model.summary(print_fn=lambda x: fh.write(x + '\n')) terminate_on_nan = TerminateOnNaN() model_checkpoint = ModelCheckpoint( 'models/vae_model/weights.{epoch:02d}.hdf5', period=10, save_weights_only=True) enc_out_labels = [ 'enc_' + s for s in vae_model._get_deduped_metrics_names() ] dec_out_labels = [ 'dec_' + s for s in decoder_discr_model._get_deduped_metrics_names() ] dis_out_labels = [ 'dis_' + s for s in discriminator_model._get_deduped_metrics_names() ] out_labels = enc_out_labels + dec_out_labels + [ 'dis_real', 'dis_gen', 'dis_noise' ] #out_labels = full_model._get_deduped_metrics_names() callback_metrics = out_labels + ['val_' + n for n in out_labels] step_callback = StepCallback(step, steps_per_epoch) output_callback = GANOutputCallback( vae_model_test, valid_dev_input[0], 1, vocab, '', fname='{}/test_output'.format(log_path)) callbacks = CallbackList([ BaseLogger(), ProgbarLogger(count_mode='steps'), step_callback, output_callback, model_checkpoint, terminate_on_nan ]) callbacks.set_model(vae_model_test) callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, 'steps': steps_per_epoch, 'verbose': True, 'do_validation': True, 'metrics': callback_metrics or [], }) callbacks.on_train_begin() initial_epoch = 0 num_train_samples = train_input[0].shape[0] index_array = np.arange(num_train_samples) steps = 0 epoch = initial_epoch while epoch < epochs: epoch_logs = {} callbacks.on_epoch_begin(epoch) index_array = _batch_shuffle(index_array, batch_size) steps_done = 0 batches = _make_batches(num_train_samples, batch_size) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_logs = {} batch_ids = index_array[batch_start:batch_end] X = _slice_arrays(train_input, batch_ids) y = _slice_arrays(train_output, batch_ids) batch_logs['batch'] = batch_index batch_logs['size'] = batch_size callbacks.on_batch_begin(batch_index, batch_logs) set_trainability(discriminator, trainable=False) enc_outs = vae_model.train_on_batch(x=X, y=y[:3]) set_trainability(discriminator, trainable=True) list_disc_loss_real = [] list_disc_loss_gen = [] list_disc_loss_noise = [] if steps < 25 or steps % 500 == 0: disc_iterations = 100 else: disc_iterations = discriminator_iterations noise_input = np.zeros(shape=(len(batch_ids), config_data['z_size'])) for disc_it in range(disc_iterations): #clip_weights(discriminator) real_idx = np.random.choice(train_input[0].shape[0], len(batch_ids), replace=False) train_real_batch = [x[real_idx] for x in train_input] #train on real data x_fake = vae_model_test.predict_on_batch( x=train_real_batch[0]) x_noise_fake = decoder_test_model.predict_on_batch( x=noise_input) train_input_discr = np.concatenate( (train_real_batch[0], train_real_batch[0], train_real_batch[0])) train_output_discr = np.concatenate( (train_real_batch[1], x_fake, x_noise_fake)) labels = np.asarray( len(batch_ids) * [1] + 2 * len(batch_ids) * [-1]) index_array_discr = np.arange(len(labels)) np.random.shuffle(index_array_discr) discr_batch = [ train_input_discr[index_array_discr], train_output_discr[index_array_discr] ] discr_batch_labels = labels[index_array_discr] dis_outs_real = discriminator_model.train_on_batch( discr_batch, discr_batch_labels) #dis_outs_real = discriminator_model.train_on_batch(train_real_batch, -np.ones(shape=(len(batch_ids), 1))) #dis_outs_gen = discriminator_model.train_on_batch([train_real_batch[0], x_fake], np.ones(shape=(len(batch_ids), 1))) #dis_outs_gen_noise = discriminator_model.train_on_batch([train_real_batch[0], x_noise_fake], np.ones(shape=(len(batch_ids), 1))) list_disc_loss_real.append(dis_outs_real) #list_disc_loss_gen.append(dis_outs_gen) #list_disc_loss_noise.append(dis_outs_gen_noise) loss_d_real = -np.mean(list_disc_loss_real) loss_d_gen = np.mean(list_disc_loss_gen) loss_d_noise = np.mean(list_disc_loss_noise) set_trainability(discriminator, trainable=False) decoder_discr_input = [X[0], noise_input] dec_outs = decoder_discr_model.train_on_batch( x=decoder_discr_input, y=-np.ones(shape=(len(batch_ids), 1))) outs = enc_outs + [dec_outs] + [loss_d_real] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) epoch_logs = {} batch_index += 1 steps_done += 1 steps += 1 # Epoch finished. if steps_done >= steps_per_epoch: valid_len = valid_output[0].shape[0] enc_val_outs = vae_model.evaluate(valid_input, valid_output[:3], verbose=False) dec_val_outs = decoder_discr_model.evaluate( [valid_input[0], noise_valid_input], -np.ones(shape=(valid_len, 1)), verbose=False) dis_val_outs = discriminator_model.evaluate( valid_input, -np.ones(shape=(valid_len, 1)), verbose=False) val_outs = enc_val_outs + [dec_val_outs] + [dis_val_outs] #val_outs = full_model.evaluate(valid_input, valid_output, verbose=False) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) epoch += 1 callbacks.on_train_end()
from keras.layers import Dense from keras.models import Sequential from keras.callbacks import ProgbarLogger import numpy as np import matplotlib.pyplot as plt n_samples = 500 def powerTwo(x): return x*x np.random.seed(10) X = np.random.uniform(-2, 2, n_samples) Y = powerTwo(X) # tarin MLP model = Sequential() model.add(Dense(100, input_dim=1, activation='relu')) model.add(Dense(1)) model.compile(loss='mae', optimizer='rmsprop', metrics=['accuracy']) model.fit(X, Y, nb_epoch=20, batch_size=2, verbose=1, validation_split = 0.1, callbacks =[ProgbarLogger()]) #try new samples new_samples = 300 new_X = np.random.uniform(-2, 2, new_samples) new_Y = powerTwo(new_X) pred_Y = model.predict(new_X) plt.plot(new_X,new_Y,'go',label="X*X") plt.plot(new_X,pred_Y,'yo',label="PredictedX*X") plt.show() print("green shows original X^2 and yellow shows predicted X^2")
def __init__(self, verbose=1): ProgbarLogger.__init__(self) self.verbose2 = verbose
# genenor.next() # genenor.next() # exit() name_experiment = params_train['name_experiment'] # exit() batch_size = params_train['batch_size'] mkdir(name_experiment) filepath = './' + name_experiment + '/' + name_experiment + '_weights_{epoch:02d}_{loss:.4f}.hdf5' checkpointer = ModelCheckpoint(filepath=filepath, verbose=1, monitor='loss', mode='min', save_best_only=True, save_weights_only=True) progbarLogger = ProgbarLogger() model = get_model(params_transform, params_train) model = initialize_model_from_vgg(model) model.load_weights(filepath='./exp1/exp1_weights_00_1849.1228.hdf5') """ COCO数据集中 train2014 : 82783 个样本 val2014 : 40504 个样本 数据集生成的时候,将val2014中的前2644个样本标记位 'isValidation = 1' 所以用于训练的样本数为 82783+40504-2644 = 120643 这里设置,我们训练model的总样本数为 6000000 """ model.fit_generator( generator=genenor, steps_per_epoch=params_train['steps_per_epoch'],
def _train_by_batch(self): # batch finite generator should be loaded within epoch loop logger.info('Start training by batch') self.validation_xy = self.load_data('val', feed_mode='all') do_validation = bool(self.validation_xy) # prepare display labels in tensorboard out_labels = self.model._get_deduped_metrics_names() callback_metrics = out_labels + ['val_' + n for n in out_labels] # prepare callbacks self.model.history = History() callbacks = [BaseLogger()] + (self.callbacks or []) + [self.model.history] # callbacks = (self.callbacks or []) + [self.model.history] if self.verbose: callbacks += [ProgbarLogger(count_mode='samples')] callbacks = CallbackList(callbacks) # it's possible to callback a different model than this model if hasattr(self.model, 'callback_model') and self.model.callback_model: callback_model = self.model.callback_model else: callback_model = self.model callbacks.set_model(callback_model) callbacks.set_params({ 'epochs': self.epochs, 'samples': self.data.nb_train, 'verbose': self.verbose, 'do_validation': do_validation, 'metrics': callback_metrics, }) callbacks.on_train_begin() for epoch in range(self.epochs): start_e = time() callbacks.on_epoch_begin(epoch) xy_gen = self.load_data('train', feed_mode='batch') logger.info('New training epoch') for batch_index, (x, y) in enumerate(xy_gen): # build batch logs batch_logs = {} if isinstance(x, list): batch_size = x[0].shape[0] elif isinstance(x, dict): batch_size = list(x.values())[0].shape[0] else: batch_size = x.shape[0] batch_logs['batch'] = batch_index batch_logs['size'] = batch_size callbacks.on_batch_begin(batch_index, batch_logs) outs = self.model.train_on_batch(x, y) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) if (batch_index + 1) % 1000 == 0 and do_validation: val_outs = self.model.evaluate(*self.validation_xy, batch_size=81920, verbose=0) batch_logs = {} if not isinstance(val_outs, list): val_outs = [val_outs] for l, o in zip(out_labels, val_outs): batch_logs['val_' + l] = o print(' - Eval inside: %.6f' % val_outs[0]) for cb in self.callbacks: if cb.__class__ == tensorBoard: cb.on_batch_end(batch_index, batch_logs, count=False) epoch_logs = {} if do_validation: val_outs = self.model.evaluate(*self.validation_xy, batch_size=81920, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_batch_end(epoch, epoch_logs) callbacks.on_epoch_end(epoch, epoch_logs) elapsed_e = timedelta(seconds=int(time() - start_e)) self.send_metric('elapsed_per_epoch', elapsed_e) if not self.no_save and do_validation and (epoch != self.epochs - 1): self.model.save( 'results/trained_models/%s_ctr_model_%.4f_epoch_%d.h5' % (self.sess_id, val_outs[0], epoch)) callbacks.on_train_end() return self.model.history
def predict_image(version, image_path, batch_size, overlap, data_format=None): def current_time_millis(): return int(round(time.time() * 1000)) def offset(size, diff, overlap): return math.floor(diff / math.ceil(diff / (size * (1 - overlap)))) def map_c(i, j, b, l): return int(((i * b) + j) / l) def map_r(i, j, b, l): return ((i * b) + j) % l if data_format is None: data_format = K.image_data_format() if data_format not in {'channels_first', 'channels_last'}: raise ValueError('Unknown data_format:', data_format) path = version.model_file.name print(_('Loading model "%(path)s".') % {'path': path}) model = load_model(os.path.join(settings.MEDIA_ROOT, path)) if len(model.inputs) != 1: raise RuntimeError('Models with more than one input are not' ' supported at the moment.') inputs = [] for i in range(len(model.inputs)): name = model.inputs[i].name pos = min( name.index('/') if '/' in name else len(name), name.index(':') if ':' in name else len(name)) name = name[:pos] inputs.append({'shape': model.inputs[i].shape.as_list(), 'name': name}) if data_format == 'channels_first': inputs[i]['grayscale'] = inputs[i]['shape'][1] == 1 inputs[i]['r'] = inputs[i]['shape'][2] inputs[i]['c'] = inputs[i]['shape'][3] elif data_format == 'channels_last': inputs[i]['r'] = inputs[i]['shape'][1] inputs[i]['c'] = inputs[i]['shape'][2] inputs[i]['grayscale'] = inputs[i]['shape'][3] == 1 inputs[i]['img'] = img_to_array( load_img(image_path, inputs[i]['grayscale'])) inputs[i]['img'] *= 1. / 255 if data_format == 'channels_first': inputs[i]['img_r'] = inputs[i]['img'].shape[1] inputs[i]['img_c'] = inputs[i]['img'].shape[2] elif data_format == 'channels_last': inputs[i]['img_r'] = inputs[i]['img'].shape[0] inputs[i]['img_c'] = inputs[i]['img'].shape[1] inputs[i]['diff_r'] = inputs[i]['img_r'] - inputs[i]['r'] inputs[i]['diff_c'] = inputs[i]['img_c'] - inputs[i]['c'] inputs[i]['offset_r'] = offset(inputs[i]['r'], inputs[i]['diff_r'], overlap) inputs[i]['offset_c'] = offset(inputs[i]['c'], inputs[i]['diff_c'], overlap) inputs[i]['nb_r'] = math.ceil( inputs[i]['diff_r'] / inputs[i]['offset_r']) + 1 inputs[i]['nb_c'] = math.ceil( inputs[i]['diff_c'] / inputs[i]['offset_c']) + 1 inputs = inputs[0] N = inputs['nb_r'] * inputs['nb_c'] steps = math.ceil(N / batch_size) metrics = [] outputs = [] for i in range(len(model.outputs)): tshape = model.outputs[i].shape.as_list() name = model.outputs[i].name pos = min( name.index('/') if '/' in name else len(name), name.index(':') if ':' in name else len(name)) name = name[:pos] activation = model.get_layer(name).activation.__name__.lower() outputs.append({'name': name, 'shape': tshape}) if len(tshape) == 2: if activation == 'softmax': outputs[i]['t'] = 'class' else: outputs[i]['t'] = 'multi' nb_classes = tshape[1] if nb_classes is None: nb_classes = model.get_layer(name).output_shape[1] nb_classes = int(nb_classes) metrics += ['%s:%s' % (name, i) for i in range(nb_classes)] if data_format == 'channels_first': shape = (nb_classes, inputs['nb_r'], inputs['nb_c']) elif data_format == 'channels_last': shape = (inputs['nb_r'], inputs['nb_c'], nb_classes) elif len(tshape) == 4: if activation == 'softmax': outputs[i]['t'] = 'class' else: outputs[i]['t'] = 'img' shape = (inputs['nb_r'], inputs['nb_c']) + tuple(tshape[1:]) outputs[i]['p'] = np.zeros(shape) history = History() callbacks = CallbackList([BaseLogger(), history, ProgbarLogger()]) callbacks.set_model(model) callbacks.set_params({ 'batch_size': batch_size, 'epochs': 1, 'steps': steps, 'samples': N, 'verbose': 1, 'do_validation': False, 'metrics': metrics, }) callbacks.on_train_begin() callbacks.on_epoch_begin(0) start_time = current_time_millis() for b in range(steps): current_index = (b * batch_size) % N if N >= current_index + batch_size: current_batch_size = batch_size else: current_batch_size = N - current_index batch_logs = {'batch': b, 'size': current_batch_size} for metric in metrics: batch_logs[metric] = 0 callbacks.on_batch_begin(b, batch_logs) bX = np.zeros((current_batch_size, ) + tuple(inputs['shape'][1:])) for j in range(current_batch_size): idx_r = map_r(b, j, batch_size, inputs['nb_r']) idx_c = map_c(b, j, batch_size, inputs['nb_r']) top = min(idx_r * inputs['offset_r'], inputs['img_r'] - inputs['r']) bottom = min(idx_r * inputs['offset_r'] + inputs['r'], inputs['img_r']) left = min(idx_c * inputs['offset_c'], inputs['img_c'] - inputs['c']) right = min(idx_c * inputs['offset_c'] + inputs['c'], inputs['img_c']) if data_format == 'channels_first': bX[j] = inputs['img'][:, top:bottom, left:right] elif data_format == 'channels_last': bX[j] = inputs['img'][top:bottom, left:right, :] p = model.predict_on_batch(bX) if type(p) != list: p = [p] for j in range(current_batch_size): for i in range(len(outputs)): idx_r = map_r(b, j, batch_size, inputs['nb_r']) idx_c = map_c(b, j, batch_size, inputs['nb_r']) if len(outputs[i]['p'].shape) == 3: if data_format == 'channels_first': outputs[i]['p'][:, idx_r, idx_c] = p[i][j] elif data_format == 'channels_last': outputs[i]['p'][idx_r, idx_c, :] = p[i][j] metric = metrics[p[i][j].argmax()] batch_logs[metric] += 1. / current_batch_size elif len(outputs[i]['p'].shape) == 5: outputs[i]['p'][idx_r, idx_c, :, :, :] = p[i][j] callbacks.on_batch_end(b, batch_logs) runtime = (current_time_millis() - start_time) / 1000 callbacks.on_epoch_end(0, {'runtime': runtime}) callbacks.on_train_end() for i in range(len(outputs)): if len(outputs[i]['shape']) == 2: if data_format == 'channels_first': shape = (outputs[i]['p'].shape[0], inputs['img_r'], inputs['img_c']) elif data_format == 'channels_last': shape = (inputs['img_r'], inputs['img_c'], outputs[i]['p'].shape[2]) elif len(tshape) == 4: if data_format == 'channels_first': shape = (outputs[i]['p'].shape[2], inputs['img_r'], inputs['img_c']) elif data_format == 'channels_last': shape = (inputs['img_r'], inputs['img_c'], outputs[i]['p'].shape[4]) count = np.zeros(shape) outputs[i]['img'] = np.zeros(shape) if len(outputs[i]['p'].shape) == 3: if data_format == 'channels_first': nb_rows = outputs[i]['p'].shape[1] nb_cols = outputs[i]['p'].shape[2] elif data_format == 'channels_last': nb_rows = outputs[i]['p'].shape[0] nb_cols = outputs[i]['p'].shape[1] elif len(outputs[i]['p'].shape) == 5: nb_rows = outputs[i]['p'].shape[0] nb_cols = outputs[i]['p'].shape[1] for j in range(nb_rows): for k in range(nb_cols): top = min(j * inputs['offset_r'], inputs['img_r'] - inputs['r']) bottom = min(j * inputs['offset_r'] + inputs['r'], inputs['img_r']) left = min(k * inputs['offset_c'], inputs['img_c'] - inputs['c']) right = min(k * inputs['offset_c'] + inputs['c'], inputs['img_c']) if data_format == 'channels_first': outputs[i]['img'][:, top:bottom, left:right] += \ outputs[i]['p'][:, j, k] count[:, top:bottom, left:right] += 1 elif data_format == 'channels_last': outputs[i]['img'][top:bottom, left:right, :] += \ outputs[i]['p'][j, k, :] count[top:bottom, left:right, :] += 1 outputs[i]['img'] /= count del outputs[i]['p'] del outputs[i]['shape'] return history.history, outputs
def add_progressbar_callback(self): self.callbacks.append(ProgbarLogger())
def on_train_begin(self, logs=None): ProgbarLogger.on_train_begin(self, logs) self.verbose = self.verbose2
def main(args): try: opts, args = getopt.getopt(args, "c:s", ["config="]) except getopt.GetoptError: print('usage: -c config.json') sys.exit(2) start_from_model = False for opt, arg in opts: if opt in ("-c", "--config"): config_fname = os.path.join('configurations', arg) elif opt == '-s': start_from_model = True if start_from_model: filemode = 'a' else: filemode = 'w' logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO, filename='logging/vae_gan/evolution.log', filemode=filemode) with open(config_fname, 'r') as json_data: config_data = json.load(json_data) tweets_path = config_data['tweets_path'] vocab_path = config_data['vocab_path'] vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb')) #== == == == == == = # Load all the Data #== == == == == == = noutputs = 5 logging.info('Load Training Data') train_input, train_output = load_data( join(tweets_path, 'en_train.tsv'), config_data, vocab, noutputs) logging.info('Load Validation Data') valid_input, valid_output = load_data( join(tweets_path, 'en_valid15.tsv'), config_data, vocab, noutputs) logging.info('Load Validation Data') valid_input2, valid_output2 = load_data( join(tweets_path, 'en_test16.tsv'), config_data, vocab, noutputs) logging.info('Load Test Data') test_input, test_output = load_data(join(tweets_path, 'en_test17.tsv'), config_data, vocab, noutputs) step = K.variable(1.) # == == == == == == == == == == = # Define and load the CNN model # == == == == == == == == == == = full_model, encoding_train_model, decoder_train_model, discriminator_train_model, decoder_inference, encoder, decoder, discriminator, discriminator_pretrain_model = vae_gan_model( config_data, vocab, step) #full_model.summary() encoding_train_model.summary() decoder_train_model.summary() discriminator_train_model.summary() decoder_inference.summary() encoder.summary() decoder.summary() discriminator.summary() #pretrain_discriminator(discriminator_pretrain_model, train_input, vocab) model_path = 'models/vae_model/' steps_per_epoch = int( ceil(config_data['samples_per_epoch'] / config_data['batch_size'])) epochs = int( ceil(config_data['nb_epochs'] * (config_data['nsamples'] / config_data['samples_per_epoch']))) batch_size = config_data['batch_size'] initial_epoch = 0 skip_texts = 0 terminate_on_nan = TerminateOnNaN() model_checkpoint = ModelCheckpoint( 'models/vae_model/weights.{epoch:02d}.hdf5', period=10, save_weights_only=True) generator = generate_data_stream(config_data['training_path'], config_data, vocab, config_data['batch_size'], skip_data=skip_texts, noutputs=noutputs) enqueuer = GeneratorEnqueuer(generator, use_multiprocessing=False, wait_time=0.01) enqueuer.start(workers=1, max_queue_size=10) output_generator = enqueuer.get() enc_out_labels = [ 'enc_' + s for s in encoding_train_model._get_deduped_metrics_names() ] dec_out_labels = [ 'dec_' + s for s in decoder_train_model._get_deduped_metrics_names() ] dis_out_labels = [ 'dis_' + s for s in discriminator_train_model._get_deduped_metrics_names() ] out_labels = enc_out_labels + dec_out_labels + dis_out_labels #out_labels = full_model._get_deduped_metrics_names() callback_metrics = out_labels + ['val_' + n for n in out_labels] step_callback = NewCallback(step, steps_per_epoch) output_callback = OutputCallback(decoder_inference, valid_input, 15, vocab, '') callbacks = CallbackList([ BaseLogger(), ProgbarLogger(count_mode='steps'), step_callback, output_callback ]) callbacks.set_model(full_model) callbacks.set_params({ 'epochs': epochs, 'steps': steps_per_epoch, 'verbose': True, 'do_validation': True, 'metrics': callback_metrics, }) callbacks.on_train_begin() epoch = initial_epoch while epoch < epochs: epoch_logs = {} callbacks.on_epoch_begin(epoch) steps_done = 0 batch_index = 0 while steps_done < steps_per_epoch: batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = batch_size X, y = next(output_generator) callbacks.on_batch_begin(batch_index, batch_logs) set_trainability(encoder, trainable=True) set_trainability(decoder, trainable=False) set_trainability(discriminator, trainable=False) enc_outs = encoding_train_model.train_on_batch(X, y[:3]) set_trainability(encoder, trainable=False) set_trainability(decoder, trainable=True) set_trainability(discriminator, trainable=False) dec_outs = decoder_train_model.train_on_batch(X, y[:4]) set_trainability(encoder, trainable=False) set_trainability(decoder, trainable=False) set_trainability(discriminator, trainable=True) dis_outs = discriminator_train_model.train_on_batch(X, y[0]) outs = enc_outs + dec_outs + [dis_outs] #outs = full_model.train_on_batch(X, y) for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) epoch_logs = {} batch_index += 1 steps_done += 1 # Epoch finished. if steps_done >= steps_per_epoch: enc_val_outs = encoding_train_model.evaluate( valid_input, valid_output[:3], verbose=False) dec_val_outs = decoder_train_model.evaluate( valid_input, valid_output[:4], verbose=False) dis_val_outs = discriminator_train_model.evaluate( valid_input, valid_output[0], verbose=False) val_outs = enc_val_outs + dec_val_outs + [dis_val_outs] #val_outs = full_model.evaluate(valid_input, valid_output, verbose=False) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) epoch += 1 callbacks.on_train_end()