def fit(self, dataloader, nb_iter=None, nb_epoch=None, iter_per_epoch=None, callbacks=[], verbose=0): """Trains the underlying Keras model. Args: dataloader (StandardDataLoader): Manages the loading of data to model. nb_iter (int): The number of iterations to train the model. nb_epoch (int): The number of epochs to train the model. iter_per_epoch (int): Defines the number of iterations per epoch. callbacks (list): List of Keras callbacks to run during training. """ nb_iter, iter_per_epoch = self._get_iterations(nb_iter, nb_epoch, iter_per_epoch) callbacks = CallbackList(callbacks) callbacks._set_model(self) callbacks.on_train_begin() try: epoch = 0 self.stop_training = False for i in xrange(nb_iter): # Begin epoch if i % iter_per_epoch == 0: callbacks.on_epoch_begin(epoch) # Execution callbacks.on_batch_begin(i) if verbose > 0: import time time.sleep(0.001) j = i % iter_per_epoch perc = int(100 * (j + 1) / iter_per_epoch) prog = ''.join(['='] * (perc / 2)) string = "[{:50s}] {:3d}%\r".format(prog, perc) sys.stdout.write(string) sys.stdout.flush() losses = self.keras_model.train_on_batch( *dataloader.get_training_batch()) callbacks.on_batch_end(i) # End epoch if (i + 1) % iter_per_epoch == 0: callbacks.on_epoch_end(epoch, logs={'losses': losses}) epoch += 1 if self.stop_training: break except KeyboardInterrupt: print "\n[BayesNet] Abort: KeyboardInterrupt" raise callbacks.on_train_end()
def fit(self, dataloader, nb_iter=None, nb_epoch=None, iter_per_epoch=None, callbacks=[], verbose=0): """Trains the underlying Keras model. Args: dataloader (StandardDataLoader): Manages the loading of data to model. nb_iter (int): The number of iterations to train the model. nb_epoch (int): The number of epochs to train the model. iter_per_epoch (int): Defines the number of iterations per epoch. callbacks (list): List of Keras callbacks to run during training. """ nb_iter, iter_per_epoch = self._get_iterations( nb_iter, nb_epoch, iter_per_epoch) callbacks = CallbackList(callbacks) callbacks._set_model(self) callbacks.on_train_begin() try: epoch = 0 self.stop_training = False for i in xrange(nb_iter): # Begin epoch if i % iter_per_epoch == 0: callbacks.on_epoch_begin(epoch) # Execution callbacks.on_batch_begin(i) if verbose > 0: import time time.sleep(0.001) j = i % iter_per_epoch perc = int(100 * (j + 1) /iter_per_epoch) prog = ''.join(['='] * (perc/2)) string = "[{:50s}] {:3d}%\r".format(prog, perc) sys.stdout.write(string); sys.stdout.flush() losses = self.keras_model.train_on_batch( *dataloader.get_training_batch()) callbacks.on_batch_end(i) # End epoch if (i + 1) % iter_per_epoch == 0: callbacks.on_epoch_end(epoch, logs={'losses': losses}) epoch += 1 if self.stop_training: break except KeyboardInterrupt: print "\n[BayesNet] Abort: KeyboardInterrupt" raise callbacks.on_train_end()
def main(args): try: opts, args = getopt.getopt(args, "c:s", ["config="]) except getopt.GetoptError: print('usage: -c config.json') sys.exit(2) start_from_model = False for opt, arg in opts: if opt in ("-c", "--config"): config_fname = os.path.join('configurations', arg) elif opt == '-s': start_from_model = True if start_from_model: filemode = 'a' else: filemode = 'w' log_path = 'logging/vae_nlg_{}'.format(int(round(time.time() * 1000))) os.mkdir(log_path) logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO, filename='{}/evolution.log'.format(log_path), filemode=filemode) with open(config_fname, 'r') as json_data: config_data = json.load(json_data) batch_size = config_data['batch_size'] epochs = config_data['nb_epochs'] discriminator_iterations = config_data['discriminator_iterations'] tweets_path = config_data['tweets_path'] vocab_path = config_data['vocab_path'] vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb')) #== == == == == == = # Load all the Data #== == == == == == = delimiter = '' noutputs = 11 logging.info('Load Training Data') train_input, train_output, train_weights, train_lex = load_text_gen_data( join(tweets_path, 'trainset.csv'), config_data, vocab, noutputs, word_based=False) logging.info('Load Validation Data') valid_input, valid_output, _, valid_lex = load_text_gen_data( join(tweets_path, 'devset.csv'), config_data, vocab, noutputs, word_based=False) logging.info('Load Output Validation Data') valid_dev_input, valid_dev_output, _, valid_dev_lex = load_text_gen_data( join(tweets_path, 'devset_reduced.csv'), config_data, vocab, noutputs, random_output=False, word_based=False) step = K.variable(1., name='step_varialbe') steps_per_epoch = ceil(train_output[0].shape[0] / config_data['batch_size']) # == == == == == == == == == == = # Define and load the CNN model # == == == == == == == == == == = vae_model_train, vae_model_test, vae_vanilla_train_model, vae_vanilla_test_model, discriminator_model, decoder_test, discriminator = get_vae_gan_model( config_data, vocab, step) with open(os.path.join(log_path, 'models.txt'), 'wt') as fh: fh.write('VAE Model Train\n') fh.write('---------\n') vae_model_train.summary(print_fn=lambda x: fh.write(x + '\n')) fh.write('VAE Model Test\n') fh.write('--------------\n') vae_model_test.summary(print_fn=lambda x: fh.write(x + '\n')) fh.write('VAE Model Pretrain\n') fh.write('---------------------------\n') vae_vanilla_train_model.summary( print_fn=lambda x: fh.write(x + '\n')) fh.write('VAE Model Pretrain Test\n') fh.write('---------------------------\n') vae_vanilla_test_model.summary( print_fn=lambda x: fh.write(x + '\n')) fh.write('Decoder Test\n') fh.write('-------------------\n') decoder_test.summary(print_fn=lambda x: fh.write(x + '\n')) fh.write('Discriminator Models\n') fh.write('-------------------\n') discriminator_model.summary(print_fn=lambda x: fh.write(x + '\n')) terminate_on_nan = TerminateOnNaN() output_callback = LexOutputCallback( vae_vanilla_test_model, valid_dev_input, valid_dev_lex, 1, vocab, delimiter, fname='{}/test_output'.format(log_path)) #output_callback_full = LexOutputCallback(vae_vanilla_test_model, valid_dev_input, valid_dev_lex, 1, vocab, delimiter, fname='{}/test_output'.format(log_path)) # # vae_vanilla_train_model.fit_generator( # generator=generate_data_stream(config_data['pretrain_path'], config_data, vocab, config_data['batch_size'], noutputs=3), # steps_per_epoch=steps_per_epoch, # epochs=ceil(config_data['pretrain_samples']/config_data['pretrain_samples_per_epoch']), # callbacks=[output_callback, terminate_on_nan], # validation_data=(valid_input, valid_output[:3]), # ) vae_vanilla_train_model.fit( x=train_input, y=train_output[:2], epochs=config_data['pretrain_epochs'], batch_size=batch_size, validation_data=(valid_input, valid_output[:2]), sample_weight=train_weights[:2], callbacks=[output_callback, terminate_on_nan]) terminate_on_nan = TerminateOnNaN() model_checkpoint = ModelCheckpoint( 'models/vae_model/weights.{epoch:02d}.hdf5', period=10, save_weights_only=True) out_labels = [ 'enc_' + s for s in vae_model_train._get_deduped_metrics_names() ] out_labels += [ 'dis_' + s for s in discriminator_model._get_deduped_metrics_names() ] callback_metrics = out_labels + ['val_' + n for n in out_labels] tensorboard = TensorBoard(log_dir='logging/tensorboard', histogram_freq=0, write_grads=True, write_images=True) step_callback = StepCallback(step, steps_per_epoch) output_callback = LexOutputCallback( vae_vanilla_test_model, valid_dev_input, valid_dev_lex, 1, vocab, delimiter, fname='{}/test_output'.format(log_path)) output_callback_full = LexOutputCallback( vae_vanilla_test_model, valid_input, valid_lex, 5, vocab, delimiter, fname='{}/test_valid_output'.format(log_path)) callbacks = CallbackList([ BaseLogger(), ProgbarLogger(count_mode='steps'), step_callback, tensorboard, output_callback, output_callback_full, model_checkpoint, terminate_on_nan ]) callbacks.set_model(vae_model_train) callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, 'steps': steps_per_epoch, 'verbose': True, 'do_validation': True, 'metrics': callback_metrics or [], }) callbacks.on_train_begin() initial_epoch = 0 num_train_samples = train_input[0].shape[0] index_array = np.arange(num_train_samples) steps = 0 epoch = initial_epoch while epoch < epochs: epoch_logs = {} callbacks.on_epoch_begin(epoch) index_array = _batch_shuffle(index_array, batch_size) steps_done = 0 batches = _make_batches(num_train_samples, batch_size) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_logs = {} batch_ids = index_array[batch_start:batch_end] X = _slice_arrays(train_input, batch_ids) y = _slice_arrays(train_output, batch_ids) sample_weights = _slice_arrays(train_weights, batch_ids) batch_logs['batch'] = batch_index batch_logs['size'] = batch_size callbacks.on_batch_begin(batch_index, batch_logs) set_trainability(discriminator, trainable=False) enc_outs = vae_model_train.train_on_batch( x=X, y=y, sample_weight=sample_weights) set_trainability(discriminator, trainable=True) list_disc_loss_real = [] if steps < 25 or steps % 500 == 0: disc_iterations = 25 else: disc_iterations = discriminator_iterations for disc_it in range(disc_iterations): real_idx = np.random.choice(train_input[0].shape[0], len(batch_ids), replace=False) disX_train = train_input[-1][ real_idx] #take input 8 as train input and the rest as targets disy_train = [x[real_idx] for x in train_input[:8] ] #take input 1-7 as targets #train on real data dis_outs_real = discriminator_model.train_on_batch( disX_train, disy_train) list_disc_loss_real.append(dis_outs_real) loss_d_real = np.mean(list_disc_loss_real, axis=0) outs = np.concatenate((enc_outs, loss_d_real)) for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) epoch_logs = {} batch_index += 1 steps_done += 1 steps += 1 # Epoch finished. if steps_done >= steps_per_epoch: valid_len = valid_output[0].shape[0] enc_val_outs = vae_model_train.evaluate(valid_input, valid_output, verbose=False) dis_val_outs = discriminator_model.evaluate( valid_input[-1], valid_input[:8], verbose=False) val_outs = enc_val_outs + dis_val_outs #val_outs = full_model.evaluate(valid_input, valid_output, verbose=False) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) epoch += 1 callbacks.on_train_end()
def fit_dataset(self, dataset, steps_per_epoch=None, batch_size=32, epochs=1, verbose=1, callbacks=None, on_sample=None, on_scores=None): """Train the model on the given dataset for a given number of epochs. Arguments --------- dataset: Instance of `BaseDataset` that provides the data to train on. steps_per_epoch: int or None, number of gradient updates before considering an epoch has passed. If None it is set to be `len(dataset.train_data) / batch_size`. batch_size: int, number of samples per gradient update epochs: int, number of times to iterate `steps_per_epoch` times verbose: {0, >0}, whether to employ the progress bar Keras callback or not callbacks: list of Keras callbacks to be called during training on_sample: callable that accepts the sampler, idxs, w, scores on_scores: callable that accepts the sampler and scores """ try: if len(dataset.train_data) < batch_size: raise ValueError(("The model cannot be trained with " "batch_size > training set")) except RuntimeError as e: assert "no size" in str(e) # Set steps_per_epoch properly if steps_per_epoch is None: steps_per_epoch = len(dataset.train_data) // batch_size # Create the callbacks list self.history = History() callbacks = [BaseLogger()] + (callbacks or []) + [self.history] if verbose > 0: callbacks += [ProgbarLogger(count_mode="steps")] callbacks = CallbackList(callbacks) #TODO: Should we be making it possible to call back a different model # than self.model.model? callbacks.set_model(self.model.model) callbacks.set_params({ "epochs": epochs, "steps": steps_per_epoch, "verbose": verbose, "do_validation": len(dataset.test_data) > 0, "metrics": self._get_metric_names() + ["val_" + n for n in self._get_metric_names()] }) # Create the sampler sampler = self.sampler(dataset, batch_size, steps_per_epoch, epochs) # Start the training loop epoch = 0 self.model.model.stop_training = False callbacks.on_train_begin() while epoch < epochs: callbacks.on_epoch_begin(epoch) for step in range(steps_per_epoch): batch_logs = {"batch": step, "size": batch_size} callbacks.on_batch_begin(step, batch_logs) # Importance sampling is done here idxs, (x, y), w = sampler.sample(batch_size) # Train on the sampled data loss, metrics, scores = self.model.train_batch(x, y, w) # Update the sampler sampler.update(idxs, scores) values = map(lambda x: x.mean(), [loss] + metrics) for l, o in zip(self._get_metric_names(), values): batch_logs[l] = o callbacks.on_batch_end(step, batch_logs) if on_scores is not None: on_scores(sampler, self._latest_scores) if on_sample is not None: on_sample(sampler, self._latest_sample_event["idxs"], self._latest_sample_event["w"], self._latest_sample_event["predicted_scores"]) if self.model.model.stop_training: break # Evaluate now that an epoch passed epoch_logs = {} if len(dataset.test_data) > 0: val = self.model.evaluate(*dataset.test_data[:], batch_size=batch_size) epoch_logs = { "val_" + l: o for l, o in zip(self._get_metric_names(), val) } callbacks.on_epoch_end(epoch, epoch_logs) if self.model.model.stop_training: break epoch += 1 callbacks.on_train_end() return self.history
def train_wgan_with_grad_penalty(prior_gen, generator, data_gen, critic, batch_size, epochs, batches_per_epoch=100, optimizer=Adam(lr=1e-4, beta_1=0, beta_2=0.9), grad_pen_coef=10., critic_gen_train_ratio=2, callbacks=None): # build model to train the critic data_shape = critic.input_shape[1:] real_critic_input = Input(shape=data_shape, name='real_in') fake_critic_input = Input(shape=data_shape, name='fake_in') interp_critic_input = Input(shape=data_shape, name='interp_in') real_critic_score = critic(real_critic_input) fake_critic_score = critic(fake_critic_input) interp_critic_score = critic(interp_critic_input) critic_loss = subtract([fake_critic_score, real_critic_score]) gradient_penalty = GradPenLayer()( [interp_critic_input, interp_critic_score]) critic_train_mdl = Model( [real_critic_input, fake_critic_input, interp_critic_input], [critic_loss, gradient_penalty]) critic_train_mdl.compile(optimizer=optimizer, loss=lambda y_true, y_pred: y_pred, loss_weights=[1., grad_pen_coef]) # build model to train generator prior_input = Input(shape=generator.input_shape[1:], name='prior_in') critic.trainable = False critic_on_generator_score = critic(generator(prior_input)) generator_train_mdl = Model(prior_input, critic_on_generator_score) generator_train_mdl.compile(optimizer=optimizer, loss=lambda y_true, y_pred: -y_pred) # init callbacks callbacks = callbacks or [] callbacks = CallbackList(callbacks) callbacks.set_model({'generator': generator, 'critic': critic}) callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, 'steps': batches_per_epoch, 'samples': batches_per_epoch * batch_size, 'prior_gen': prior_gen, 'data_gen': data_gen, }) # train print('Training on {} samples for {} epochs'.format( batches_per_epoch * batch_size, epochs)) callbacks.on_train_begin() for e in range(epochs): print('Epoch {}/{}'.format(e + 1, epochs)) callbacks.on_epoch_begin(e) progbar = Progbar(target=batches_per_epoch * batch_size) dummy_y = np.array([None] * batch_size) for b in range(batches_per_epoch): callbacks.on_batch_begin(b) batch_losses = np.zeros(shape=3) for critic_upd in range(critic_gen_train_ratio): real_batch = data_gen(batch_size) fake_batch = generator.predict(prior_gen(batch_size)) weights = np.random.uniform(size=batch_size) weights = weights.reshape((-1, ) + (1, ) * (len(real_batch.shape) - 1)) interp_batch = weights * real_batch + (1. - weights) * fake_batch x_batch = { 'real_in': real_batch, 'fake_in': fake_batch, 'interp_in': interp_batch } cur_losses = np.array( critic_train_mdl.train_on_batch(x=x_batch, y=[dummy_y, dummy_y])) batch_losses += cur_losses generator_train_mdl.train_on_batch(x=prior_gen(batch_size), y=dummy_y) losses_names = ('total_loss', 'critic_loss', 'gradient_pen') progbar.add(batch_size, zip(losses_names, batch_losses)) callbacks.on_batch_end(b) progbar.update(batches_per_epoch * batch_size) callbacks.on_epoch_end(e) callbacks.on_train_end()
def _train_by_batch(self): # batch finite generator should be loaded within epoch loop logger.info('Start training by batch') self.validation_xy = self.load_data('val', feed_mode='all') do_validation = bool(self.validation_xy) # prepare display labels in tensorboard out_labels = self.model._get_deduped_metrics_names() callback_metrics = out_labels + ['val_' + n for n in out_labels] # prepare callbacks self.model.history = History() callbacks = [BaseLogger()] + (self.callbacks or []) + [self.model.history] # callbacks = (self.callbacks or []) + [self.model.history] if self.verbose: callbacks += [ProgbarLogger(count_mode='samples')] callbacks = CallbackList(callbacks) # it's possible to callback a different model than this model if hasattr(self.model, 'callback_model') and self.model.callback_model: callback_model = self.model.callback_model else: callback_model = self.model callbacks.set_model(callback_model) callbacks.set_params({ 'epochs': self.epochs, 'samples': self.data.nb_train, 'verbose': self.verbose, 'do_validation': do_validation, 'metrics': callback_metrics, }) callbacks.on_train_begin() for epoch in range(self.epochs): start_e = time() callbacks.on_epoch_begin(epoch) xy_gen = self.load_data('train', feed_mode='batch') logger.info('New training epoch') for batch_index, (x, y) in enumerate(xy_gen): # build batch logs batch_logs = {} if isinstance(x, list): batch_size = x[0].shape[0] elif isinstance(x, dict): batch_size = list(x.values())[0].shape[0] else: batch_size = x.shape[0] batch_logs['batch'] = batch_index batch_logs['size'] = batch_size callbacks.on_batch_begin(batch_index, batch_logs) outs = self.model.train_on_batch(x, y) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) if (batch_index + 1) % 1000 == 0 and do_validation: val_outs = self.model.evaluate(*self.validation_xy, batch_size=81920, verbose=0) batch_logs = {} if not isinstance(val_outs, list): val_outs = [val_outs] for l, o in zip(out_labels, val_outs): batch_logs['val_' + l] = o print(' - Eval inside: %.6f' % val_outs[0]) for cb in self.callbacks: if cb.__class__ == tensorBoard: cb.on_batch_end(batch_index, batch_logs, count=False) epoch_logs = {} if do_validation: val_outs = self.model.evaluate(*self.validation_xy, batch_size=81920, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_batch_end(epoch, epoch_logs) callbacks.on_epoch_end(epoch, epoch_logs) elapsed_e = timedelta(seconds=int(time() - start_e)) self.send_metric('elapsed_per_epoch', elapsed_e) if not self.no_save and do_validation and (epoch != self.epochs - 1): self.model.save( 'results/trained_models/%s_ctr_model_%.4f_epoch_%d.h5' % (self.sess_id, val_outs[0], epoch)) callbacks.on_train_end() return self.model.history
def predict_image(version, image_path, batch_size, overlap, data_format=None): def current_time_millis(): return int(round(time.time() * 1000)) def offset(size, diff, overlap): return math.floor(diff / math.ceil(diff / (size * (1 - overlap)))) def map_c(i, j, b, l): return int(((i * b) + j) / l) def map_r(i, j, b, l): return ((i * b) + j) % l if data_format is None: data_format = K.image_data_format() if data_format not in {'channels_first', 'channels_last'}: raise ValueError('Unknown data_format:', data_format) path = version.model_file.name print(_('Loading model "%(path)s".') % {'path': path}) model = load_model(os.path.join(settings.MEDIA_ROOT, path)) if len(model.inputs) != 1: raise RuntimeError('Models with more than one input are not' ' supported at the moment.') inputs = [] for i in range(len(model.inputs)): name = model.inputs[i].name pos = min( name.index('/') if '/' in name else len(name), name.index(':') if ':' in name else len(name)) name = name[:pos] inputs.append({'shape': model.inputs[i].shape.as_list(), 'name': name}) if data_format == 'channels_first': inputs[i]['grayscale'] = inputs[i]['shape'][1] == 1 inputs[i]['r'] = inputs[i]['shape'][2] inputs[i]['c'] = inputs[i]['shape'][3] elif data_format == 'channels_last': inputs[i]['r'] = inputs[i]['shape'][1] inputs[i]['c'] = inputs[i]['shape'][2] inputs[i]['grayscale'] = inputs[i]['shape'][3] == 1 inputs[i]['img'] = img_to_array( load_img(image_path, inputs[i]['grayscale'])) inputs[i]['img'] *= 1. / 255 if data_format == 'channels_first': inputs[i]['img_r'] = inputs[i]['img'].shape[1] inputs[i]['img_c'] = inputs[i]['img'].shape[2] elif data_format == 'channels_last': inputs[i]['img_r'] = inputs[i]['img'].shape[0] inputs[i]['img_c'] = inputs[i]['img'].shape[1] inputs[i]['diff_r'] = inputs[i]['img_r'] - inputs[i]['r'] inputs[i]['diff_c'] = inputs[i]['img_c'] - inputs[i]['c'] inputs[i]['offset_r'] = offset(inputs[i]['r'], inputs[i]['diff_r'], overlap) inputs[i]['offset_c'] = offset(inputs[i]['c'], inputs[i]['diff_c'], overlap) inputs[i]['nb_r'] = math.ceil( inputs[i]['diff_r'] / inputs[i]['offset_r']) + 1 inputs[i]['nb_c'] = math.ceil( inputs[i]['diff_c'] / inputs[i]['offset_c']) + 1 inputs = inputs[0] N = inputs['nb_r'] * inputs['nb_c'] steps = math.ceil(N / batch_size) metrics = [] outputs = [] for i in range(len(model.outputs)): tshape = model.outputs[i].shape.as_list() name = model.outputs[i].name pos = min( name.index('/') if '/' in name else len(name), name.index(':') if ':' in name else len(name)) name = name[:pos] activation = model.get_layer(name).activation.__name__.lower() outputs.append({'name': name, 'shape': tshape}) if len(tshape) == 2: if activation == 'softmax': outputs[i]['t'] = 'class' else: outputs[i]['t'] = 'multi' nb_classes = tshape[1] if nb_classes is None: nb_classes = model.get_layer(name).output_shape[1] nb_classes = int(nb_classes) metrics += ['%s:%s' % (name, i) for i in range(nb_classes)] if data_format == 'channels_first': shape = (nb_classes, inputs['nb_r'], inputs['nb_c']) elif data_format == 'channels_last': shape = (inputs['nb_r'], inputs['nb_c'], nb_classes) elif len(tshape) == 4: if activation == 'softmax': outputs[i]['t'] = 'class' else: outputs[i]['t'] = 'img' shape = (inputs['nb_r'], inputs['nb_c']) + tuple(tshape[1:]) outputs[i]['p'] = np.zeros(shape) history = History() callbacks = CallbackList([BaseLogger(), history, ProgbarLogger()]) callbacks.set_model(model) callbacks.set_params({ 'batch_size': batch_size, 'epochs': 1, 'steps': steps, 'samples': N, 'verbose': 1, 'do_validation': False, 'metrics': metrics, }) callbacks.on_train_begin() callbacks.on_epoch_begin(0) start_time = current_time_millis() for b in range(steps): current_index = (b * batch_size) % N if N >= current_index + batch_size: current_batch_size = batch_size else: current_batch_size = N - current_index batch_logs = {'batch': b, 'size': current_batch_size} for metric in metrics: batch_logs[metric] = 0 callbacks.on_batch_begin(b, batch_logs) bX = np.zeros((current_batch_size, ) + tuple(inputs['shape'][1:])) for j in range(current_batch_size): idx_r = map_r(b, j, batch_size, inputs['nb_r']) idx_c = map_c(b, j, batch_size, inputs['nb_r']) top = min(idx_r * inputs['offset_r'], inputs['img_r'] - inputs['r']) bottom = min(idx_r * inputs['offset_r'] + inputs['r'], inputs['img_r']) left = min(idx_c * inputs['offset_c'], inputs['img_c'] - inputs['c']) right = min(idx_c * inputs['offset_c'] + inputs['c'], inputs['img_c']) if data_format == 'channels_first': bX[j] = inputs['img'][:, top:bottom, left:right] elif data_format == 'channels_last': bX[j] = inputs['img'][top:bottom, left:right, :] p = model.predict_on_batch(bX) if type(p) != list: p = [p] for j in range(current_batch_size): for i in range(len(outputs)): idx_r = map_r(b, j, batch_size, inputs['nb_r']) idx_c = map_c(b, j, batch_size, inputs['nb_r']) if len(outputs[i]['p'].shape) == 3: if data_format == 'channels_first': outputs[i]['p'][:, idx_r, idx_c] = p[i][j] elif data_format == 'channels_last': outputs[i]['p'][idx_r, idx_c, :] = p[i][j] metric = metrics[p[i][j].argmax()] batch_logs[metric] += 1. / current_batch_size elif len(outputs[i]['p'].shape) == 5: outputs[i]['p'][idx_r, idx_c, :, :, :] = p[i][j] callbacks.on_batch_end(b, batch_logs) runtime = (current_time_millis() - start_time) / 1000 callbacks.on_epoch_end(0, {'runtime': runtime}) callbacks.on_train_end() for i in range(len(outputs)): if len(outputs[i]['shape']) == 2: if data_format == 'channels_first': shape = (outputs[i]['p'].shape[0], inputs['img_r'], inputs['img_c']) elif data_format == 'channels_last': shape = (inputs['img_r'], inputs['img_c'], outputs[i]['p'].shape[2]) elif len(tshape) == 4: if data_format == 'channels_first': shape = (outputs[i]['p'].shape[2], inputs['img_r'], inputs['img_c']) elif data_format == 'channels_last': shape = (inputs['img_r'], inputs['img_c'], outputs[i]['p'].shape[4]) count = np.zeros(shape) outputs[i]['img'] = np.zeros(shape) if len(outputs[i]['p'].shape) == 3: if data_format == 'channels_first': nb_rows = outputs[i]['p'].shape[1] nb_cols = outputs[i]['p'].shape[2] elif data_format == 'channels_last': nb_rows = outputs[i]['p'].shape[0] nb_cols = outputs[i]['p'].shape[1] elif len(outputs[i]['p'].shape) == 5: nb_rows = outputs[i]['p'].shape[0] nb_cols = outputs[i]['p'].shape[1] for j in range(nb_rows): for k in range(nb_cols): top = min(j * inputs['offset_r'], inputs['img_r'] - inputs['r']) bottom = min(j * inputs['offset_r'] + inputs['r'], inputs['img_r']) left = min(k * inputs['offset_c'], inputs['img_c'] - inputs['c']) right = min(k * inputs['offset_c'] + inputs['c'], inputs['img_c']) if data_format == 'channels_first': outputs[i]['img'][:, top:bottom, left:right] += \ outputs[i]['p'][:, j, k] count[:, top:bottom, left:right] += 1 elif data_format == 'channels_last': outputs[i]['img'][top:bottom, left:right, :] += \ outputs[i]['p'][j, k, :] count[top:bottom, left:right, :] += 1 outputs[i]['img'] /= count del outputs[i]['p'] del outputs[i]['shape'] return history.history, outputs
def evaluate(model, save_path, num_outputs, liver_only=False, **kwargs): model, callbacks, gen = prepare_model(model=model, save_path=save_path, num_outputs=num_outputs, liver_only=liver_only, **kwargs) print(' > Evaluating the model...') from scipy.misc import imsave # Create directory, if needed save_predictions_to = os.path.join(save_path, "predictions") if not os.path.exists(save_predictions_to): os.makedirs(save_predictions_to) # Initialize callbacks val_callback_list = [BaseLogger()] if not liver_only: val_callback_list.extend( [callbacks['dice_lesion'], callbacks['dice_lesion_inliver']]) if len(model.outputs) == 2 or liver_only: val_callback_list.append(callbacks['dice_liver']) val_callbacks = CallbackList(val_callback_list) val_callbacks.set_params({ 'nb_epoch': 0, 'nb_sample': 0, 'verbose': False, 'do_validation': True, 'metrics': model.metrics_names }) val_callbacks.on_train_begin() val_callbacks.on_epoch_begin(0) # Create theano function inputs = model.inputs + model.targets + model.sample_weights if model.uses_learning_phase and \ not isinstance(K.learning_phase(), int): inputs += [K.learning_phase()] predict_and_test_function = K.function( \ inputs, model.outputs+[model.total_loss]+model.metrics_tensors, updates=model.state_updates) # Loop through batches, applying function and callbacks flow = repeat_flow(gen['valid_callback'].flow(), num_outputs=num_outputs) for batch_num, batch in enumerate(flow): x, y, sample_weights = model._standardize_user_data(batch[0], batch[1]) ins = x + y + sample_weights if model.uses_learning_phase and \ not isinstance(K.learning_phase(), int): ins += [0.] outputs = predict_and_test_function(ins) if num_outputs == 1: predictions = outputs[0:1] val_metrics = outputs[1:] else: predictions = outputs[0:2] val_metrics = outputs[2:] ## Save images #def process_slice(s): #s = np.squeeze(s).copy() #s[s<0]=0 #s[s>1]=1 #s[0,0]=1 #s[0,1]=0 #return s #for i in range(len(batch[0])): #s_pred_list = [] #if num_outputs==1: #s_pred_list = [process_slice(predictions[i])] #else: #for j in range(num_outputs): #s_pred_list.append(process_slice(predictions[j][i])) #s_input = process_slice(batch[0][i]) #if num_outputs==1: #s_truth = process_slice(batch[1][i]/2.) #else: #s_truth = process_slice(batch[1][0][i]/2.) #out_image = np.concatenate([s_input]+s_pred_list+[s_truth], #axis=1) #imsave(os.path.join(save_predictions_to, #"{}_{}.png".format(batch_num, i)), #out_image) # Update metrics val_logs = OrderedDict(zip(model.metrics_names, val_metrics)) val_logs.update({'batch': batch_num, 'size': len(batch[0])}) val_callbacks.on_batch_end(batch_num, val_logs) # Update metrics val_callbacks.on_epoch_end(0, val_logs) # Output metrics for m in val_logs: if m not in ['batch', 'size']: print("{}: {}".format(m, val_logs[m]))
def fit(self, x, y, batch_size=None, nsteps=None, epochs=1, verbose=1, callbacks=None, validation_data=None): assert self.is_compiled, "Must compile model first" assert epochs > 0 x = x if type(x) is list else [x] y = y if type(y) is list else [y] if nsteps is None: total_len = len(y[0]) if type(y) is list else len(y) nsteps = total_len // batch_size # BaseLogger should always be the first metric since it computes the stats on epoch end base_logger = BaseLogger( stateful_metrics=["val_%s" % m for m in self.metrics_name] + ['val_loss', 'size']) base_logger_params = {'metrics': ['loss'] + self.metrics_name} if validation_data: base_logger_params['metrics'] += [ 'val_%s' % m for m in base_logger_params['metrics'] ] base_logger.set_params(base_logger_params) hist = History() if callbacks is None: callbacks = [base_logger] + [hist] elif type(callbacks) is list: callbacks = [base_logger] + callbacks + [hist] else: callbacks = [base_logger] + [callbacks] + [hist] callback_list = CallbackList(callbacks=callbacks) callback_list.set_model(self) callback_list.on_train_begin() self.callbacks = callback_list for epoch in range(epochs): g = batchify(x, y, batch_size) if batch_size else None t = trange(nsteps) if verbose == 1 else range(nsteps) callback_list.on_epoch_begin(epoch) for it in t: x_, y_ = next(g) if g else (None, None) batch_logs = self.train_on_batch(x_, y_) callback_list.on_batch_end(it, batch_logs) curr_loss = base_logger.totals['loss'] / base_logger.seen if verbose == 1: t.set_postfix(loss="%.4f" % curr_loss) if verbose == 2: if it % 1000 == 0: print("%s %i/%i, loss=%.5f" % (datetime.datetime.now().strftime("%H:%M:%S"), it, nsteps, curr_loss), flush=True) if validation_data: val_logs = self.evaluate(validation_data[0], validation_data[1]) base_logger.on_batch_end(None, val_logs) epoch_logs = {} callback_list.on_epoch_end(epoch=epoch, logs=epoch_logs) if verbose: if validation_data: to_print = ['loss'] + self.metrics_name + ['val_loss'] + [ 'val_%s' % m for m in self.metrics_name ] else: to_print = ['loss'] + self.metrics_name prog = ", ".join([ "%s=%.4f" % (name, hist.history[name][-1]) for name in to_print ]) print("Epoch %i, %s" % (epoch, prog), flush=True) if self.stop_training: break return hist.history
print("loading model from checkpoint") vae.load_weights(str(ckpt_file)) callbacks.set_model(vae) # Do training vae.stop_training = False callbacks.on_train_begin() for epoch in range(args.resume, epochs): print("Epoch {}/{}".format(epoch, epochs)) callbacks.on_epoch_begin(epoch) for step in tqdm(range(args.epoch_steps)): callbacks.on_batch_begin(step) batch = next(flow_gen) vae.train_on_batch(batch, None) callbacks.on_batch_end(step) # evaluate val_outs = vae.evaluate(next(flow_gen)) logs = {"val_loss": val_outs} callbacks.on_epoch_end(epoch, logs) if vae.stop_training: print("Training stopped early.") break callbacks.on_train_end() print("saving model") vae.save_weights(str(ckpt_file)) # build a model to project inputs on the latent space encoder = Model(x, z_mean)
def main(args): try: opts, args = getopt.getopt(args, "c:s", ["config="]) except getopt.GetoptError: print('usage: -c config.json') sys.exit(2) start_from_model = False for opt, arg in opts: if opt in ("-c", "--config"): config_fname = os.path.join('configurations', arg) elif opt == '-s': start_from_model = True if start_from_model: filemode = 'a' else: filemode = 'w' logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO, filename='logging/vae_gan/evolution.log', filemode=filemode) with open(config_fname, 'r') as json_data: config_data = json.load(json_data) tweets_path = config_data['tweets_path'] vocab_path = config_data['vocab_path'] vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb')) #== == == == == == = # Load all the Data #== == == == == == = noutputs = 5 logging.info('Load Training Data') train_input, train_output = load_data( join(tweets_path, 'en_train.tsv'), config_data, vocab, noutputs) logging.info('Load Validation Data') valid_input, valid_output = load_data( join(tweets_path, 'en_valid15.tsv'), config_data, vocab, noutputs) logging.info('Load Validation Data') valid_input2, valid_output2 = load_data( join(tweets_path, 'en_test16.tsv'), config_data, vocab, noutputs) logging.info('Load Test Data') test_input, test_output = load_data(join(tweets_path, 'en_test17.tsv'), config_data, vocab, noutputs) step = K.variable(1.) # == == == == == == == == == == = # Define and load the CNN model # == == == == == == == == == == = full_model, encoding_train_model, decoder_train_model, discriminator_train_model, decoder_inference, encoder, decoder, discriminator, discriminator_pretrain_model = vae_gan_model( config_data, vocab, step) #full_model.summary() encoding_train_model.summary() decoder_train_model.summary() discriminator_train_model.summary() decoder_inference.summary() encoder.summary() decoder.summary() discriminator.summary() #pretrain_discriminator(discriminator_pretrain_model, train_input, vocab) model_path = 'models/vae_model/' steps_per_epoch = int( ceil(config_data['samples_per_epoch'] / config_data['batch_size'])) epochs = int( ceil(config_data['nb_epochs'] * (config_data['nsamples'] / config_data['samples_per_epoch']))) batch_size = config_data['batch_size'] initial_epoch = 0 skip_texts = 0 terminate_on_nan = TerminateOnNaN() model_checkpoint = ModelCheckpoint( 'models/vae_model/weights.{epoch:02d}.hdf5', period=10, save_weights_only=True) generator = generate_data_stream(config_data['training_path'], config_data, vocab, config_data['batch_size'], skip_data=skip_texts, noutputs=noutputs) enqueuer = GeneratorEnqueuer(generator, use_multiprocessing=False, wait_time=0.01) enqueuer.start(workers=1, max_queue_size=10) output_generator = enqueuer.get() enc_out_labels = [ 'enc_' + s for s in encoding_train_model._get_deduped_metrics_names() ] dec_out_labels = [ 'dec_' + s for s in decoder_train_model._get_deduped_metrics_names() ] dis_out_labels = [ 'dis_' + s for s in discriminator_train_model._get_deduped_metrics_names() ] out_labels = enc_out_labels + dec_out_labels + dis_out_labels #out_labels = full_model._get_deduped_metrics_names() callback_metrics = out_labels + ['val_' + n for n in out_labels] step_callback = NewCallback(step, steps_per_epoch) output_callback = OutputCallback(decoder_inference, valid_input, 15, vocab, '') callbacks = CallbackList([ BaseLogger(), ProgbarLogger(count_mode='steps'), step_callback, output_callback ]) callbacks.set_model(full_model) callbacks.set_params({ 'epochs': epochs, 'steps': steps_per_epoch, 'verbose': True, 'do_validation': True, 'metrics': callback_metrics, }) callbacks.on_train_begin() epoch = initial_epoch while epoch < epochs: epoch_logs = {} callbacks.on_epoch_begin(epoch) steps_done = 0 batch_index = 0 while steps_done < steps_per_epoch: batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = batch_size X, y = next(output_generator) callbacks.on_batch_begin(batch_index, batch_logs) set_trainability(encoder, trainable=True) set_trainability(decoder, trainable=False) set_trainability(discriminator, trainable=False) enc_outs = encoding_train_model.train_on_batch(X, y[:3]) set_trainability(encoder, trainable=False) set_trainability(decoder, trainable=True) set_trainability(discriminator, trainable=False) dec_outs = decoder_train_model.train_on_batch(X, y[:4]) set_trainability(encoder, trainable=False) set_trainability(decoder, trainable=False) set_trainability(discriminator, trainable=True) dis_outs = discriminator_train_model.train_on_batch(X, y[0]) outs = enc_outs + dec_outs + [dis_outs] #outs = full_model.train_on_batch(X, y) for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) epoch_logs = {} batch_index += 1 steps_done += 1 # Epoch finished. if steps_done >= steps_per_epoch: enc_val_outs = encoding_train_model.evaluate( valid_input, valid_output[:3], verbose=False) dec_val_outs = decoder_train_model.evaluate( valid_input, valid_output[:4], verbose=False) dis_val_outs = discriminator_train_model.evaluate( valid_input, valid_output[0], verbose=False) val_outs = enc_val_outs + dec_val_outs + [dis_val_outs] #val_outs = full_model.evaluate(valid_input, valid_output, verbose=False) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) epoch += 1 callbacks.on_train_end()
def main(args): try: opts, args = getopt.getopt(args, "c:s", ["config="]) except getopt.GetoptError: print('usage: -c config.json') sys.exit(2) start_from_model = False for opt, arg in opts: if opt in ("-c", "--config"): config_fname = os.path.join('configurations', arg) elif opt == '-s': start_from_model = True if start_from_model: filemode = 'a' else: filemode = 'w' log_path = 'logging/vae_nlg_{}'.format(int(round(time.time() * 1000))) os.mkdir(log_path) logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO, filename='{}/evolution.log'.format(log_path), filemode=filemode) with open(config_fname, 'r') as json_data: config_data = json.load(json_data) batch_size = config_data['batch_size'] epochs = config_data['nb_epochs'] discriminator_iterations = config_data['discriminator_iterations'] tweets_path = config_data['tweets_path'] vocab_path = config_data['vocab_path'] vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb')) #== == == == == == = # Load all the Data #== == == == == == = noutputs = 5 logging.info('Load Training Data') train_input, train_output = load_text_pairs( join(tweets_path, 'training_set.tsv'), config_data, vocab, noutputs) logging.info('Load Validation Data') valid_input, valid_output = load_text_pairs( join(tweets_path, 'vaild_set.tsv'), config_data, vocab, noutputs) logging.info('Load Output Validation Data') valid_dev_input, valid_dev_output = load_text_pairs( join(tweets_path, 'test_set.tsv'), config_data, vocab, noutputs) #train_input = [x[:1213] for x in train_input] #train_output = [x[:1213] for x in train_output] noise_valid_input = np.zeros(shape=(valid_input[0].shape[0], config_data['z_size'])) step = K.variable(1.) steps_per_epoch = ceil(train_output[0].shape[0] / config_data['batch_size']) # == == == == == == == == == == = # Define and load the CNN model # == == == == == == == == == == = vae_model, vae_model_test, decoder_discr_model, decoder_test_model, discriminator_model, discriminator = get_vae_gan_model( config_data, vocab, step) with open(os.path.join(log_path, 'models.txt'), 'wt') as fh: fh.write('VAE Model\n') fh.write('---------\n') vae_model.summary(print_fn=lambda x: fh.write(x + '\n')) fh.write('VAE Model Test\n') fh.write('--------------\n') vae_model_test.summary(print_fn=lambda x: fh.write(x + '\n')) fh.write('Decoder Discriminator Model\n') fh.write('---------------------------\n') decoder_discr_model.summary(print_fn=lambda x: fh.write(x + '\n')) fh.write('Decoder Test Model\n') fh.write('---------------------------\n') decoder_test_model.summary(print_fn=lambda x: fh.write(x + '\n')) fh.write('Discriminator Model\n') fh.write('-------------------\n') discriminator_model.summary(print_fn=lambda x: fh.write(x + '\n')) terminate_on_nan = TerminateOnNaN() model_checkpoint = ModelCheckpoint( 'models/vae_model/weights.{epoch:02d}.hdf5', period=10, save_weights_only=True) enc_out_labels = [ 'enc_' + s for s in vae_model._get_deduped_metrics_names() ] dec_out_labels = [ 'dec_' + s for s in decoder_discr_model._get_deduped_metrics_names() ] dis_out_labels = [ 'dis_' + s for s in discriminator_model._get_deduped_metrics_names() ] out_labels = enc_out_labels + dec_out_labels + [ 'dis_real', 'dis_gen', 'dis_noise' ] #out_labels = full_model._get_deduped_metrics_names() callback_metrics = out_labels + ['val_' + n for n in out_labels] step_callback = StepCallback(step, steps_per_epoch) output_callback = GANOutputCallback( vae_model_test, valid_dev_input[0], 1, vocab, '', fname='{}/test_output'.format(log_path)) callbacks = CallbackList([ BaseLogger(), ProgbarLogger(count_mode='steps'), step_callback, output_callback, model_checkpoint, terminate_on_nan ]) callbacks.set_model(vae_model_test) callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, 'steps': steps_per_epoch, 'verbose': True, 'do_validation': True, 'metrics': callback_metrics or [], }) callbacks.on_train_begin() initial_epoch = 0 num_train_samples = train_input[0].shape[0] index_array = np.arange(num_train_samples) steps = 0 epoch = initial_epoch while epoch < epochs: epoch_logs = {} callbacks.on_epoch_begin(epoch) index_array = _batch_shuffle(index_array, batch_size) steps_done = 0 batches = _make_batches(num_train_samples, batch_size) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_logs = {} batch_ids = index_array[batch_start:batch_end] X = _slice_arrays(train_input, batch_ids) y = _slice_arrays(train_output, batch_ids) batch_logs['batch'] = batch_index batch_logs['size'] = batch_size callbacks.on_batch_begin(batch_index, batch_logs) set_trainability(discriminator, trainable=False) enc_outs = vae_model.train_on_batch(x=X, y=y[:3]) set_trainability(discriminator, trainable=True) list_disc_loss_real = [] list_disc_loss_gen = [] list_disc_loss_noise = [] if steps < 25 or steps % 500 == 0: disc_iterations = 100 else: disc_iterations = discriminator_iterations noise_input = np.zeros(shape=(len(batch_ids), config_data['z_size'])) for disc_it in range(disc_iterations): #clip_weights(discriminator) real_idx = np.random.choice(train_input[0].shape[0], len(batch_ids), replace=False) train_real_batch = [x[real_idx] for x in train_input] #train on real data x_fake = vae_model_test.predict_on_batch( x=train_real_batch[0]) x_noise_fake = decoder_test_model.predict_on_batch( x=noise_input) train_input_discr = np.concatenate( (train_real_batch[0], train_real_batch[0], train_real_batch[0])) train_output_discr = np.concatenate( (train_real_batch[1], x_fake, x_noise_fake)) labels = np.asarray( len(batch_ids) * [1] + 2 * len(batch_ids) * [-1]) index_array_discr = np.arange(len(labels)) np.random.shuffle(index_array_discr) discr_batch = [ train_input_discr[index_array_discr], train_output_discr[index_array_discr] ] discr_batch_labels = labels[index_array_discr] dis_outs_real = discriminator_model.train_on_batch( discr_batch, discr_batch_labels) #dis_outs_real = discriminator_model.train_on_batch(train_real_batch, -np.ones(shape=(len(batch_ids), 1))) #dis_outs_gen = discriminator_model.train_on_batch([train_real_batch[0], x_fake], np.ones(shape=(len(batch_ids), 1))) #dis_outs_gen_noise = discriminator_model.train_on_batch([train_real_batch[0], x_noise_fake], np.ones(shape=(len(batch_ids), 1))) list_disc_loss_real.append(dis_outs_real) #list_disc_loss_gen.append(dis_outs_gen) #list_disc_loss_noise.append(dis_outs_gen_noise) loss_d_real = -np.mean(list_disc_loss_real) loss_d_gen = np.mean(list_disc_loss_gen) loss_d_noise = np.mean(list_disc_loss_noise) set_trainability(discriminator, trainable=False) decoder_discr_input = [X[0], noise_input] dec_outs = decoder_discr_model.train_on_batch( x=decoder_discr_input, y=-np.ones(shape=(len(batch_ids), 1))) outs = enc_outs + [dec_outs] + [loss_d_real] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) epoch_logs = {} batch_index += 1 steps_done += 1 steps += 1 # Epoch finished. if steps_done >= steps_per_epoch: valid_len = valid_output[0].shape[0] enc_val_outs = vae_model.evaluate(valid_input, valid_output[:3], verbose=False) dec_val_outs = decoder_discr_model.evaluate( [valid_input[0], noise_valid_input], -np.ones(shape=(valid_len, 1)), verbose=False) dis_val_outs = discriminator_model.evaluate( valid_input, -np.ones(shape=(valid_len, 1)), verbose=False) val_outs = enc_val_outs + [dec_val_outs] + [dis_val_outs] #val_outs = full_model.evaluate(valid_input, valid_output, verbose=False) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) epoch += 1 callbacks.on_train_end()
def fit_generator(self, generator, n_steps_per_epoch, n_epochs=1, validation_data=None, n_validation_steps=None, callbacks=None): """Train the network on batches of data generated from `generator` :param generator: a generator yielding batches indefinitely, where each batch is a tuple of (inputs, targets) :type generator: generator :param n_steps_per_epoch: number of batches to train on in one epoch :type n_steps_per_epoch: int :param n_epochs: number of epochs to train the model :type n_epochs: int :param validation_data: generator yielding batches to evaluate the loss on at the end of each epoch, where each batch is a tuple of (inputs, targets) :type validation_data: generator :param n_validation_steps: number of batches to evaluate on from `validation_data` :param callbacks: callbacks to be used during training :type callbacks: list[object] :raises RuntimeError: if only one of `validation_data` and `n_validation_steps` are passed in """ default_callbacks = self._load_default_callbacks() default_callbacks.append(ProgbarLogger(count_mode='steps')) if callbacks: default_callbacks.extend(callbacks) callbacks = CallbackList(default_callbacks) self._assert_compiled() invalid_inputs = ((validation_data is not None and not n_validation_steps) or (n_validation_steps and validation_data is None)) if invalid_inputs: msg = ('`validation_data` and `n_validation_steps` must both be ' 'passed, or neither.') raise RuntimeError(msg) if self.device: self.network.to(self.device) metrics = ['loss'] if self.n_outputs > 1: for idx_output in range(1, self.n_outputs + 1): metrics.append('loss{}'.format(idx_output)) if validation_data is not None: metrics.append('val_loss') if self.n_outputs > 1: for idx_output in range(1, self.n_outputs + 1): metrics.append('val_loss{}'.format(idx_output)) for metric_name in self.metric_names: metrics.append(metric_name) if validation_data is not None: metrics.append('val_{}'.format(metric_name)) callbacks.set_params({ 'epochs': n_epochs, 'metrics': metrics, 'steps': n_steps_per_epoch, 'verbose': True }) callbacks.set_model(self) callbacks.on_train_begin() for idx_epoch in range(n_epochs): if self.stop_training: break epoch_logs = {} callbacks.on_epoch_begin(idx_epoch) for idx_batch in range(n_steps_per_epoch): batch_logs = {'batch': idx_batch, 'size': 1} callbacks.on_batch_begin(idx_batch, batch_logs) generator_output = next(generator) if len(generator_output) != 2: msg = ('Output of generator should be a tuple of ' '(inputs, targets), but instead got a {}: ' '{}.').format(type(generator_output), str(generator_output)) inputs, targets = generator_output train_outputs = self.train_on_batch(inputs, targets) batch_logs['loss'] = train_outputs[0] if self.n_outputs > 1: for idx_output in range(1, self.n_outputs + 1): batch_logs['loss{}'.format(idx_output)] = ( train_outputs[idx_output]) idx_metric_values = (1 if self.n_outputs == 1 else self.n_outputs + 1) it = zip(self.metric_names, train_outputs[idx_metric_values:]) for metric_name, train_output in it: batch_logs[metric_name] = train_output callbacks.on_batch_end(idx_batch, batch_logs) if self.stop_training: break if validation_data: val_outputs = self.evaluate_generator(validation_data, n_validation_steps) epoch_logs['val_loss'] = val_outputs[0] if self.n_outputs > 1: for idx_output in range(1, self.n_outputs + 1): epoch_logs['val_loss{}'.format(idx_output)] = ( val_outputs[idx_output]) idx_metric_values = (1 if self.n_outputs == 1 else self.n_outputs + 1) it = zip(self.metric_names, val_outputs[idx_metric_values:]) for metric_name, val_output in it: metric_name = 'val_{}'.format(metric_name) epoch_logs[metric_name] = val_output callbacks.on_epoch_end(idx_epoch, epoch_logs) callbacks.on_train_end()
def fit(self, x, y, batch_size, n_epochs=1, callbacks=None, validation_data=None): """Trains the network on the given data for a fixed number of epochs :param x: input data to train on :type x: torch.Tensor :param y: target data to train on :type y: torch.Tensor :param batch_size: number of samples to use per forward and backward pass :type batch_size: int :param n_epochs: number of epochs (iterations of the dataset) to train the model :type n_epochs: int :param callbacks: callbacks to be used during training :type callbacks: list[object] :param validation_data: data on which to evaluate the loss and metrics at the end of each epoch :type validation_data: tuple(numpy.ndarray) """ default_callbacks = self._load_default_callbacks() default_callbacks.append(ProgbarLogger(count_mode='samples')) if callbacks: default_callbacks.extend(callbacks) callbacks = CallbackList(default_callbacks) self._assert_compiled() if self.device: self.network.to(self.device) metrics = ['loss'] if self.n_outputs > 1: for idx_output in range(1, self.n_outputs + 1): metrics.append('loss{}'.format(idx_output)) if validation_data is not None: metrics.append('val_loss') if self.n_outputs > 1: for idx_output in range(1, self.n_outputs + 1): metrics.append('val_loss{}'.format(idx_output)) for metric_name in self.metric_names: metrics.append(metric_name) if validation_data is not None: metrics.append('val_{}'.format(metric_name)) index_array = np.arange(x.shape[0]) callbacks.set_params({ 'batch_size': batch_size, 'epochs': n_epochs, 'metrics': metrics, 'steps': None, 'samples': x.shape[0], 'verbose': True }) callbacks.set_model(self) callbacks.on_train_begin() for idx_epoch in range(n_epochs): if self.stop_training: break epoch_logs = {} callbacks.on_epoch_begin(idx_epoch) np.random.shuffle(index_array) batches = make_batches(len(index_array), batch_size) for idx_batch, (idx_start, idx_end) in enumerate(batches): batch_logs = {'batch': idx_batch, 'size': idx_end - idx_start} callbacks.on_batch_begin(idx_batch, batch_logs) inputs = x[index_array[idx_start:idx_end]] if self.n_outputs > 1: targets = [] for idx_output in range(self.n_outputs): targets.append( y[idx_output][index_array[idx_start:idx_end]]) else: targets = y[index_array[idx_start:idx_end]] train_outputs = self.train_on_batch(inputs, targets) batch_logs['loss'] = train_outputs[0] if self.n_outputs > 1: for idx_output in range(1, self.n_outputs + 1): batch_logs['loss{}'.format(idx_output)] = ( train_outputs[idx_output]) idx_metric_values = (1 if self.n_outputs == 1 else self.n_outputs + 1) it = zip(self.metric_names, train_outputs[idx_metric_values:]) for metric_name, train_output in it: batch_logs[metric_name] = train_output callbacks.on_batch_end(idx_batch, batch_logs) if self.stop_training: break if validation_data: val_outputs = self.evaluate(validation_data[0], validation_data[1], batch_size) epoch_logs['val_loss'] = val_outputs[0] if self.n_outputs > 1: for idx_output in range(1, self.n_outputs + 1): epoch_logs['val_loss{}'.format(idx_output)] = ( val_outputs[idx_output]) idx_metric_values = (1 if self.n_outputs == 1 else self.n_outputs + 1) it = zip(self.metric_names, val_outputs[idx_metric_values:]) for metric_name, val_output in it: metric_name = 'val_{}'.format(metric_name) epoch_logs[metric_name] = val_output callbacks.on_epoch_end(idx_epoch, epoch_logs) callbacks.on_train_end()
def predict(model, batch_size, num_outputs, save_path, evaluate=False, liver_only=False, save_predictions=False, initial_epoch=0, **kwargs): model, callbacks, gen = prepare_model(model=model, num_outputs=num_outputs, liver_only=liver_only, evaluate=evaluate, **kwargs) # Set up prediction file. if save_predictions: save_path = os.path.join(save_path, "predictions.zarr") if os.path.exists(save_path): os.remove(save_path) # Initialize callbacks val_callback_list = [BaseLogger()] if not liver_only: val_callback_list.extend( [callbacks['dice_lesion'], callbacks['dice_lesion_inliver']]) if len(model.outputs) == 2 or liver_only: val_callback_list.append(callbacks['dice_liver']) val_callbacks = CallbackList(val_callback_list) val_callbacks.set_params({ 'nb_epoch': 0, 'nb_sample': 0, 'verbose': False, 'do_validation': True, 'metrics': model.metrics_names }) val_callbacks.on_train_begin() val_callbacks.on_epoch_begin(0) # Create theano function if evaluate: inputs = model.inputs + model.targets + model.sample_weights if model.uses_learning_phase and \ not isinstance(K.learning_phase(), int): inputs += [K.learning_phase()] predict_function = K.function(inputs, model.outputs + [model.total_loss] + model.metrics_tensors, updates=model.state_updates) else: inputs = model.inputs if model.uses_learning_phase and \ not isinstance(K.learning_phase(), int): inputs += [K.learning_phase()] predict_function = K.function(inputs, model.outputs, updates=model.state_updates) # Predict for all data. print(' > Predicting...') for key in gen: print(' - DATA: {}'.format(key)) # Duplicate inputs and outputs (and add outputs) as necessary. flow = repeat_flow(gen[key].flow(), num_outputs=num_outputs) # Set up file. if save_predictions: zgroup = zarr.open_group(store=save_path, mode='a', path="/") zarr_kwargs = { 'chunks': (1, 512, 512), 'compressor': zarr.Blosc(cname='lz4', clevel=9, shuffle=1) } # Predict and write to file. batch_num = 0 for vol_num, volume in enumerate(flow): print("Predicting on `{}` - {}/{}" "".format(key, vol_num + 1, len(gen[key]))) # Begin writing to file. if save_predictions: vol_idx = volume[-1] subgroup = zgroup.create_group(str(vol_idx)) num_channels = np.sum(model.output_shape[i][1] \ for i in range(num_outputs)) output_shape = \ (len(volume[0]), num_channels)+model.output_shape[0][2:] subgroup.empty("volume", shape=output_shape, dtype=np.float32, **zarr_kwargs) segmentation = volume[1] if isinstance(segmentation, list): segmentation = segmentation[0] subgroup.create_dataset("segmentation", shape=segmentation.shape, data=segmentation, dtype=np.int16, **zarr_kwargs) # Iterate through volume batch-wise. for idx0, idx1 in zip( range(0, len(volume[0]), batch_size), range(batch_size, len(volume[0]) + batch_size + 1, batch_size)): # Prepare data for joint evaluation and prediction. if evaluate: batch = (volume[0][idx0:idx1], volume[1][idx0:idx1]) x, y, sample_weights = model._standardize_user_data( batch[0], batch[1]) ins = x + y + sample_weights else: batch = (volume[0][idx0:idx1], ) ins = _standardize_input_data(batch[0], model._feed_input_names, model._feed_input_shapes, check_batch_axis=False, exception_prefix='input') if model.uses_learning_phase and \ not isinstance(K.learning_phase(), int): ins += [0.] # Jointly evaluate and predict. outputs = predict_function(ins) if num_outputs == 1: predictions = outputs[0:1] if evaluate: val_metrics = outputs[1:] elif num_outputs == 2: predictions = outputs[0:2] if evaluate: val_metrics = outputs[2:] else: raise ValueError("num_outputs must be 1 or 2") # Write predictions. predictions = np.concatenate(predictions, axis=1) subgroup['volume'][idx0:idx1] = predictions # Update metrics if evaluate: val_logs = OrderedDict( zip(model.metrics_names, val_metrics)) val_logs.update({ 'batch': batch_num, 'size': len(batch[0]) }) val_callbacks.on_batch_end(batch_num, val_logs) batch_num += 1 if evaluate: # Update metrics val_callbacks.on_epoch_end(0, val_logs) # Output metrics for m in val_logs: if m not in ['batch', 'size']: print("{}: {}".format(m, val_logs[m]))