def train_callbacks(rendergan, output_dir, nb_visualise, real_hdf5_fname, distribution, lr_schedule=None, overwrite=False): save_gan_cb = SaveGAN(rendergan, join(output_dir, "models/{epoch:03d}/{name}.hdf5"), every_epoch=10, hdf5_attrs=get_distribution_hdf5_attrs(distribution)) nb_score = 1000 sample_fn = predict_wrapper( rendergan.sample_generator_given_z.predict, rendergan.sample_generator_given_z_output_names) real = next(train_data_generator(real_hdf5_fname, nb_score, 1))['data'] vis_cb = VisualiseTag3dAndFake(nb_samples=nb_visualise // 2, output_dir=join(output_dir, 'visualise_tag3d_fake'), show=False, preprocess=lambda x: np.clip(x, -1, 1)) vis_all = VisualiseAll(nb_samples=nb_visualise // len(rendergan.sample_generator_given_z.outputs), output_dir=join(output_dir, 'visualise_all'), show=False, preprocess=lambda x: np.clip(x, -1, 1)) vis_fake_sorted = VisualiseFakesSorted( nb_samples=nb_visualise, output_dir=join(output_dir, 'visualise_fakes_sorted'), show=False, preprocess=lambda x: np.clip(x, -1, 1)) vis_real_sorted = VisualiseRealsSorted( nb_samples=nb_visualise, output_dir=join(output_dir, 'visualise_reals_sorted'), show=False, preprocess=lambda x: np.clip(x, -1, 1)) def default_lr_schedule(lr): return { 200: lr / 4, 250: lr / 4**2, 300: lr / 4**3, } def lr_scheduler(opt): return LearningRateScheduler(opt, lr_schedule(float(K.get_value(opt.lr)))) if lr_schedule is None: lr_schedule = default_lr_schedule g_optimizer = rendergan.gan.g_optimizer d_optimizer = rendergan.gan.d_optimizer lr_schedulers = [ lr_scheduler(g_optimizer), lr_scheduler(d_optimizer), ] hist_dir = join(output_dir, "history") os.makedirs(hist_dir, exist_ok=True) hist = HistoryPerBatch(hist_dir) def history_plot(e, logs={}): fig, _ = hist.plot(save_as="{:03d}.png".format(e), metrics=['g_loss', 'd_loss']) plt.close(fig) # allows fig to be garbage collected hist_save = OnEpochEnd(history_plot, every_nth_epoch=20) sample_outdir = join(output_dir, 'samples') os.makedirs(sample_outdir, exist_ok=True) store_samples_cb = StoreSamples(sample_outdir, distribution, overwrite) dscore_outdir = join(output_dir, 'd_score_hist') os.makedirs(dscore_outdir, exist_ok=True) dscore = DScoreHistogram(dscore_outdir) nb_sample = max(nb_score, nb_visualise) sample_cb = SampleGAN(sample_fn, rendergan.discriminator.predict, rendergan.gan.random_z(nb_sample), real, callbacks=[ vis_cb, vis_fake_sorted, vis_all, vis_real_sorted, dscore, store_samples_cb ]) return [sample_cb, save_gan_cb, hist, hist_save] + lr_schedulers
def train(self): marker = os.path.basename(self.output_dir) os.makedirs(self.output_dir, exist_ok=True) self.save() self.summary() # save samples gt_val = h5py.File(self.gt_val_fname) print("bits", gt_val["bits"].shape) nb_vis_samples = 20**2 label_output_sizes = self.get_label_output_sizes() if self.use_combined_data: gt_train = h5py.File(self.gt_train_fname) data_gen = self.combined_generator_factory(gt_train, label_output_sizes) else: data_gen = self.data_generator_factory() truth_gen_only_bits = self.truth_generator_factory(gt_val, missing_label_sizes=[]) self.check_generator(data_gen, "train") self.check_generator(truth_gen_only_bits, "test") vis_out = next(data_gen(nb_vis_samples)) vis_bits = np.array(vis_out[1][:nb_bits]).T save_samples(vis_out[0][:, 0], vis_bits, self.outname(marker + "_train_samples.png")) gt_data, gt_bits, gt_masks = next(truth_gen_only_bits(nb_vis_samples)) gt_bits = np.array(gt_bits).T print("gt_data", gt_data.shape, gt_data.min(), gt_data.max()) print("gt_bits", gt_bits.shape, gt_bits.min(), gt_bits.max()) save_samples(gt_data[:, 0], gt_bits, self.outname(marker + "_val_samples.png")) # build model bs = self.batch_size model = self.get_model(label_output_sizes) # setup training hist = HistoryPerBatch(self.output_dir, extra_metrics=['bits_loss', 'val_bits_loss']) hist_saver = OnEpochEnd(lambda e, l: hist.save(), every_nth_epoch=5) def lr_schedule(optimizer): lr = K.get_value(optimizer.lr) return { 40: lr / 10., } scheduler = LearningRateScheduler( model.optimizer, lr_schedule(model.optimizer)) hdf5_attrs = get_distribution_hdf5_attrs(self.get_label_distributions()) hdf5_attrs['decoder_uses_hist_equalization'] = self.use_hist_equalization checkpointer = SaveModelAndWeightsCheckpoint( self.model_fname(), monitor='val_bits_loss', verbose=0, save_best_only=True, hdf5_attrs=hdf5_attrs) plot_history = hist.plot_callback( fname=self.outname(marker + '_loss.png'), metrics=['bits_loss', 'val_bits_loss']) # train truth_gen = self.truth_generator_factory(gt_val, label_output_sizes) callbacks = [CollectBitsLoss(), scheduler, checkpointer, hist, plot_history, hist_saver] if int(self.verbose) == 0: callbacks.append(DotProgressBar()) model.fit_generator( data_gen(bs), samples_per_epoch=bs*self.nb_batches_per_epoch, nb_epoch=self.nb_epoch, callbacks=callbacks, verbose=self.verbose, validation_data=truth_gen(bs), nb_val_samples=gt_val['tags'].shape[0], nb_worker=1, max_q_size=4*10, pickle_safe=False ) evaluate_decoder.run(self, cache=False)
def train(self): marker = os.path.basename(self.output_dir) os.makedirs(self.output_dir, exist_ok=True) self.save() self.summary() # save samples gt_val = h5py.File(self.gt_val_fname) print("bits", gt_val["bits"].shape) nb_vis_samples = 20**2 data_gen = self.data_generator_factory() truth_gen_only_bits = self.truth_generator_factory( gt_val, missing_label_sizes=[]) check_samples = 100 self.check_generator(data_gen(check_samples), "train") self.check_generator(truth_gen_only_bits(check_samples), "test") vis_out = next(data_gen(nb_vis_samples)) vis_bits = np.array(vis_out[1][:nb_bits]).T save_samples(vis_out[0][:, 0], vis_bits, self.outname(marker + "_train_samples.png")) gt_data, gt_bits = next(truth_gen_only_bits(nb_vis_samples)) gt_bits = np.array(gt_bits).T print("gt_data", gt_data.shape, gt_data.min(), gt_data.max()) print("gt_bits", gt_bits.shape, gt_bits.min(), gt_bits.max()) save_samples(gt_data[:, 0], gt_bits, self.outname(marker + "_val_samples.png")) # build model bs = self.batch_size label_output_sizes = self.get_label_output_sizes() model = self.get_model(label_output_sizes) # setup training hist = HistoryPerBatch(self.output_dir, extra_metrics=['bits_loss', 'val_bits_loss']) hist_saver = OnEpochEnd(lambda e, l: hist.save(), every_nth_epoch=5) def lr_schedule(optimizer): lr = K.get_value(optimizer.lr) return { 40: lr / 10., } scheduler = LearningRateScheduler(model.optimizer, lr_schedule(model.optimizer)) hdf5_attrs = get_distribution_hdf5_attrs( self.get_label_distributions()) hdf5_attrs[ 'decoder_uses_hist_equalization'] = self.use_hist_equalization checkpointer = SaveModelAndWeightsCheckpoint(self.model_fname(), monitor='val_bits_loss', verbose=0, save_best_only=True, hdf5_attrs=hdf5_attrs) plot_history = hist.plot_callback( fname=self.outname(marker + '_loss.png'), metrics=['bits_loss', 'val_bits_loss']) # train truth_gen = self.truth_generator_factory(gt_val, label_output_sizes) callbacks = [ CollectBitsLoss(), scheduler, checkpointer, hist, plot_history, hist_saver ] if int(self.verbose) == 0: callbacks.append(DotProgressBar()) model.fit_generator(data_gen(bs), samples_per_epoch=bs * self.nb_batches_per_epoch, nb_epoch=self.nb_epoch, callbacks=callbacks, verbose=self.verbose, validation_data=truth_gen(bs), nb_val_samples=gt_val['tags'].shape[0], nb_worker=1, max_q_size=4 * 10, pickle_safe=False) evaluate_decoder.run(self, cache=False)
def run(output_dir, force, tags_3d_hdf5_fname, nb_units, depth, nb_epoch, filter_size, project_factor, nb_dense): batch_size = 64 basename = "network_tags3d_n{}_d{}_e{}".format(nb_units, depth, nb_epoch) output_basename = os.path.join(output_dir, basename) tag_dataset = DistributionHDF5Dataset(tags_3d_hdf5_fname) tag_dataset._dataset_created = True print("Got {} images from the 3d model".format(tag_dataset.nb_samples)) weights_fname = output_basename + ".hdf5" if os.path.exists(weights_fname) and not force: raise OSError("File {} already exists. Use --force to override it") elif os.path.exists(weights_fname) and force: os.remove(weights_fname) os.makedirs(output_dir, exist_ok=True) def generator(batch_size): for batch in tag_dataset.iter(batch_size): labels = [] for name in batch['labels'].dtype.names: labels.append(batch['labels'][name]) assert not np.isnan(batch['tag3d']).any() assert not np.isnan(batch['depth_map']).any() labels = np.concatenate(labels, axis=-1) yield labels, [batch['tag3d'], batch['depth_map']] labels = next(generator(batch_size))[0] print("labels.shape ", labels.shape) print("labels.dtype ", labels.dtype) nb_input = next(generator(batch_size))[0].shape[1] x = Input(shape=(nb_input, )) tag3d, depth_map = tag3d_network_dense(x, nb_units=nb_units, depth=depth, nb_dense_units=nb_dense) g = Model(x, [tag3d, depth_map]) # optimizer = SGD(momentum=0.8, nesterov=True) optimizer = Nadam() g.compile(optimizer, loss=['mse', 'mse'], loss_weights=[1, 1 / 3.]) scheduler = AutomaticLearningRateScheduler(optimizer, 'loss', epoch_patience=5, min_improvement=0.0002) history = HistoryPerBatch() save = SaveModels({basename + '_snapshot_{epoch:^03}.hdf5': g}, output_dir=output_dir, hdf5_attrs=tag_dataset.get_distribution_hdf5_attrs()) history_plot = history.plot_callback(fname=output_basename + "_loss.png", every_nth_epoch=10) g.fit_generator(generator(batch_size), samples_per_epoch=800 * batch_size, nb_epoch=nb_epoch, verbose=1, callbacks=[scheduler, save, history, history_plot]) nb_visualize = 18**2 vis_labels, (tags_3d, depth_map) = next(generator(nb_visualize)) predict_tags_3d, predict_depth_map = g.predict(vis_labels) def zip_and_save(fname, *args): clipped = list(map(lambda x: np.clip(x, 0, 1)[:, 0], args)) print(clipped[0].shape) tiled = zip_tile(*clipped) print(tiled.shape) scipy.misc.imsave(fname, tiled) zip_and_save(output_basename + "_predict_tags.png", tags_3d, predict_tags_3d) zip_and_save(output_basename + "_predict_depth_map.png", depth_map, predict_depth_map) save_model(g, weights_fname, attrs=tag_dataset.get_distribution_hdf5_attrs()) with open(output_basename + '.json', 'w+') as f: f.write(g.to_json()) with open(output_basename + '_loss_history.json', 'w+') as f: json.dump(history.history, f) fig, _ = history.plot() fig.savefig(output_basename + "_loss.png") print("Saved weights to: {}".format(weights_fname))
def test_history_per_batch_plot(outdir): hist = HistoryPerBatch() hist.params = {} hist.params['metrics'] = ['loss', 'val_loss'] hist.on_train_begin(0) path_cb = str(outdir.join("callback_plot.png")) plot_cb = hist.plot_callback(fname=path_cb) n = 50 mean = 1/np.arange(1, n+1) std = 1/np.arange(1, n+1) for e in range(n): hist.on_epoch_begin(e) for b in range(100): hist.on_batch_begin(b) hist.on_batch_end(b, logs={'loss': float(np.random.normal(mean[e], std[e], 1))}) hist.on_epoch_end(e, logs={'val_loss': float(np.random.normal(mean[e], std[e], 1))}) plot_cb.on_epoch_end(e) fig, axes = hist.plot() path1 = str(outdir.join("callback_history.png")) fig.savefig(path1) path2 = str(outdir.join("callback_history2.png")) hist.plot(save_as=path2) filecmp.cmp(path1, path2) filecmp.cmp(path_cb, path2)
def test_history_per_batch(tmpdir): hist = HistoryPerBatch(str(tmpdir)) hist.params = {} hist.params['metrics'] = ['loss', 'val_loss'] hist.on_epoch_begin(0) losses = [[]] for i in range(5): loss = float(np.random.sample(1)) hist.on_batch_end(i, logs={'loss': loss}) losses[-1].append(loss) hist.on_epoch_end(0, logs={'loss': 1, 'val_loss': 2}) losses.append([]) hist.on_epoch_begin(1) for i in range(5): loss = float(np.random.sample(1)) hist.on_batch_end(i, logs={'loss': loss}) losses[-1].append(loss) hist.on_epoch_end(1, logs={'loss': 1, 'val_loss': 2}) losses.append([]) hist.on_epoch_begin(2) for i in range(5): loss = float(np.random.sample(1)) hist.on_batch_end(i, logs={'loss': loss}) losses[-1].append(loss) hist.on_epoch_end(2, logs={'loss': 1, 'val_loss': 2}) with pytest.warns(DeprecationWarning): assert hist.history['loss'] == losses assert hist.epoch_history['loss'] == [1, 1, 1] assert hist.epoch_history['val_loss'] == [2, 2, 2] hist.on_train_end() assert tmpdir.join("history.json").exists() assert tmpdir.join("history.png").exists()
def run(output_dir, force, tags_3d_hdf5_fname, nb_units, depth, nb_epoch, filter_size, project_factor, nb_dense): batch_size = 64 basename = "network_tags3d_n{}_d{}_e{}".format(nb_units, depth, nb_epoch) output_basename = os.path.join(output_dir, basename) tag_dataset = DistributionHDF5Dataset(tags_3d_hdf5_fname) tag_dataset._dataset_created = True print("Got {} images from the 3d model".format(tag_dataset.nb_samples)) weights_fname = output_basename + ".hdf5" if os.path.exists(weights_fname) and not force: raise OSError("File {} already exists. Use --force to override it") elif os.path.exists(weights_fname) and force: os.remove(weights_fname) os.makedirs(output_dir, exist_ok=True) def generator(batch_size): for batch in tag_dataset.iter(batch_size): labels = [] for name in batch['labels'].dtype.names: labels.append(batch['labels'][name]) assert not np.isnan(batch['tag3d']).any() assert not np.isnan(batch['depth_map']).any() labels = np.concatenate(labels, axis=-1) yield labels, [batch['tag3d'], batch['depth_map']] labels = next(generator(batch_size))[0] print("labels.shape ", labels.shape) print("labels.dtype ", labels.dtype) nb_input = next(generator(batch_size))[0].shape[1] x = Input(shape=(nb_input,)) tag3d, depth_map = tag3d_network_dense(x, nb_units=nb_units, depth=depth, nb_dense_units=nb_dense) g = Model(x, [tag3d, depth_map]) # optimizer = SGD(momentum=0.8, nesterov=True) optimizer = Nadam() g.compile(optimizer, loss=['mse', 'mse'], loss_weights=[1, 1/3.]) scheduler = AutomaticLearningRateScheduler( optimizer, 'loss', epoch_patience=5, min_improvement=0.0002) history = HistoryPerBatch() save = SaveModels({basename + '_snapshot_{epoch:^03}.hdf5': g}, output_dir=output_dir, hdf5_attrs=tag_dataset.get_distribution_hdf5_attrs()) history_plot = history.plot_callback(fname=output_basename + "_loss.png", every_nth_epoch=10) g.fit_generator(generator(batch_size), samples_per_epoch=800*batch_size, nb_epoch=nb_epoch, verbose=1, callbacks=[scheduler, save, history, history_plot]) nb_visualize = 18**2 vis_labels, (tags_3d, depth_map) = next(generator(nb_visualize)) predict_tags_3d, predict_depth_map = g.predict(vis_labels) def zip_and_save(fname, *args): clipped = list(map(lambda x: np.clip(x, 0, 1)[:, 0], args)) print(clipped[0].shape) tiled = zip_tile(*clipped) print(tiled.shape) scipy.misc.imsave(fname, tiled) zip_and_save(output_basename + "_predict_tags.png", tags_3d, predict_tags_3d) zip_and_save(output_basename + "_predict_depth_map.png", depth_map, predict_depth_map) save_model(g, weights_fname, attrs=tag_dataset.get_distribution_hdf5_attrs()) with open(output_basename + '.json', 'w+') as f: f.write(g.to_json()) with open(output_basename + '_loss_history.json', 'w+') as f: json.dump(history.history, f) fig, _ = history.plot() fig.savefig(output_basename + "_loss.png") print("Saved weights to: {}".format(weights_fname))