def load_data(name: str, batch_size: int): dataset = get_dataset(name)() assert dataset.has_labels, f'No labels for given dataset {name}' kw = dict(batch_size=batch_size, drop_remainder=True) test_l = dataset.create_dataset(partition='test', inc_labels=1.0, **kw) sample_images, y = [(x[:16], y[:16]) for x, y in test_l.take(1)][0] # inputs structure images, labels = tf.data.experimental.get_structure(test_l) images_shape = images.shape[1:] labels_shape = labels.shape[1:] return dataset, (sample_images, y), (images_shape, labels_shape)
def create_vae(args) -> Tuple[VariationalAutoencoder, ImageDataset]: dist = 'bernoulli' if args.ds == 'shapes3d' else 'qlogistic' key = f'model_{args.model}' if key not in globals(): raise ValueError(f'Cannot find model with name: {args.model}') model = globals()[key](args.zdim, dist) model: VariationalAutoencoder model.build([None] + IMAGE_SHAPE) ds = get_dataset(args.ds) ds: ImageDataset return model, ds
def main(args: Namespace): # === 1. get data and set metadata ds = get_dataset(args.ds) global IMAGE_SHAPE, ZDIM, DS, N_LABELS ZDIM = int(args.zdim) IMAGE_SHAPE = list(ds.shape) DS = ds N_LABELS = float(args.y) # === 2. create model vae = create_model(args) if args.eval: vae.load_weights(get_model_path()) evaluate(vae, args) else: train(vae, ds, args, label_percent=args.y, oversample_ratio=0.5)
def main(args: Arguments): # === 0. set configs ds = get_dataset(args.ds) # === 1. get model model = None for k, v in globals().items(): if inspect.isfunction(v) and 'model_' == k[:6] and \ k.split('_')[-1] == args.vae: model = v(args) model.build(ds.full_shape) break if model is None: model = get_model(args, return_dataset=False) # === 2. eval if args.eval: model.load_weights(get_model_path(args), raise_notfound=True, verbose=True) evaluate(model, ds, args) # === 3. train else: train(model, ds, args)
def on_load_data(self, cfg): ds = get_dataset(cfg.ds)() ds.sample_images(save_path=os.path.join(self.save_path, 'samples.png')) kw = dict(batch_size=128, drop_remainder=True) train = ds.create_dataset(partition='train', inc_labels=float(cfg.semi), **kw) train_u = ds.create_dataset(partition='train', inc_labels=False, **kw) valid = ds.create_dataset(partition='valid', inc_labels=1.0, **kw) valid_u = ds.create_dataset(partition='valid', inc_labels=False, **kw) # reduce batch_size here, otherwise, mllk take ~ 7GB VRAM kw['batch_size'] = 8 test = ds.create_dataset(partition='test', inc_labels=1.0, **kw) test_u = ds.create_dataset(partition='test', inc_labels=False, **kw) self.ds = ds self.train, self.train_u = train, train_u self.valid, self.valid_u = valid, valid_u self.test, self.test_u = test, test_u if cfg.verbose: print("Dataset:", ds) print(" train:", train) print(" train_u:", train_u)
def on_load_data(self, cfg): dataset = get_dataset(cfg.ds)() train = dataset.create_dataset(partition='train', inc_labels=False) valid = dataset.create_dataset(partition='valid', inc_labels=False) test = dataset.create_dataset(partition='test', inc_labels=True) # sample x_valid = [x for x in valid.take(1)][0][:16] self.x_test = [xy[0] for xy in test.take(1)][0][:16] ### input description input_spec = tf.data.experimental.get_structure(train) fig = plt.figure(figsize=(12, 12)) for i, x in enumerate(x_valid.numpy()): if x.shape[-1] == 1: x = np.squeeze(x, axis=-1) plt.subplot(4, 4, i + 1) plt.imshow(x, cmap='Greys_r' if x.ndim == 2 else None) plt.axis('off') plt.tight_layout() fig.savefig(os.path.join(self.save_path, '%s.pdf' % cfg.ds)) plt.close(fig) ### store self.input_dtype = input_spec.dtype self.input_shape = input_spec.shape[1:] self.train, self.valid, self.test = train, valid, test
def on_load_data(self, cfg): self.dataset = get_dataset(cfg.ds)() kw = dict(batch_size=cfg.batch_size, drop_remainder=True) self.train_u = self.dataset.create_dataset(partition='train', inc_labels=False, **kw) self.valid_u = self.dataset.create_dataset(partition='valid', inc_labels=False, **kw) self.test_u = self.dataset.create_dataset(partition='test', inc_labels=True, **kw) self.train_l = self.dataset.create_dataset(partition='train', inc_labels=0.1, **kw) self.valid_l = self.dataset.create_dataset(partition='valid', inc_labels=1.0, **kw) self.test_l = self.dataset.create_dataset(partition='test', inc_labels=1.0, **kw) # sample self.sample_images, y = [(x[:16], y[:16]) for x, y in self.test_l.take(1)][0] if np.any(np.sum(y, axis=1) > 1): if np.any(y > 1): self.labels_dist = "nb" # negative binomial else: self.labels_dist = "bernoulli" else: self.labels_dist = "onehot" # inputs structure images, labels = tf.data.experimental.get_structure( self.train_l)['inputs'] self.images_shape = images.shape[1:] self.labels_shape = labels.shape[1:]
'set_cfg', 'get_output_dir', 'get_model', 'get_model_path', 'get_results_path', 'get_args', 'run_multi', 'train', ] _root_path: str = '/tmp/model' _logging_interval: float = 5. _valid_interval: float = 80 _n_valid_batches: int = 400 _extra_path = {} _DS: Dict[str, ImageDataset] = defaultdictkey(lambda name: get_dataset(name)) # =========================================================================== # Helper for evaluation # =========================================================================== @dataclasses.dataclass class Arguments: vae: str = '' ds: str = '' zdim: Union[str, int] = 32 it: int = 80000 bs: int = 32 clipnorm: float = 100. dpi: int = 100 points: int = 4000
def run_task(args, evaluation=False): from odin.fuel import get_dataset from odin.networks import get_networks, get_optimizer_info from odin.bay.vi import DisentanglementGym, Correlation, DimReduce ######## arguments model: SemafoVAE = args['model'] dsname: str = args['ds'] py: float = args['py'] coef: Interpolation = args['coef'] model_name = model.__name__.lower() ######## prepare path logdir = f'{outdir}/{dsname}_{py}/{model_name}_{coef.name}' if OVERRIDE and os.path.exists(logdir) and not evaluation: shutil.rmtree(logdir) print(f'Override model at path: {logdir}') if not os.path.exists(logdir): os.makedirs(logdir) modelpath = f'{logdir}/model' ######## dataset ds = get_dataset(dsname) tanh_norm = True if dsname == 'celeba' else False train = ds.create_dataset('train', batch_size=32, label_percent=py, normalize='tanh' if tanh_norm else 'probs') valid = ds.create_dataset( 'valid', batch_size=32, label_percent=True, normalize='tanh' if tanh_norm else 'probs').shuffle( 1000, seed=1, reshuffle_each_iteration=True) ######## model networks = get_networks(dsname, centerize_image=False if tanh_norm else True, is_semi_supervised=True) vae = model(alpha=1. / py, mi_coef=coef, **networks) vae.build((None, ) + ds.shape) vae.load_weights(modelpath, verbose=True) ######## evaluation if evaluation: if vae.step.numpy() <= 1: return evaluate(vae, ds, expdir=f'{logdir}/analysis', title=f'{dsname}{py}_{model_name}_{coef.name}') return ######## training best_llk_x = [] best_llk_y = [] def callback(): llk_x = [] llk_y = [] for x, y in valid.take(300): px, (qz, qy) = vae(x, training=False) llk_x.append(px.log_prob(x)) llk_y.append(qy.log_prob(y)) llk_x = tf.reduce_mean(tf.concat(llk_x, axis=0)) llk_y = tf.reduce_mean(tf.concat(llk_y, axis=0)) best_llk_x.append(llk_x) best_llk_y.append(llk_y) if llk_x >= np.max(best_llk_x): vae.save_weights(modelpath) vae.trainer.print(f'{model_name} {dsname} {py} ' f'best weights at iter#{vae.step.numpy()} ' f'llk_x={llk_x:.2f} llk_y={llk_y:.2f}') opt_info = get_optimizer_info(dsname) opt_info['max_iter'] += 20000 vae.fit( train, logdir=logdir, on_valid_end=callback, valid_interval=60, logging_interval=2, nan_gradients_policy='stop', compile_graph=True, skip_fitted=True, **opt_info, ) print(f'Trained {model_name} {dsname} {py} {vae.step.numpy()}(steps)')
def main(vae, ds, args, parser): n_labeled = SEMI_SETTING[ds] vae = get_vae(vae) ds = get_dataset(ds) batch_size = BS_SETTING[ds.name] assert isinstance( ds, ImageDataset), f'Only support image dataset but given {ds}' vae: Type[VariationalAutoencoder] ds: ImageDataset is_semi = vae.is_semi_supervised() ## skip unsupervised system, if there are semi-supervised modifications if not is_semi: for key in ('alpha', 'coef', 'ratio'): if parser.get_default(key) != getattr(args, key): print('Skip semi-supervised training for:', args) return ## prepare the arguments kw = {} ## path name = f'{ds.name}_{vae.__name__.lower()}' path = f'{ROOT}/{name}' anno = [] if args.zdim > 0: anno.append(f'z{int(args.zdim)}') if is_semi: anno.append(f'a{args.alpha:g}') anno.append(f'r{args.ratio:g}') kw['alpha'] = args.alpha if issubclass(vae, (SemafoBase, MIVAE)): anno.append(f'c{args.coef:g}') kw['mi_coef'] = args.coef if len(anno) > 0: path += f"_{'_'.join(anno)}" if args.override and os.path.exists(path): shutil.rmtree(path) print('Override:', path) if not os.path.exists(path): os.makedirs(path) print(path) ## data train = ds.create_dataset('train', batch_size=batch_size, label_percent=n_labeled if is_semi else False, oversample_ratio=args.ratio) valid = ds.create_dataset('valid', label_percent=1.0, batch_size=batch_size // 2) ## create model vae = vae( **kw, **get_networks(ds.name, zdim=int(args.zdim), is_semi_supervised=is_semi)) vae.build((None, ) + ds.shape) print(vae) vae.load_weights(f'{path}/model', verbose=True) best_llk = [] ## training def callback(): llk = [] y_true = [] y_pred = [] for x, y in tqdm(valid.take(500)): P, Q = vae(x, training=False) P = as_tuple(P) llk.append(P[0].log_prob(x)) if is_semi: y_true.append(np.argmax(y, -1)) y_pred.append(np.argmax(get_ymean(P[1]), -1)) # accuracy if is_semi: y_true = np.concatenate(y_true, axis=0) y_pred = np.concatenate(y_pred, axis=0) acc = accuracy_score(y_true=y_true, y_pred=y_pred) else: acc = 0 # log-likelihood llk = tf.reduce_mean(tf.concat(llk, 0)) best_llk.append(llk) text = f'#{vae.step.numpy()} llk={llk:.2f} acc={acc:.2f}' if llk >= np.max(best_llk): vae.save_weights(f'{path}/model') vae.trainer.print(f'best llk {text}') else: vae.trainer.print(f'worse llk {text}') # tensorboard summary tf.summary.scalar('llk_valid', llk) tf.summary.scalar('acc_valid', acc) optim_info = get_optimizer_info(ds.name, batch_size=batch_size) if args.it > 0: optim_info['max_iter'] = int(args.it) vae.fit( train, skip_fitted=True, logdir=path, on_valid_end=callback, clipnorm=100, logging_interval=10, valid_interval=180, nan_gradients_policy='stop', **optim_info, ) ## evaluating vae.load_weights(f'{path}/model', verbose=True) if args.eval: evaluate(vae, ds, path, f'{name}')
def main(cfg: dict): assert cfg.vae is not None, \ ('No VAE model given, select one of the following: ' f"{', '.join(i.__name__.lower() for i in get_vae())}") assert cfg.ds is not None, \ ('No dataset given, select one of the following: ' 'mnist, dsprites, shapes3d, celeba, cortex, newsgroup20, newsgroup5, ...') ### load dataset ds = get_dataset(name=cfg.ds) ds_kw = dict(batch_size=batch_size, drop_remainder=True) ### path, save the output to the subfolder with dataset name output_dir = get_output_dir(subfolder=cfg.ds.lower()) gym_train_path = os.path.join(output_dir, 'gym_train') gym_valid_path = os.path.join(output_dir, 'gym_valid') gym_test_path = os.path.join(output_dir, 'gym_test') model_path = os.path.join(output_dir, 'model') ### prepare model init model = get_vae(cfg.vae) model_kw = inspect.getfullargspec(model.__init__).args[1:] model_kw = {k: v for k, v in cfg.items() if k in model_kw} is_semi_supervised = ds.has_labels and model.is_semi_supervised() if is_semi_supervised: train = ds.create_dataset(partition='train', label_percent=0.1, **ds_kw) valid = ds.create_dataset(partition='valid', label_percent=1.0, **ds_kw) else: train = ds.create_dataset(partition='train', label_percent=0., **ds_kw) valid = ds.create_dataset(partition='valid', label_percent=0., **ds_kw) ### create the model vae = model(path=model_path, **get_networks(cfg.ds, centerize_image=True, is_semi_supervised=is_semi_supervised, skip_generator=cfg.skip), **model_kw) vae.build((None, ) + ds.shape) vae.load_weights(raise_notfound=False, verbose=True) vae.early_stopping.mode = 'max' gym = create_gym(dsname=cfg.ds, vae=vae) ### fit the network def callback(): metrics = vae.trainer.last_valid_metrics llk = metrics['llk_image'] if 'llk_image' in metrics else metrics[ 'llk_dense_type'] vae.early_stopping.update(llk) signal = vae.early_stopping(verbose=True) if signal > 0: vae.save_weights(overwrite=True) # create the return metrics return dict(**gym.train()(prefix='train/', dpi=150), **gym.valid()(prefix='valid/', dpi=150)) ### evaluation if cfg.eval: vae.load_weights() gym.train() gym(save_path=gym_train_path, dpi=200, verbose=True) gym.valid() gym(save_path=gym_valid_path, dpi=200, verbose=True) gym.test() gym(save_path=gym_test_path, dpi=200, verbose=True) ### fit else: vae.early_stopping.patience = 10 vae.fit(train, valid=valid, epochs=-1, clipnorm=100, valid_interval=30, logging_interval=2, skip_fitted=True, on_valid_end=callback, logdir=output_dir, compile_graph=True, track_gradients=True, **get_optimizer_info(cfg.ds)) vae.early_stopping.plot_losses( path=os.path.join(output_dir, 'early_stopping.png')) vae.plot_learning_curves( os.path.join(output_dir, 'learning_curves.png'))