def all_plots(model, mnist): if model.architecture[-1] == 2: # only works for 2-D latent print("Plotting in latent space...") plot_all_in_latent(model, mnist) print("Exploring latent...") plot.exploreLatent(model, nx=20, ny=20, range_=(-4, 4), outdir=PLOTS_DIR) for n in (24, 30, 60, 100): plot.exploreLatent(model, nx=n, ny=n, ppf=True, outdir=PLOTS_DIR, name="explore_ppf{}".format(n)) print("Interpolating...") interpolate_digits(model, mnist) print("Plotting end-to-end reconstructions...") plot_all_end_to_end(model, mnist) print("Morphing...") morph_numbers(model, mnist, ns=[9, 8, 7, 6, 5, 4, 3, 2, 1, 0]) print("Plotting 10 MNIST digits...") for i in range(10): plot.justMNIST(get_mnist(i, mnist), name=str(i), outdir=PLOTS_DIR)
def main(): # get training parameters with open('params.json') as f: gan_params = json.load(f) print('--params--') pprint(gan_params) dataset_base_dir = './data_set' for param in gan_params: model_name = param["model-name"] epochs = int(param["epochs"]) mnist_type = param["mnist-type"] mnist = utils.get_mnist(dataset_base_dir, mnist_type) print('Training {:s} with epochs: {:d}, dataset: {:s}'.format( model_name, epochs, mnist_type)) # get appropriate module and it's class to start training module_name = import_module(model_name) gan_class = getattr(module_name, model_name.upper()) net = gan_class(model_name, mnist_type, mnist, epochs) net.train() return
def morph_numbers(model, mnist, ns=None, n_per_morph=10): if not ns: import random ns = random.sample(range(10), 10) # non-in-place shuffle xs = np.squeeze([get_mnist(n, mnist) for n in ns]) mus, _ = model.encode(xs) plot.morph(model, mus, n_per_morph=n_per_morph, outdir=PLOTS_DIR, name="morph_{}".format("".join(str(n) for n in ns)))
def apply_opt(self): # dataset if self._opt.dataset == "MNIST": train_data, test_data = utils.get_mnist() self._train_set = torch.utils.data.DataLoader( train_data, batch_size=self._opt.batch_size, shuffle=True, num_workers=self._opt.num_workers) self._test_set = torch.utils.data.DataLoader( test_data, batch_size=self._opt.batch_size, shuffle=True, num_workers=self._opt.num_workers) self._initialize_model(dims=self._opt.layer_dims) print("MNIST experiment") elif self._opt.dataset == "IBNet": train_data = utils.CustomDataset('2017_12_21_16_51_3_275766', train=True) test_data = utils.CustomDataset('2017_12_21_16_51_3_275766', train=False) self._train_set = torch.utils.data.DataLoader( train_data, batch_size=self._opt.batch_size, shuffle=True, num_workers=self._opt.num_workers) self._test_set = torch.utils.data.DataLoader( test_data, batch_size=self._opt.batch_size, shuffle=True, num_workers=self._opt.num_workers) self._initialize_model(dims=self._opt.layer_dims) print("IBnet experiment") else: raise RuntimeError( 'Do not have {name} dataset, Please be sure to use the existing dataset' .format(name=self._opt.dataset)) # construct saving directory save_root_dir = self._opt.save_root_dir dataset = self._opt.dataset time = datetime.datetime.today().strftime('%m_%d_%H_%M') model = ''.join( list(map(lambda x: str(x) + '_', self._model.layer_dims))) folder_name = dataset + '_' + self._opt.experiment_name + '_Time_' + time + '_Model_' + model self._path_to_dir = save_root_dir + '/' + folder_name + '/' print(self._path_to_dir) if not os.path.exists(self._path_to_dir): os.makedirs(self._path_to_dir) self._logger = Logger(opt=self._opt, plot_name=folder_name) self._json = JsonParser()
def __init__(self): ''' ''' self.BATCH_SIZE = 32 self.EPOCHS = 100000 # self.image_dim = [-1, 32, 32, 3] # CIFAR-10 self.image_dim = [-1, 28, 28, 1] # MNIST # self.discriminator_input_dim = (None, 32, 32, 3) # CIFAR-10 self.discriminator_input_dim = (None, 784) # MNIST self.net = Network() # self.data = utils.get_cifar10() # CIFAR-10 self.data = utils.get_mnist() # MNIST # self.logdir = "train_logs/cifar10/" # CIFAR-10 self.logdir = "train_logs/mnist/" # MNIST
def mnist(): X, Y = utils.get_mnist() print ("y shape", Y.shape, Y[0:10]) Y = None X = X.reshape(len(X), 28,28, 1) dim = X.shape[1] colors = X.shape[-1] # mnist d_sizes = {"conv_layers":[(2,5,2,False), (64,5,2,True)], "dense_layers":[(1024,True)]} g_sizes = {"z":100, "projection": 128, "bn_after_project": False, "conv_layers": [(128,5,2,True),(colors,5,2,False)], "dense_layers":[(1024,True)],"output_activation": tf.sigmoid} gan = DCGAN(dim, colors, d_sizes, g_sizes) gan.fit(X)
def main(): # Load data train, test = get_mnist() # Initialize iterators train_iter = iterators.SerialIterator(train, batch_size=32, shuffle=True) val_iter = iterators.SerialIterator(test, batch_size=50, repeat=False, shuffle=False) # Define model model = MLP(10, 10) optimizer = optimizers.SGD() optimizer.setup(model) training_losses, validation_losses = run(train_iter, val_iter, test, model, optimizer, 200)
def __init__(self): self.progress_bar = 0 self._device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # device setup load_config = JsonParser() # training args self.model_name = 'IBNet_test_save_Time_05_27_20_09_Model_12_12_10_7_5_4_3_2_2_' self.path =os.path.join('./results', self.model_name)# info plane dir self._opt = load_config.read_json_as_argparse(self.path) # load training args # force the batch size to 1 for calculation convinience self._opt.batch_size = 1 # dataset if self._opt.dataset == "MNIST": train_data, test_data = utils.get_mnist() if not self._opt.full_mi: # self._train_set = torch.utils.data.DataLoader(train_data, batch_size=1, shuffle=False, num_workers=0) self._test_set = torch.utils.data.DataLoader(test_data, batch_size=1, shuffle=False, num_workers=0) else: dataset = torch.utils.data.ConcatDataset([train_data, test_data]) self._test_set = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False, num_workers=0) print("MNIST experiment") elif self._opt.dataset == "IBNet": train_data = utils.CustomDataset('2017_12_21_16_51_3_275766', train=True) test_data = utils.CustomDataset('2017_12_21_16_51_3_275766', train=False) # self._train_set = torch.utils.data.DataLoader(train_data, batch_size=1, shuffle=False, num_workers=0) if not self._opt.full_mi: self._test_set = torch.utils.data.DataLoader(test_data, batch_size=1, shuffle=False, num_workers=0) else: dataset = torch.utils.data.ConcatDataset([train_data, test_data]) self._test_set = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False, num_workers=0) print("IBnet experiment") else: raise RuntimeError('Do not have {name} dataset, Please be sure to use the existing dataset'.format(name = self._opt.dataset)) # get model self._model = Model(activation = self._opt.activation ,dims = self._opt.layer_dims, train = False) # get measure # self.measure = measure.kde() self.measure = measure.EVKL() # our new measure
def main(): Xtrain, Ytrain, Xtest, Ytest = get_mnist() pca = PCA() reduced = pca.fit_transform(Xtrain) plt.scatter(reduced[:, 0], reduced[:, 1], s=100, c=Ytrain, alpha=.5) plt.show() plt.plot(pca.explained_variance_ratio_) plt.show() #cumulative variance #choose k = number of dimensions that gives us 95%-99% variance cumulative = [] last = 0 for v in pca.explained_variance_ratio_: cumulative.append(last + v) last = cumulative[-1] plt.plot(cumulative) plt.show()
data[j] = 1 else: data[j] = -1 return data def binarize(_arr): arr = np.float32(_arr) arr[arr <= 0] = -1 arr[arr > 0] = 1 return arr if __name__ == '__main__': # prepare initial data data1, target1 = get_mnist(0, 1) data1 = data1.ravel() data = np.vstack([data1]) data = binarize(data) net = HopfieldNet(28 * 28) net.store(data) # get noisy example example = binarize(data1) noise_idx = np.random.choice(range(28 * 28), 250) example[noise_idx] *= -1 plt.imshow(example.reshape(28, 28), cmap='gray', interpolation='none') plt.xticks([]) plt.yticks([])
""" def __init__(self, predictor): super(Classifier, self).__init__() with self.init_scope(): self.predictor = predictor def __call__(self, x, t): y = self.predictor(x) loss = F.softmax_cross_entropy(y, t) accuracy = F.accuracy(y, t) report({'loss': loss, 'accuracy': accuracy}, self) return loss # Retrieve train & test data train, test = get_mnist() # split test inputs and labels inputs, labels = np.array([tup[0] for tup in test ]), np.array([tup[1] for tup in test]) # Set up model & classifier model = MLP(10, 10) classifier = Classifier(model) optimizer = optimizers.SGD() optimizer.setup(classifier) # Implement the training loop iterator = RandomIterator(train, 32) av_loss = [] ep_loss = [] test_loss = []
# -*- coding: utf-8 -*- """ Created on Mon Apr 27 09:04:38 2020 @author: MY PC """ from models import ML_MultiLogisticRegression from utils import get_mnist, One_Hot_Encode mnist_data = get_mnist() X = mnist_data[:, 1:] Y = mnist_data[:, 0] Y = One_Hot_Encode(Y) X_train = X[0:int(X.shape[0] * 0.9)] Y_train = Y[0:int(X.shape[0] * 0.9)] X_test = X[int(X.shape[0] * 0.9):] Y_test = Y[int(X.shape[0] * 0.9):] X_train /= 255.0 model = ML_MultiLogisticRegression() model.fit(X_train, Y_train, epochs=800, showfig=True, learning_rate=0.1) X_test /= 255.0 test_acc = model.evaluate(X_test, Y_test)
plt.xlabel("Epoch") plt.show() for i in range(4): gen_input = np.float32(np.random.uniform(size=[1, 1])) generation = generative_net( gen_input ) # we need to keep the variable type around, to compute stuff plt.imshow(np.reshape(generation.data, newshape=[28, 28]).transpose()) plt.show() if __name__ == "__main__": n_iter = 500 batch_size = 50 train_data, test_data = get_mnist(n_train=1000, n_test=100, with_label=False, classes=[0], n_dim=3) train_iter = RandomIterator(train_data, batch_size) test_iter = RandomIterator(test_data, batch_size) discriminative_net = networks.DiscriminativeMLP(n_hidden=20) generative_net = networks.GenerativeMLP(n_hidden=200) discriminative_net = networks.DiscriminativeConvolutional().to_gpu() generative_net = networks.GenerativeDeconvolutional(256).to_gpu() train()
# data = (np.float32(data) / 255.).reshape(64, 28, 28) # data -= np.mean(data) # for i in xrange(64): # print i # coder = SpatialCoder(data[i]) # # result = coder.gabor_encode() # result = coder.gabor_convolve() # # result = coder.hog_encode() # result = coder.cluster_lvl1(result) # plt.subplot(8, 8, i) # plt.xticks(()) # plt.yticks(()) # plt.imshow(result, interpolation='none') # plt.show() data, target = get_mnist(20000, 20001) data = (np.float32(data) / 255.).reshape(28, 28) data -= np.mean(data) coder = SpatialCoder(data) omap = coder.gabor_convolve() # result = coder.cluster_lvl1(omap) # plt.subplot(2, 1, 0) # plt.xticks(()) # plt.yticks(()) plt.imshow(omap, interpolation='none') # plt.subplot(2, 1, 1) # plt.xticks(()) # plt.yticks(()) # plt.imshow(result, interpolation='none') plt.show()
def get_preprocessed_mnist(start, end): data, target = get_mnist(start, end) return binarize(np.vstack([ resize(item.reshape(28, 28), (dim, dim)).ravel() for item in data ]))
y[i] = max_value_count return y # Score function gives accuracy of prediction def score(self, X, y): pred = self.predict(X) return np.mean( y == pred) if __name__ == '__main__': # getting the data (NOTE: I have used only 2000 data point.) # You can change this X, y = get_mnist(2000) Ntrain = int(0.7 * len(X)) # Train and Test splits Xtrain, ytrain = X[:Ntrain], y[:Ntrain] Xtest, ytest = X[Ntrain:], y[Ntrain:] train_score = [] test_score = [] # choosing optimal K is difficult # I have used K = 1 to 5 # You can change K as per your requirement for k in (1, 2, 3, 4, 5): t0 = datetime.now()
# ROFFO GIORGIO from __future__ import print_function, division from builtins import range, input import matplotlib.pyplot as plt import warnings warnings.filterwarnings("ignore", category=FutureWarning) from Classifier import BayesGMM from utils import get_mnist # Load a dataset: in this case I'm working on MNIST X, Y = get_mnist() # Initialize the Bayesian Classifier wit Gaussian Mixture Models clf = BayesGMM() # Train the classifier clf.fit(X, Y) # show one sample for each class # also show the mean image learned for k in range(clf.K): sample, mean = clf.sample_given_y(k) plt.subplot(1, 2, 1) plt.imshow(sample, cmap='gray') plt.title("Sample")
def main(): train_x, train_y, valid_x, valid_y, test_x, test_y = get_mnist() num_epochs = args.epochs eta = args.lr batch_size = args.batch_size # input x = T.matrix("x") y = T.ivector("y") #x.tag.test_value = np.random.randn(3, 784).astype("float32") #y.tag.test_value = np.array([1,2,3]) #drop_switch.tag.test_value = 0 #import ipdb; ipdb.set_trace() hidden_1 = BinaryDense(input=x, n_in=784, n_out=2048, name="hidden_1") act_1 = Activation(input=hidden_1.output, activation="relu", name="act_1") hidden_2 = BinaryDense(input=act_1.output, n_in=2048, n_out=2048, name="hidden_2") act_2 = Activation(input=hidden_2.output, activation="relu", name="act_2") hidden_3 = BinaryDense(input=act_2.output, n_in=2048, n_out=2048, name="hidden_3") act_3 = Activation(input=hidden_3.output, activation="relu", name="act_3") output = BinaryDense(input=act_3.output, n_in=2048, n_out=10, name="output") softmax = Activation(input=output.output, activation="softmax", name="softmax") # loss xent = T.nnet.nnet.categorical_crossentropy(softmax.output, y) cost = xent.mean() # errors y_pred = T.argmax(softmax.output, axis=1) errors = T.mean(T.neq(y, y_pred)) # updates + clipping (+-1) params_bin = hidden_1.params_bin + hidden_2.params_bin + hidden_3.params_bin params = hidden_1.params + hidden_2.params + hidden_3.params grads = [T.grad(cost, param) for param in params_bin] # calculate grad w.r.t binary parameters updates = [] for p, g in zip( params, grads ): # gradient update on full precision weights (NOT binarized wts) updates.append((p, clip_weights(p - eta * g)) #sgd + clipping update ) # compiling train, predict and test fxns train = theano.function(inputs=[x, y], outputs=cost, updates=updates) predict = theano.function(inputs=[x], outputs=y_pred) test = theano.function(inputs=[x, y], outputs=errors) # train checkpoint = ModelCheckpoint(folder="snapshots") logger = Logger("logs/{}".format(time())) for epoch in range(num_epochs): print "Epoch: ", epoch print "LR: ", eta epoch_hist = {"loss": []} t = tqdm(range(0, len(train_x), batch_size)) for lower in t: upper = min(len(train_x), lower + batch_size) loss = train(train_x[lower:upper], train_y[lower:upper].astype(np.int32)) t.set_postfix(loss="{:.2f}".format(float(loss))) epoch_hist["loss"].append(loss.astype(np.float32)) # epoch loss average_loss = sum(epoch_hist["loss"]) / len(epoch_hist["loss"]) t.set_postfix(loss="{:.2f}".format(float(average_loss))) logger.log_scalar(tag="Training Loss", value=average_loss, step=epoch) # validation accuracy val_acc = 1.0 - test(valid_x, valid_y.astype(np.int32)) print "Validation Accuracy: ", val_acc logger.log_scalar(tag="Validation Accuracy", value=val_acc, step=epoch) checkpoint.check(val_acc, params) # Report Results on test set best_val_acc_filename = checkpoint.best_val_acc_filename print "Using ", best_val_acc_filename, " to calculate best test acc." load_model(path=best_val_acc_filename, params=params) test_acc = 1.0 - test(test_x, test_y.astype(np.int32)) print "Test accuracy: ", test_acc
help="initialize data augmentation", action='store_true') parser.add_argument('-s', "--dataset", help="choose 'mnist' or 'cifar10' ") args = parser.parse_args() with open('config/model.json') as f: model_config = json.load(f) with open('config/training.json') as f: train_config = json.load(f) if __name__ == "__main__": model = get_SimpleNet(model_config['0']) if args.dataset == 'mnist': X_train, Y_train, X_test, Y_test = get_mnist(model_config['0']) else: X_train, Y_train, X_test, Y_test = get_cifar10(model_config['0']) train_batch_size = train_config['train_batch_size'] train_validation_split = train_config['train_validation_split'] epochs = train_config['epochs'] if args.data_aug: #Image Data Generator datagen = ImageDataGenerator(width_shift_range=0.2, horizontal_flip=True, height_shift_range=0.2, rotation_range=20) datagen.fit(X_train) checkpointer = ModelCheckpoint(filepath='saved_models/' + model.name +
import pandas import datetime as dt from nn.layer import * from nn.criterion import * from optim.optimizer import * from util.common import * from dataset.transformer import * from dataset import mnist from utils import get_mnist init_engine() # Get and store MNIST into RDD of Sample, please edit the "mnist_path" accordingly. mnist_path = "datasets/mnist" (train_data, test_data) = get_mnist(sc, mnist_path) print train_data.count() print test_data.count() # Parameters learning_rate = 0.2 training_epochs = 15 batch_size = 2048 display_step = 1 # Network Parameters n_hidden_1 = 256 # 1st layer number of features n_hidden_2 = 256 # 2nd layer number of features n_input = 784 # MNIST data input (img shape: 28*28) n_classes = 10 # MNIST total classes (0-9 digits) def multilayer_perceptron(n_hidden_1, n_hidden_2, n_input, n_classes):
import Encoder as e import math from sklearn import neighbors, datasets from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score n_epoch = 20 n_train_per_class = 5000 batchsize = 128 weight_decay = 0.000025 initial_alpha = 0.0002 final_alpha = 0.000002 beta_1 = 0.5 beta_2 = 0.999 latent_dim = 50 train_data, test_data = u.get_mnist(n_train=n_train_per_class, n_test=100, with_label=True, classes=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) #train_data, test_data = chsets.get_mnist() # get the correct total length n_train = train_data._length # the discriminator is tasked with classifying examples as real or fake Disc = CustomClassifier(predictor=d.Discriminator(latent_dim), lossfun=f.sigmoid_cross_entropy) Disc.compute_accuracy = False Gen = g.Generator() Enc = e.Encoder(latent_dim) # Use Adam optimizer # learning rate, beta1, beta2 disc_optimizer = optimizers.Adam(initial_alpha, beta1=beta_1, beta2=beta_2)
from utils import get_mnist, sample_reconstructions, plot_samples from vae import SampleLayer, vae_log_likelihood import theano import theano.tensor as T import lasagne from lasagne.nonlinearities import rectify, identity import numpy as np from sklearn.cross_validation import train_test_split if __name__ == '__main__': data = get_mnist() train, test = train_test_split(data, test_size=0.1) _train = theano.shared(train, borrow=True) _test = theano.shared(test, borrow=True) batch_size = 500 latent_size = 2 target = T.matrix() encoder = lasagne.layers.InputLayer((None, train.shape[1]), target) encoder = lasagne.layers.DenseLayer(encoder, num_units=100, nonlinearity=rectify) mean = lasagne.layers.DenseLayer(encoder, num_units=latent_size, nonlinearity=identity) log_sigma = lasagne.layers.DenseLayer(encoder, num_units=latent_size, nonlinearity=identity) z = SampleLayer(mean=mean, log_sigma=log_sigma) decoder1 = lasagne.layers.DenseLayer(z, num_units=100, nonlinearity=rectify)
os.makedirs( figsdir, exist_ok=True ) # exist_ok makes function do nothing if directory already exists input_type = 'image' representation_type = 'image' output_type = ['image'] output_dim = [(3, 28, 28)] problem = 'privacy' input_dim = (3, 28, 28) eval_rate = 100 #500 # evaluate on last epoch gamma = 1 epochs = 501 batch_size = 2048 trainset, testset = utils.get_mnist() network = variational_privacy_fairness.VPAF( input_type=input_type, representation_type=representation_type, output_type=output_type, problem=problem, gamma=gamma, input_dim=input_dim, output_dim=output_dim).to(device) network.apply(utils.weight_init ) #apply calls the function recursively to each submodule network.train() network.fit(trainset, testset, epochs=epochs,
cfg = {} cfg['SGD_BATCHSIZE'] = 256 cfg['SGD_LEARNINGRATE'] = 0.001 cfg['NUM_EPOCHS'] = 10000 # changed from 10000 #cfg['ACTIVATION'] = 'relu' cfg['ACTIVATION'] = 'tanh' # How many hidden neurons to put into each of the layers #cfg['LAYER_DIMS'] = [1024, 20, 20, 20] #cfg['LAYER_DIMS'] = [32, 28, 24, 20, 16, 12, 8, 8] cfg['LAYER_DIMS'] = [128, 64, 32, 16, 16] # 0.967 w. 128 #cfg['LAYER_DIMS'] = [20, 20, 20, 20, 20, 20] # 0.967 w. 128 ARCH_NAME = '-'.join(map(str,cfg['LAYER_DIMS'])) trn, tst = utils.get_mnist() # Where to save activation and weights data cfg['SAVE_DIR'] = 'rawdata/' + cfg['ACTIVATION'] + '_' + ARCH_NAME # In[ ]: input_layer = keras.layers.Input((trn.X.shape[1],)) clayer = input_layer for n in cfg['LAYER_DIMS']: clayer = keras.layers.Dense(n, activation=cfg['ACTIVATION'])(clayer) output_layer = keras.layers.Dense(trn.nb_classes, activation='softmax')(clayer) model = keras.models.Model(inputs=input_layer, outputs=output_layer)
redire_spark_logs() show_bigdl_info_logs() init_engine() # Parameters batch_size = int(options.batchSize) learning_rate = float(options.learningRate) learning_rate_decay = float(options.learningrateDecay) if options.action == "train": def get_end_trigger(): if options.endTriggerType.lower() == "epoch": return MaxEpoch(options.endTriggerNum) else: return MaxIteration(options.endTriggerNum) train_data = get_mnist(sc, "train", options.dataPath)\ .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD),rec_tuple[1])).map(lambda t: Sample.from_ndarray(t[0], t[1])) test_data = get_mnist(sc, "test", options.dataPath)\ .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TEST_MEAN, mnist.TEST_STD), rec_tuple[1])).map(lambda t: Sample.from_ndarray(t[0], t[1])) train_data = train_data.sample(False, float(options.trainingSetSize)) optimizer = Optimizer(model=build_model(n_input, n_hidden, n_classes), training_rdd=train_data, criterion=CrossEntropyCriterion(), optim_method=SGD( learningrate=learning_rate, learningrate_decay=learning_rate_decay), end_trigger=get_end_trigger(), batch_size=options.batchSize) optimizer.set_validation(batch_size=options.batchSize, val_rdd=test_data,
if acc is not None: log += ', accuracy %.5f' % (acc) print(log) if batch_number != 0 and delta <= threshold: return idx = current_slice() p = target_prediction(dec_model, x) loss = dec_model.train_on_batch(x[idx], p[idx]) batch_number += 1 if __name__ == '__main__': X, Y = utils.get_mnist() print("MNIST loaded") encoder = utils.load_model('encoder') print("Encoder loaded") fl = False try: dec_model = utils.load_model('dec') print('DEC loaded') except EnvironmentError: dec_model = get_model(encoder, X) print('Training DEC') train_dec(dec_model, X, Y)
]) model.compile(loss='mse', optimizer=SGD(lr=lr, decay=0, momentum=momentum)) model.fit(outputs[i], outputs[i], batch_size=batch_size, epochs=pt_epochs, callbacks=[lr_schedule], verbose=1) enc_model = Sequential([enc_i]) enc_model.compile(loss='mse', optimizer=SGD(lr=lr, decay=0, momentum=momentum)) out = enc_model.predict(outputs[i]) outputs.append(out) autoencoder = Sequential(encoders + decoders[::-1]) autoencoder.compile(loss='mse', optimizer=SGD(lr=lr, decay=0, momentum=momentum)) autoencoder.fit(x, x, batch_size=batch_size, epochs=ae_epochs) encoder = Sequential(encoders) encoder.compile(loss='mse', optimizer=SGD(lr=lr, decay=0, momentum=momentum)) return encoder if __name__ == '__main__': X, Y = utils.get_mnist() if TESTING: X, Y = utils.get_mnist(35000) encoder = train_autoencoder(X) utils.save_model('encoder', encoder) print("Encoder trained and stored")
def sample(self, arr): return np.random.binomial(n=1, p=arr) def sample_h_given_v(self, v): probs = self.propup(v) states = self.sample(probs) return probs, states def sample_v_given_h(self, h): probs = self.propdown(h) states = self.sample(probs) return probs, states if __name__ == '__main__': data, target = get_mnist(random=True, num=1000) data = binarize(data, 0., 1.) dim = 28 h = 100 x = 10 y = 10 rbm = BernoulliRBM( dim * dim, h, learning_rate=0.1, sparsity_target=0.05, regulatization_param=0.01 ) rbm.train(data, epochs=1000) for i in xrange(h):
def train(batch_size, Epochs = float("Inf"), L_times = {}, weights = {"L_TID":15, "L_CONST":15}, save = True, cont = False, plt_ = True , hours = float("Inf"), lr = 0.0003, show = False): base = f"./models/trainings/{utils.get_local_time()}" params = locals() os.mkdir(base) with open(f"{base}/params.txt", 'wt') as f: f.write(str(params)) with open(f"{base}/params_dict", 'wb') as f: pickle.dump(params, f) hours = hours * (60 * 60) # sec in hour for k in L_times.keys(): if isinstance(k, int): L_times[k] = lambda x: L_times[k] if cont: g = torch.load(f"{cont}/g_net") D = torch.load(f"{cont}/D_net") else: g = Nets.g_net D = Nets.D_net g, D = g.to(device), D.to(device) f = Nets.f_net_features.to(device) opt_g = optim.Adam(g.parameters(), lr= lr, betas=(0.5, 0.999)) opt_D = optim.Adam(D.parameters(), lr= lr, betas=(0.5, 0.999)) opts = {'g':opt_g, 'D':opt_D} dl = {'s':utils.get_svhn(batch_size) , 't':utils.get_mnist(batch_size)} dl_test = utils.get_svhn(32, "test") dl_test_mnist = utils.get_mnist(32, "test") names = ['L_GANG', 'L_CONST', 'L_D', 'L_TID'] prob_types = ['trans', 'real_trans', 'real'] cum_norm, cum_loss, n = {c:0 for c in names}, {c:[] for c in names}, {c:0 for c in names} cum_prob = {typ:[] for typ in prob_types} def myplt(path = None): g.eval() ; D.eval() print(f" >>>> Epoch {e} - Svhn") plt.figure(figsize = (15, 10)) x ,_ = next(iter(dl_test)) x = x[:4, :, :, :] utils.plt_row_images(x.cpu()) if path: plt.savefig(f"{path}/svhn.jpg") if show: plt.show() plt.figure(figsize = (15, 10)) x = x.to(device) y = g(f(x)) utils.plt_row_images(y.cpu()) if path: plt.savefig(f"{path}/svhn_G.jpg") if show: plt.show() print(f" >>>> Epoch {e} - Mnist") plt.figure(figsize = (15, 10)) x ,_ = next(iter(dl_test_mnist)) x = x[:4, :, :, :] utils.plt_row_images(x.cpu()) if path: plt.savefig(f"{path}/mnist.jpg") if show: plt.show() plt.figure(figsize = (15, 10)) x = x.to(device) with torch.no_grad(): y = g(f(x)) utils.plt_row_images(y.cpu()) if path: plt.savefig(f"{path}/mnist_G.jpg") if show: plt.show() plt.figure(figsize = (15, 10)) for i, name in enumerate(names): plt.subplot(2,2,i+1) plt.title(name) cp = cum_loss[name] if len(cp) > 200: cp = cp[50:] plt.plot(cp) if path: plt.savefig(f"{path}/loss.jpg") if show: plt.show() plt.figure(figsize = (15, 10)) for typ in prob_types: cp = cum_prob[typ] plt.plot(cp, label = typ) plt.legend() plt.title("Disc Probs") if path: plt.savefig(f"{path}/probs.jpg") if show: plt.show() g.train() ; D.train() def tr_step(name, x, times = 1): c = 'D' if n in ['L_D'] else 'g' if times == 0: cum_loss[name].append(cum_loss[name][-1] if len(cum_loss[name]) else 0) return closs = 0 cp = {typ:0 for typ in prob_types} for i in range(times): loss = get_loss(name, x, f, g, D, weight = weights.get(name, 1)) if name == "L_D": loss, p = loss for typ in p: cp[typ] += p[typ].cpu().item() closs += loss opts[c].zero_grad() ; loss.backward() ; opts[c].step() cum_loss[name].append(closs.cpu().item() / times) if name == "L_D": for typ in prob_types: cum_prob[typ].append(cp[typ] / times) n[name] += times start_time = time.time() e = 0 while True: x = {} iters = {k:iter(dl[k]) for k in dl} i = 0 while get_next_batch(x, iters): for k in x: x[k] = x[k].to(device) tr_step('L_D', x, times = L_times.get("L_D", lambda x: 1)(i)) tr_step('L_GANG', x, times = L_times.get("L_GANG", lambda x: 1)(i)) tr_step('L_CONST', x, times = L_times.get("L_CONST", lambda x: 0 if x % 10 else 1)(i)) tr_step('L_TID', x, times = L_times.get("L_TID", lambda x: 1)(i)) i += 1 if i % 500 == 0 and i > 1: myplt() print(f">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> EPOCH {e} END") if save and e > 0 and e % 5 == 0: os.mkdir(f"{base}/{e}") myplt(path = f"{base}/{e}") for name, model in [("g_net", g), ("D_net", D)]: torch.save(model.cpu(), f"{base}/{e}/{name}") model.to(device) print(f"CP -- models saved to {base}/{e}/{name}") e += 1 if e > Epochs: break if (time.time() - start_time) > hours: break if save: for name, model in [("g_net", g), ("D_net", D)]: torch.save(model.cpu(), f"{base}/{name}") utils.rmdir_if_empty(base)