def test_get_data(self): (x_train, y_train), (x_val, y_val), (x_test, y_test) = mnist.get_data() self.assertEqual(x_train.shape, (50000, 28*28)) self.assertEqual(y_train.shape, (50000, 10)) self.assertEqual(x_val.shape, (10000, 28*28)) self.assertEqual(y_val.shape, (10000, 10)) self.assertEqual(x_test.shape, (10000, 28*28)) self.assertEqual(y_test.shape, (10000, 10))
def visualize(device, num_quadrant_inputs, pre_trained_baseline, pre_trained_cvae, num_images, num_samples, image_path=None): # Load sample random data datasets, _, dataset_sizes = get_data( num_quadrant_inputs=num_quadrant_inputs, batch_size=num_images ) dataloader = DataLoader(datasets['val'], batch_size=num_images, shuffle=True) batch = next(iter(dataloader)) inputs = batch['input'].to(device) outputs = batch['output'].to(device) originals = batch['original'].to(device) # Make predictions with torch.no_grad(): baseline_preds = pre_trained_baseline(inputs).view(outputs.shape) predictive = Predictive(pre_trained_cvae.model, guide=pre_trained_cvae.guide, num_samples=num_samples) cvae_preds = predictive(inputs)['y'].view(num_samples, num_images, 28, 28) # Predictions are only made in the pixels not masked. This completes # the input quadrant with the prediction for the missing quadrants, for # visualization purpose baseline_preds[outputs == -1] = inputs[outputs == -1] for i in range(cvae_preds.shape[0]): cvae_preds[i][outputs == -1] = inputs[outputs == -1] # adjust tensor sizes inputs = inputs.unsqueeze(1) inputs[inputs == -1] = 1 baseline_preds = baseline_preds.unsqueeze(1) cvae_preds = cvae_preds.view(-1, 28, 28).unsqueeze(1) # make grids inputs_tensor = make_grid(inputs, nrow=num_images, padding=0) originals_tensor = make_grid(originals, nrow=num_images, padding=0) separator_tensor = torch.ones((3, 5, originals_tensor.shape[-1])).to(device) baseline_tensor = make_grid(baseline_preds, nrow=num_images, padding=0) cvae_tensor = make_grid(cvae_preds, nrow=num_images, padding=0) # add vertical and horizontal lines for tensor in [originals_tensor, baseline_tensor, cvae_tensor]: for i in range(num_images - 1): tensor[:, :, (i + 1) * 28] = 0.3 for i in range(num_samples - 1): cvae_tensor[:, (i + 1) * 28, :] = 0.3 # concatenate all tensors grid_tensor = torch.cat([inputs_tensor, separator_tensor, originals_tensor, separator_tensor, baseline_tensor, separator_tensor, cvae_tensor], dim=1) # plot tensors imshow(grid_tensor, image_path=image_path)
def generate_table( device, num_quadrant_inputs, pre_trained_baseline, pre_trained_cvae, num_particles, col_name, ): # Load sample random data datasets, dataloaders, dataset_sizes = get_data( num_quadrant_inputs=num_quadrant_inputs, batch_size=32 ) # Load sample data criterion = MaskedBCELoss() loss_fn = Trace_ELBO(num_particles=num_particles).differentiable_loss baseline_cll = 0.0 cvae_mc_cll = 0.0 num_preds = 0 df = pd.DataFrame(index=["NN (baseline)", "CVAE (Monte Carlo)"], columns=[col_name]) # Iterate over data. bar = tqdm(dataloaders["val"], desc="Generating predictions".ljust(20)) for batch in bar: inputs = batch["input"].to(device) outputs = batch["output"].to(device) num_preds += 1 # Compute negative log likelihood for the baseline NN with torch.no_grad(): preds = pre_trained_baseline(inputs) baseline_cll += criterion(preds, outputs).item() / inputs.size(0) # Compute the negative conditional log likelihood for the CVAE cvae_mc_cll += loss_fn( pre_trained_cvae.model, pre_trained_cvae.guide, inputs, outputs ).detach().item() / inputs.size(0) df.iloc[0, 0] = baseline_cll / num_preds df.iloc[1, 0] = cvae_mc_cll / num_preds return df
import matplotlib.pyplot as plt import numpy as np from keras.models import Sequential from keras.layers import Dense from sklearn.linear_model import LogisticRegression import wisconsin_cancer import mnist np.random.seed(10) cancer_data = wisconsin_cancer.get_data(0.7, 0.2) mnistData = mnist.get_data(0.7, 0.2) datasets = [[mnistData, "MNIST"], [cancer_data, "Wisconsin cancer data"]] for dataset, name in datasets: x_train = dataset['x_train'] y_train = dataset['y_train'] x_test = dataset['x_test'] y_test = dataset['y_test'] model = Sequential() model.add(Dense(x_train.shape[1], activation='relu')) model.add(Dense(32, activation='relu')) model.add(Dense(y_train.shape[1], activation='softmax')) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) hist = model.fit(x_train,
RS = 20150101 # We'll use matplotlib for graphics. import matplotlib.pyplot as plt import matplotlib.patheffects as PathEffects import matplotlib #%matplotlib inline #We import seaborn to make nice plots. sns.set_style('darkgrid') sns.set_palette('muted') sns.set_context("notebook", font_scale=1.5, rc={"lines.linewidth": 2.5}) N = 10000 #Number of data we want to use in this project data, target = mn.get_data(N) #First use PCA to reduce data into 50 dimensions to speed up computation pca = PCA(n_components=50) pca.fit(data) W = pca.components_ pre_data = np.matmul(data, W.T) ###TSNE### digits_proj = TSNE(n_components=2, random_state=RS).fit_transform(pre_data) mn.scatter(digits_proj, target) ##Isomap##
import sys, os sys.path.append(os.pardir) # 親ディレクトリのファイルをインポートするための設定 import numpy as np import pickle from dataset.mnist import load_mnist from mnist import get_data, init_network, predict x, t = get_data() network = init_network() batch_size = 100 accuracy_cnt = 0 for i in range(0, len(x), batch_size): x_batch = x[i:i + batch_size] y_batch = predict(network, x_batch) p = np.argmax(y_batch, axis=1) accuracy_cnt += np.sum(p == t[i:i + batch_size]) print("Accuracy: " + str(float(accuracy_cnt) / len(x)))
# Compute the negative conditional log likelihood for the CVAE cvae_mc_cll += loss_fn(pre_trained_cvae.model, pre_trained_cvae.guide, inputs, outputs).detach().item() / inputs.size(0) df.iloc[0, 0] = baseline_cll / num_preds df.iloc[1, 0] = cvae_mc_cll / num_preds return df if __name__ == '__main__': device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # Dataset datasets, dataloaders, dataset_sizes = get_data(num_quadrant_inputs=1, batch_size=128) baseline_net = BaselineNet(500, 500) baseline_net.load_state_dict( torch.load('/Users/carlossouza/Downloads/baseline_net_q1.pth', map_location='cpu')) baseline_net.eval() cvae_net = CVAE(200, 500, 500, baseline_net) cvae_net.load_state_dict( torch.load('/Users/carlossouza/Downloads/cvae_net_q1.pth', map_location='cpu')) cvae_net.eval() visualize(device=device, num_quadrant_inputs=1, pre_trained_baseline=baseline_net,
help="Number of images to generate (uses the first num_images of the test set)" ) args = parser.parse_args() output_f = args.output_folder os.makedirs(output_f,exist_ok=True) num_bits = args.num_bits num_images = 32 full_model = keras.models.load_model(args.model,compile=False, custom_objects=special.custom_objs) encoder = full_model.get_layer('encoder') latent = full_model.get_layer('latent_params') latent_sampler = full_model.get_layer('geosampler') decoder = full_model.get_layer('decoder') x_train,x_test,_,_ = mnist.get_data() input_data = x_test[:num_images] encoder_out = encoder.predict(input_data) latent_params = latent.predict(encoder_out) latent_out = latent_sampler.predict(latent_params) # decoded_latent = decoder.predict(latent_out) def predict_partial(num_bits): masked_vals = latent_out.copy() masked_vals[:,num_bits:] = 0 return decoder.predict(masked_vals) results_list = [] for bit_num in range(1,num_bits+1): results_list.append(predict_partial(bit_num)[...,0])
def mnist_restore_test(): images, labels = mnist.get_data() images = mnist.im2binary(images) # plt.imshow(images[0,:,:], cmap='gray') plt.rcParams.update({'font.size': 11}) save_path = 'data/mnist_restore_test/' os.makedirs(save_path, exist_ok=True) # base parameters iterations = 100 num_images = 10 annealing = [(8., 100)] coocurrence = (annealing[-1][0], 10) synchron_update = True noise_probability = 0.8 # test_restore345(images, labels, iterations, num_images, annealing, coocurrence, synchron_update, noise_probability, save_path) # variate number of iterations quality_sync = [] quality_async = [] iterationslist = np.arange(10, 151, 10) for iterations in iterationslist: print(f'iterations = {iterations}, sync') hidden_layers, q = test_restore345(images, labels, iterations, num_images, annealing, coocurrence, True, noise_probability, save_path) quality_sync.append(q) print(f'iterations = {iterations}, async') hidden_layers, q = test_restore345(images, labels, iterations, num_images, annealing, coocurrence, False, noise_probability, save_path) quality_async.append(q) learn_epochs = np.sum(np.array(annealing, dtype=np.int), axis=0)[1] cooccur_epochs = coocurrence[1] iterations_as_epochs = iterationslist * (learn_epochs + cooccur_epochs) # sync_text = 'sync' if synchron_update else 'async' title = f'restore quality, {num_images} images, {noise_probability} noise, {hidden_layers} hidden layer, {learn_epochs} learn epochs, {cooccur_epochs} cooccur epochs, T={annealing[-1][0]:.0f}' # plt_quality(title, 'iterations', iterationslist, quality, save_path, xticks=iterationslist) plt_quality(title + ', sync', 'epochs', iterations_as_epochs, [quality_sync], save_path) plt_quality(title + ', async', 'epochs', iterations_as_epochs, [quality_async], save_path) plt_quality(title, 'epochs', iterations_as_epochs, [quality_sync, quality_async], save_path, legend=['synchron', 'asynchron']) # variate temperature iterations = 100 quality_sync = [] quality_async = [] temps = np.arange(1, 21, 1) scan = True # set to False if all data already exists and to just regenerate plot for T in temps: annealing = [(float(T), annealing[-1][1])] coocurrence = (annealing[-1][0], coocurrence[1]) iterationslist = np.arange(10, iterations + 1, 10) if scan: # just generate the intermediate plots and keep the final iterations for i in iterationslist: print(f'T = {T}, iterations = {i}, sync') hidden_layers, q = test_restore345(images, labels, i, num_images, annealing, coocurrence, True, noise_probability, save_path) else: print(f'T = {T}, iterations = {iterations}, sync') hidden_layers, q = test_restore345(images, labels, iterations, num_images, annealing, coocurrence, True, noise_probability, save_path) quality_sync.append(q) if scan: for i in iterationslist: print(f'T = {T}, iterations = {i}, async') hidden_layers, q = test_restore345(images, labels, i, num_images, annealing, coocurrence, False, noise_probability, save_path) else: print(f'T = {T}, iterations = {iterations}, async') hidden_layers, q = test_restore345(images, labels, iterations, num_images, annealing, coocurrence, False, noise_probability, save_path) quality_async.append(q) learn_epochs = np.sum(np.array(annealing, dtype=np.int), axis=0)[1] cooccur_epochs = coocurrence[1] # sync_text = 'sync' if synchron_update else 'async' title = f'restore quality, {iterations} iterations, {num_images} images, {noise_probability} noise, {hidden_layers} hidden layer, {learn_epochs} learn epochs, {cooccur_epochs} cooccur epochs' plt_quality(title + ', sync', 'temperature T', temps, [quality_sync], save_path, xticks=temps) plt_quality(title + ', async', 'temperature T', temps, [quality_async], save_path, xticks=temps) plt_quality_sep_y(title, 'temperature T', temps, [quality_sync, quality_async], save_path, xticks=temps, legend=['synchron', 'asynchron'])
import numpy as np import matplotlib.pyplot as plt import seaborn as sns from sgd import sgd import neural_model import activations import learning_rate import mnist import metrics from sklearn.metrics import confusion_matrix data = mnist.get_data(0.6, 0.2) epochs = 15000 epochs_without_progress = 500 mini_batch_size = 40 total_steps = epochs * len(data['x_train']) // mini_batch_size x_train = data['x_train'] y_train = data['y_train'] x_val = data['x_validate'] y_val = data['y_validate'] x_test = data['x_test'] y_test = data['y_test'] learning_rates = [ learning_rate.Learning_rate(base=base, decay=decay).ramp_up(10).compile(total_steps) for base, decay in [(0.005, 0.0001), (0.002, 5e-05)]
hyperparameters = dict( # "eigh" on covariance matrix or "svd" on data matrix decomposition="svd", # whether to remix after whitening zca=True, # compute fisher based on supervised "loss" or model "output" objective="output", # eigenvalue bias eigenvalue_bias=1e-3, variance_bias=1e-8, batch_normalize=False, whiten_inputs=False, share_parameters=True) datasets = mnist.get_data() features = T.matrix("features") targets = T.ivector("targets") #theano.config.compute_test_value = "warn" #features.tag.test_value = datasets["valid"]["features"] #targets.tag.test_value = datasets["valid"]["targets"] # compilation helpers compile_memo = dict() def compile(variables=(), updates=()): key = (util.tupelo(variables), tuple(OrderedDict(updates).items())) try:
import sys from collections import OrderedDict import numpy as np import theano import theano.tensor as T import util, activation, initialization, steprules, whitening, mnist learning_rate = 1e-3 # use batch normalization in addition to PRONG (i.e. PRONG+) batch_normalize = False data = mnist.get_data() n_outputs = 10 dims = [784, 500, 300, 100, n_outputs] layers = [ dict(f=activation.tanh, c=util.shared_floatx((m,), initialization.constant(0)), # input mean U=util.shared_floatx((m, m), initialization.identity()), # input whitening matrix W=util.shared_floatx((m, n), initialization.orthogonal()), # weight matrix g=util.shared_floatx((n,), initialization.constant(1)), # gammas (for batch normalization) b=util.shared_floatx((n,), initialization.constant(0))) # bias for m, n in util.safezip(dims[:-1], dims[1:])] layers[-1]["f"] = activation.logsoftmax features, targets = T.matrix("features"), T.ivector("targets") #theano.config.compute_test_value = "warn" #features.tag.test_value = data["valid"]["features"][:11] #targets.tag.test_value = data["valid"]["targets"][:11]
#### Main file to import only one time the database #### then keeping the data and running the algo-script #### allowing fast re-running, avoiding long import time. import sys from imp import reload from mnist import get_data import class_algo part1Cache = None if __name__ == "__main__": while True: if not part1Cache: part1Cache = get_data() try: class_algo.main_classiffy(part1Cache) except Exception as e: print(e) print("Press enter to re-run the script, CTRL-C to exit") sys.stdin.readline() try: reload(class_algo) except Exception as e: print(e)
import numpy as np import matplotlib.pyplot as plt from mnist import get_data filename = 'mnist_test.csv' def imshow_data(label: np.ndarray, dataset: np.ndarray) -> None: plt.figure(figsize=(7, 10)) for i in range(1, 10): plt.subplot(3, 3, i) plt.title(f'label = {label[i]}', pad=10) plt.imshow(dataset[i], cmap='gray') plt.axis('off') plt.show() label, dataset = get_data(filename) imshow_data(label, dataset)