import tensorflow as tf

import mnist_data

import tensorflow.contrib.slim as slim

import time

import seaborn as sns

from Assign_Dataset import *

from tensorflow.examples.tutorials.mnist import input_data

from keras.datasets import mnist

from Support import *

from Mnist_DataHandle import *

from HSICSupport import *

from scipy.misc import imsave as ims

from utils import *

import keras

from keras.datasets import reuters

from keras.models import Sequential

from keras.layers import Dense, Dropout, Activation

from keras.preprocessing.text import Tokenizer

distributions = tf.distributions

from Mixture_Models import *

def merge(images, size):

return img

def custom_layer(input_matrix, mix, resue=False):

return outputs

def KL_Dropout2(log_alpha):

return -tf.reduce_sum(mdkl)

# Gateway

def autoencoder(x_hat, x, dim_img, dim_z, n_hidden, keep_prob, last_term,Component_Count):

return y, z, loss, -marginal_likelihood,KL_divergence,dropout_a,dropout_samples

def decoder(z, dim_img, n_hidden):

y = bernoulli_MLP_decoder(z, n_hidden, dim_img, 1.0, reuse=True)

return y

def HiddenOuputs(x_hat, x, dim_img, dim_z, n_hidden, keep_prob, last_term):

return z1_samples, z2_samples, z3_samples, z4_samples,mix

def Final_Output(z1_samples, z2_samples, z3_samples, z4_samples,mix_samples, dim_img, dim_z, n_hidden, keep_prob):

return y

def Final_Output_Confusion(z1_samples,z2_samples,z3_samples,z4_samples,dim_img, dim_z, n_hidden, keep_prob):

return y

def CalculateMSE(x1,x2):

c = tf.square(x1 - x2)

return tf.reduce_mean(c)

n_hidden = 500

IMAGE_SIZE_MNIST = 28

dim_img = IMAGE_SIZE_MNIST ** 2 # number of pixels for a MNIST image

dim_z = 50

# train

n_epochs = 100

batch_size = 128

learn_rate = 0.001

train_total_data, train_size, _, _, test_data, test_labels = mnist_data.prepare_MNIST_data()

n_samples = train_size

# input placeholders

# In denoising-autoencoder, x_hat == x + noise, otherwise x_hat == x

x_hat = tf.placeholder(tf.float32, shape=[None, dim_img], name='input_img')

x = tf.placeholder(tf.float32, shape=[None, dim_img], name='target_img')

# dropout

keep_prob = tf.placeholder(tf.float32, name='keep_prob')

# input for PMLR

z_in = tf.placeholder(tf.float32, shape=[None, dim_z], name='latent_variable')

last_term = tf.placeholder(tf.float32,shape=[10])

Component_Count = tf.placeholder(tf.float32)

# network architecture

y, z, loss, neg_marginal_likelihood, KL_divergence,dropoutA,dropoutSamples = autoencoder(x_hat, x, dim_img, dim_z, n_hidden, keep_prob,

last_term,Component_Count)

z1_samples, z2_samples, z3_samples, z4_samples,mix = HiddenOuputs(x_hat, x, dim_img, dim_z, n_hidden, keep_prob, last_term)

#y1, y2, y3, y4 = Outpiuts_Component(x_hat, x, dim_img, dim_z, n_hidden, keep_prob)

# optimization

t_vars = tf.trainable_variables()

train_op = tf.train.AdamOptimizer(learn_rate).minimize(loss, var_list=t_vars)

# train

total_batch = int(n_samples / batch_size)

min_tot_loss = 1e99

ADD_NOISE = False

train_data2_ = train_total_data[:, :-mnist_data.NUM_LABELS]

train_y = train_total_data[:, 784:784 + mnist_data.NUM_LABELS]

# MNIST dataset

(x_train, y_train), (x_test, y_test) = mnist.load_data()

x_train = x_train.astype('float32') / 255

x_test = x_test.astype('float32') / 255

x_fixed = train_data2_[0:128]

saver = tf.train.Saver()

isWeight = False

with tf.Session() as sess:

sess.run(tf.global_variables_initializer(), feed_dict={keep_prob: 0.9})

lastvalue = np.ones((4))

if isWeight:

saver.restore(sess, 'models/Dropout_Sinmple_4_Gaussian')

bc = 0

'''

x_train = np.reshape(x_train, (-1, 28 * 28))

colorStr = ['blue', 'red', 'black', 'cyan', 'darkcyan', 'darkred', 'green', 'hotpink', 'lightblue', 'magenta']

marks = ['o', 'v', '^', '<', '+', '>', 'x', 'h', 'H', 's']

import matplotlib.pyplot as plt

for i in range(10):

x_l = x_train[i * batch_size:i * batch_size + batch_size]

y_l = y_train[i * batch_size:i * batch_size + batch_size]

z1_samples_, z2_samples_, z3_samples_, z4_samples_, mix_ = sess.run(

(z1_samples, z2_samples, z3_samples, z4_samples, mix),

feed_dict={x_hat: x_l, x: x_l, keep_prob: 0.9})

z1_samples_ = z1_samples_ * mix_[:, 0:1]

z2_samples_ = z2_samples_ * mix_[:, 1:2]

z3_samples_ = z3_samples_ * mix_[:, 2:3]

z4_samples_ = z4_samples_ * mix_[:, 3:4]

for i in range(np.shape(z1_samples_)[0]):

index = y_l[i]

# plt.scatter(z1_samples_[i, 0], z1_samples_[i, 1],color=colorStr[index])

plt.scatter(z1_samples_[i,0], z1_samples_[i,1],color=colorStr[index],marker=marks[index])

# plt.scatter(z2_samples_[i,0], z2_samples_[i,1],color=colorStr[index],marker=marks[index])

# plt.scatter(z3_samples_[i, 0], z3_samples_[i, 1],color=colorStr[index],marker=marks[index])

#plt.scatter(z4_samples_[i, 0], z4_samples_[i, 1], color=colorStr[index], marker=marks[index])

plt.show()

b = 0

'''

tIndex = 5

x_fixed = x_train[batch_size*tIndex:batch_size*tIndex+batch_size]

x_fixed = x_test[batch_size*tIndex:batch_size*tIndex+batch_size]

x_fixed = np.reshape(x_fixed, (-1, 28 * 28))

reco,dropoutA,dropout_Samples, = sess.run((y,dropoutA,dropoutSamples),feed_dict={x_hat: x_fixed, x: x_fixed, keep_prob: 1.0})

reco = np.reshape(reco,(-1,28,28))

tIndex = 7

x_fixed = x_train[batch_size * tIndex:batch_size * tIndex + batch_size]

x_fixed = x_test[batch_size * tIndex:batch_size * tIndex + batch_size]

x_fixed = np.reshape(x_fixed, (-1, 28 * 28))

reco = sess.run(y, feed_dict={x_hat: x_fixed, x: x_fixed, keep_prob: 1.0})

reco = np.reshape(reco, (-1, 28, 28))

x_fixed = np.reshape(x_fixed, (-1, 28, 28))

ims("results/" + "MNIST__Real_Bernoulli" + str(0) + ".png", merge(x_fixed[:64], [8, 8]))

ims("results/" + "MNIST__Reco_Bernoulli" + str(0) + ".png", merge(reco[:64], [8, 8]))

# yy = sess.run(y,feed_dict={x_hat: x_fixed, x: x_fixed, keep_prob: 1.0})

# yy = np.reshape(yy,(-1,28,28))

# ims("results/" + "VAE" + str(0) + ".jpg", merge(yy[:64], [8, 8]))

z_in1 = tf.placeholder(tf.float32, shape=[None, dim_z])

z_in2 = tf.placeholder(tf.float32, shape=[None, dim_z])

z_in3 = tf.placeholder(tf.float32, shape=[None, dim_z])

z_in4 = tf.placeholder(tf.float32, shape=[None, dim_z])

mix_in = tf.placeholder(tf.float32, shape=[None, 4])

x_in1 = tf.placeholder(tf.float32, shape=[None, 28 * 28])

x_in2 = tf.placeholder(tf.float32, shape=[None, 28 * 28])

mse = CalculateMSE(x_in1, x_in2)

yy = Final_Output(z_in1, z_in2, z_in3, z_in4, mix_in, dim_img, dim_z, n_hidden, keep_prob)

x_fixed = np.reshape(x_fixed, (-1, 28 * 28))

z1_samples_, z2_samples_, z3_samples_, z4_samples_, mix_ = sess.run(

(z1_samples, z2_samples, z3_samples, z4_samples, mix),

feed_dict={x_hat: x_fixed, x: x_fixed, keep_prob: 1.0})

y1 = sess.run(yy, feed_dict={z_in1: z1_samples_, z_in2: z2_samples_, z_in3: z3_samples_, z_in4: z4_samples_,

keep_prob: 1.0, mix_in: mix_})

y1 = np.reshape(y1, (-1, 28, 28))

ims("results/" + "AAA" + str(0) + ".png", merge(y1[:64], [8, 8]))

sum1 = 0

for t2 in range(batch_size):

noCount = 0

for t1 in range(4):

if dropout_Samples[t2, t1] < 0.8:

noCount = noCount + 1

yesCount = 4 - noCount

sum1 = sum1 + yesCount

sum1 = float(sum1 / batch_size)

y1 = np.reshape(y1, (-1, 28 * 28))

reco = np.reshape(reco, (-1, 28 * 28))

mse1 = sess.run(mse, feed_dict={x_in1: x_fixed, x_in2: reco})

mse2 = sess.run(mse, feed_dict={x_in1: x_fixed, x_in2: y1})

mse1 = mse1 * 28 * 28

mse2 = mse2 * 28 * 28

my = dropoutA

bc = 0

#training

lastvalue = np.ones((10))

valid_reco = Give_Reconstruction(x_hat, x)

x_valid = x_test[0:batch_size]

x_valid = np.reshape(x_valid, (-1, 28 * 28))

bestScore = 100

x_train = np.reshape(x_train, (-1, 28 * 28))

x_fixed = x_train[0:batch_size]

x_fixed = np.reshape(x_fixed,(-1,28*28))

bestScore = 100000

n_epochs = 100

for epoch in range(n_epochs):

# Random shuffling

index = [i for i in range(np.shape(x_train)[0])]

random.shuffle(index)

x_train = x_train[index]

# Loop over all batches

for i in range(total_batch):

# Compute the offset of the current minibatch in the data.

batch_xs_input = x_train[i * batch_size:i * batch_size + batch_size]

batch_xs_target = batch_xs_input

if ADD_NOISE:

batch_xs_input = batch_xs_input * np.random.randint(2, size=batch_xs_input.shape)

batch_xs_input += np.random.randint(2, size=batch_xs_input.shape)

_, tot_loss, loss_likelihood, loss_divergence,dropout_a = sess.run(

(train_op, loss, neg_marginal_likelihood, KL_divergence,dropoutA),

feed_dict={x_hat: batch_xs_input, x: batch_xs_target, keep_prob: 1.0})

y_PRR = sess.run(y, feed_dict={x_hat: x_fixed, keep_prob: 1})

y_RPR = np.reshape(y_PRR, (-1, 28, 28))

ims("results/" + "VAE" + str(epoch) + ".jpg", merge(y_RPR[:64], [8, 8]))

print("epoch %f: L_tot %03.2f L_likelihood %03.2f L_divergence %03.2f" % (

epoch, tot_loss, loss_likelihood, loss_divergence))

if epoch > 0:

x_fixed_image = np.reshape(x_fixed, (-1, 28, 28))

ims("results/" + "Real" + str(epoch) + ".jpg", merge(x_fixed_image[:64], [8, 8]))

if bestScore > loss_likelihood:

bestScore = loss_likelihood

#saver.save(sess, 'models/Dropout_Sinmple_4_Gaussian')

#saver.save(sess, 'models/Dropout_Sinmple_4_Gaussian')