def buildGraph(K, D): tf.compat.v1.set_random_seed(421) MU = tf.Variable(tf.random.normal(shape=[K, D])) psi = tf.Variable(tf.random.normal(shape=[K, 1])) phi = tf.Variable(tf.random.normal(shape=[K, 1])) X = tf.compat.v1.placeholder(tf.float32, [None, D]) sigma = tf.sqrt(tf.exp(phi)) log_pi = hlp.logsoftmax(psi) pi = tf.math.exp(log_pi) logPDF = log_GaussPDF(X, MU, sigma) post = log_posterior(logPDF, log_pi) assignments = tf.math.argmax(post, axis=1) loss = -tf.reduce_sum( hlp.reduce_logsumexp(tf.transpose(log_pi) + logPDF, keep_dims=True)) optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=0.1, beta1=0.9, beta2=0.99, epsilon=1e-5) train = optimizer.minimize(loss) return MU, sigma, pi, X, assignments, loss, train
def GMM(K, D): tf.set_random_seed(421) X = tf.placeholder(tf.float32, shape=(None, D), name="trainData") means = tf.Variable(tf.random_normal(shape=[K, D], stddev=1.0), name="means") phi = tf.Variable(tf.random_normal(shape=[K, 1], stddev=1.0), name="Phi") psi = tf.Variable(tf.random_normal(shape=[K, 1], stddev=1.0), name="Psi") sigma = tf.sqrt(tf.exp(phi)) psi_soft = hlp.logsoftmax(psi) prob = tf.exp(psi_soft) log_gauss = log_GaussPDF(X, means, sigma) loss = -tf.reduce_sum(hlp.reduce_logsumexp( log_gauss + tf.transpose(tf.log(prob)), 1), axis=0) # Returns a Nx1 vector of best_cluster assignments best_cluster = tf.argmax(log_posterior(log_gauss, prob), axis=1) optimizer = tf.train.AdamOptimizer(learning_rate=0.1, beta1=0.9, beta2=0.99, epsilon=1e-5).minimize(loss) return X, means, prob, best_cluster, loss, optimizer, log_gauss
def calculate_loss(log_PDF, log_pi): P = log_PDF + tf.squeeze(log_pi) P = hlp.reduce_logsumexp(P, 1, True) loss = -1 * tf.reduce_mean(P) return loss
def MoG(): X = tf.placeholder(tf.float32, [None, D], name="X") MU = tf.get_variable('mean', dtype=tf.float32, shape=[K, D], initializer=tf.initializers.random_normal()) Psi = tf.get_variable('variance', dtype=tf.float32, shape=[K, 1], initializer=tf.initializers.random_normal()) Pi = tf.get_variable('posterior', dtype=tf.float32, shape=[K, 1], initializer=tf.initializers.random_normal()) log_Pi = hlp.logsoftmax(Pi) Sigma2 = tf.exp(Psi) Gauss_PDF = log_GaussPDF(X, MU, Sigma2) Log_Post = log_posterior(Gauss_PDF, log_Pi) Belong = tf.arg_max(Log_Post, dimension=1) lossfunc = -tf.reduce_sum(hlp.reduce_logsumexp(Log_Post)) optimizer = tf.train.AdamOptimizer(learning_rate=0.01, beta1=0.9, beta2=0.99, epsilon=1e-5) train = optimizer.minimize(loss=lossfunc) return X, MU, Psi, Pi, lossfunc, Belong, train
def compute_loss(X, mu, sigma, pi): log_pi = tf.squeeze(hlp.logsoftmax(pi)) log_pdf = log_GaussPDF(X, mu, sigma) log_loss = -1 * tf.reduce_sum( hlp.reduce_logsumexp(log_pdf + log_pi, keep_dims=True)) return log_loss, log_pdf, log_pi
def main(): data = np.load('data2D.npy') [num_pts, dim] = np.shape(data) is_valid = False K = 5 # For Validation set if is_valid: valid_batch = int(num_pts / 3.0) np.random.seed(45689) rnd_idx = np.arange(num_pts) np.random.shuffle(rnd_idx) val_data = data[rnd_idx[:valid_batch]] data = data[rnd_idx[valid_batch:]] X = tf.placeholder(tf.float32, [num_pts, dim], name='X') mu = tf.Variable(tf.truncated_normal([K, dim],stddev=0.05),name="mu") phi = tf.Variable(tf.zeros([K, 1]),name="sigma") sigma = tf.exp(phi) #clip_op = tf.assign(sigma, tf.clip_by_value(sigma, 0, np.infty)) log_PDF = log_GaussPDF(X, mu,sigma) #temp_pi = np.float32(np.ones((K,1))) #temp_pi[:] = 0 pi = tf.Variable(tf.truncated_normal([K, 1],stddev=0.05),name="pi") log_pi = hlp.logsoftmax(pi) log_Posterior = log_posterior(log_PDF, log_pi) logPX = tf.reduce_sum(hlp.reduce_logsumexp(tf.add(tf.transpose(log_pi),log_PDF))) print logPX.shape Loss = -logPX train_op = tf.train.AdamOptimizer(learning_rate = 0.01).minimize(Loss) sess = tf.Session() distances = distanceFunc(X, mu) nearestIndices = tf.argmax(log_Posterior, 1) partitions = tf.dynamic_partition(X,tf.to_int32(nearestIndices),K) with sess.as_default(): init = tf.global_variables_initializer() sess.run(init) for i in range(800): sess.run(train_op,feed_dict={X:np.float32(data)}) #sess.run(clip_op) print sess.run(mu) print sess.run(tf.exp(log_pi)) print(sess.run(Loss,feed_dict={X:np.float32(data)})) updated_centroid_value = sess.run(mu) part = sess.run(partitions,feed_dict={X:np.float32(data)}) for data in part: print len(data) plt.figure() colour = plt.cm.rainbow(np.linspace(0,1,len(updated_centroid_value))) for i, centroid in enumerate(updated_centroid_value): print len(part[i]) for j,point in enumerate(part[i]): plt.scatter(point[0], point[1], c=colour[i]) plt.plot(centroid[0], centroid[1], markersize=35, marker="x", color='k') plt.savefig( 'cluster5' + '.png') #plt.show() plt.figure() plt.scatter(data[:,0], data[:,1]) plt.savefig('Originaldata' + '.png')
def MoG(dataset, K, alpha): N, D = num_pts, dim X = tf.placeholder(tf.float32, shape=(N, D), name="X") MU = tf.get_variable(name="MU", initializer=tf.random.normal(shape=[K, D])) sigma = tf.get_variable(name="sigma", initializer=tf.random.normal(shape=[K, 1])) pi = tf.get_variable(name="pi", initializer=tf.random.normal(shape=[1, K])) # Take the expononent of the sigma as per instructions sexp = tf.exp(sigma) # compute the P(xn | zn = K) log_PDF = log_GaussPDF(X, MU, sexp) #print(hlp.logsoftmax(pi).shape, log_PDF.shape) sum_l = hlp.reduce_logsumexp(hlp.logsoftmax(pi) + log_PDF) #print(sum_l.shape) loss = -1 * tf.reduce_sum(sum_l) opt = tf.train.AdamOptimizer(learning_rate=alpha, beta1=0.9, beta2=0.99, epsilon=1e-5).minimize(loss) # Find the log posterioer for plotting the clusters at the end sm = hlp.logsoftmax(log_PDF) return MU, X, loss, opt, sigma, pi, sm
def GMM(K): # define the model parameters MU = tf.Variable( tf.truncated_normal(shape=[K, dim]), name="MU", ) sigma = tf.Variable(tf.truncated_normal(shape=[K, 1]), name="sigma") sigma = tf.exp(sigma) log_pi = tf.Variable(tf.truncated_normal(shape=[K, 1]), name="pi") log_pi = hlp.logsoftmax(log_pi) # input data X = tf.placeholder(tf.float32, [None, dim], name='data') # call the log_PDF log_PDF = log_GaussPDF(X, MU, sigma) # find the most possible cluster for each point log_post = log_posterior(log_PDF, log_pi) assigned = tf.argmax(log_post, 1) # define the loss function #print(loss.get_shape().as_list()) loss = tf.add(log_PDF, tf.transpose(log_pi)) loss = hlp.reduce_logsumexp(loss) loss = -1 * tf.reduce_sum(loss) # train the model train = tf.train.AdamOptimizer(learning_rate=0.1, beta1=0.9, beta2=0.99, epsilon=1e-5).minimize(loss) return MU, sigma, log_pi, X, assigned, loss, train
def NLL_loss(log_gauss_PDF, log_pi): inner = tf.squeeze(log_pi) + log_gauss_PDF summed_over_clusters = hlp.reduce_logsumexp(inner) loss = -tf.reduce_sum(summed_over_clusters) return loss
def log_posterior(log_PDF, log_pi): # Input # log_PDF: log Gaussian PDF N X K # log_pi: K X 1 # Outputs # log_post: N X K # TODO log_pi = tf.transpose(log_pi) return log_PDF + log_pi - hlp.reduce_logsumexp(log_PDF + log_pi, keep_dims=True)
def log_posterior(log_PDF, log_pi): # Input # log_PDF: log Gaussian PDF N X K # log_pi: K X 1 log_pi = tf.reshape(log_pi, [1, K]) PDF = tf.exp(log_PDF) pi = tf.exp(log_pi) # Outputs # log_post: N X K return log_pi + log_PDF - hlp.reduce_logsumexp(PDF * pi, 1, True)
def log_posterior(log_pdf, log_pi): # Input # log_pdf: log Gaussian PDF N X K # log_pi: K X 1 # Outputs # log_post: N X K anormalized = log_pdf + tf.reshape(log_pi, [-1]) return anormalized - hlp.reduce_logsumexp( anormalized, reduction_indices=1, keep_dims=True)
def log_posterior(log_PDF, log_pi): # Input # log_PDF: log Gaussian PDF N X K # log_pi: K X 1 # Outputs # log_post: N X K numerator_log = tf.add(log_PDF, tf.transpose(log_pi)) denominator_log = -hlp.reduce_logsumexp(numerator_log, 1, True) return tf.add(numerator_log, denominator_log)
def log_posterior(log_PDF, log_pi): # Input # log_PDF: log Gaussian PDF N X K # log_pi: K X 1 # Outputs # log_post: N X K log_density = log_PDF + tf.transpose(log_pi) log_sums = hlp.reduce_logsumexp(log_density) log_sums = tf.expand_dims(log_sums, 1) return tf.subtract(log_density, log_sums)
def log_posterior(log_PDF, log_pi): # Input # log_PDF: log Gaussian PDF N X K # log_pi: K X 1 # Outputs # log_post: N X K # TODO coeff = tf.squeeze(log_pi) + log_PDF return coeff - tf.expand_dims(hlp.reduce_logsumexp(coeff), 1)
def log_posterior(log_PDF, log_pi): # Input # log_PDF: log Gaussian PDF N X K # log_pi: K X 1 # Outputs # log_post: N X K log_pi = tf.squeeze(log_pi) sum_term = hlp.reduce_logsumexp(log_PDF + log_pi, keep_dims=True) return tf.add(log_pi, log_PDF) - sum_term
def log_posterior(log_PDF, log_pi): # Input # log_PDF: log Gaussian PDF N X K # log_pi: K X 1 # Outputs # log_post: N X K log_pi = tf.squeeze(log_pi) log_prob = tf.add(log_pi,log_PDF) log_sum = hlp.reduce_logsumexp(log_prob + log_pi,keep_dims=True) output = log_prob - log_sum return output
def log_posterior(log_PDF, log_pi): # Input # log_PDF: log Gaussian PDF N X K # log_pi: K X 1 # Outputs # log_post: N X K # TODO up = tf.add(tf.transpose(log_pi), log_PDF) down = hlp.reduce_logsumexp(tf.transpose(log_pi) + log_PDF, keep_dims=True) return up - down
def log_posterior(log_PDF, log_pi): # Input # log_PDF: log Gaussian PDF N X K # log_pi: K X 1 # Outputs # log_post: N X K num = log_PDF + tf.reshape(log_pi, [1, -1]) denom = hlp.reduce_logsumexp(num, 1, True) return num - denom
def log_posterior(log_PDF, log_pi): # Input # log_PDF: log Gaussian PDF N X K # log_pi: K X 1 # Outputs # log_post: N X K # TODO log_pi = tf.transpose(log_pi) prob = log_pi + log_PDF prob_sum = hlp.reduce_logsumexp(prob, keep_dims=True) posterior = prob - prob_sum return posterior
def log_posterior(log_PDF, log_pi): # Input # log_PDF: log Gaussian PDF N X K # log_pi: K X 1 # Outputs # log_post: N X K log_pi = tf.squeeze(log_pi) sum_log = hlp.reduce_logsumexp(tf.add(log_PDF, log_pi), keep_dims=True) log_post = tf.subtract(tf.add(log_PDF, log_pi), sum_log) return log_post
def log_posterior(log_PDF, log_pi): # Input # log_PDF: log Gaussian PDF N X K # log_pi: K X 1 # Outputs # log_post: N X K # part 2.2 # log P(Z = k| x) = log P(x|mu, sigma) + log_pi - logsumexp(z) term1 = tf.squeeze(log_pi) + log_PDF # N x K log_post = term1 - tf.expand_dims(hlp.reduce_logsumexp(term1), 1) return log_post
def buildGraph(): tf.reset_default_graph() # Clear any previous junk tf.set_random_seed(45689) trainingInput = tf.placeholder(tf.float32, shape=(None, dim)) centroid, variance, logPi = initializeVars() logPDF = log_GaussPDF(trainingInput, centroid, variance) logPosterior = log_posterior(logPDF, logPi) #What dimension should this be? loss = -1.0 * tf.reduce_sum( hlp.reduce_logsumexp(logPDF + logPi, keep_dims=True)) optimizer = tf.train.AdamOptimizer(learning_rate=0.01, beta1=0.9, beta2=0.99, epsilon=1e-5).minimize(loss) return optimizer, loss, tf.exp(logPosterior), centroid, variance, tf.exp( logPi), trainingInput
def log_posterior(log_PDF, log_pi): #logP(z|X) # Input # log_PDF: log Gaussian PDF N X K # log_pi: K X 1 # Outputs # log_post: N X K # TODO N = log_PDF.shape[0] part1 = log_PDF + log_pi # NxK # print(part1.get_shape) part2 = hlp.reduce_logsumexp(log_PDF + log_pi, reduction_indices=1) # Nx part2 = tf.reshape(part2, (N, 1)) # Nx1 # print(part2.get_shape) log_pst = part1 - part2 # NxK # print(log_pst.get_shape) return log_pst
def MOGLoss(K): N, D = num_pts, dim X = tf.placeholder(tf.float32, shape=(N, D), name="X_VAL") MU = tf.get_variable(name="MU_VAL", initializer=tf.random.normal(shape=[K, D])) sigma = tf.get_variable(shape=(K, 1), name="sigma_VAL") pi = tf.get_variable(shape=(1, K), name="pi_VAL") # Take the expononent of the sigma as per instructions sexp = tf.exp(sigma) # compute the P(xn | zn = K) log_PDF = log_GaussPDF(X, MU, sexp) sum_l = hlp.reduce_logsumexp(hlp.logsoftmax(pi) + log_PDF) loss = -1 * tf.reduce_sum(sum_l) # Find the log posterioer for plotting the clusters at the end sm = hlp.logsoftmax(log_PDF) return MU, X, loss, sigma, pi, sm
def buildGraph(): k = 3 x = tf.placeholder(tf.float32, [None, dim]) mu = tf.get_variable("MU", initializer=tf.truncated_normal(shape=(k, dim))) sigma = tf.get_variable("sigma", initializer=tf.truncated_normal(shape=(k, 1))) pi = tf.get_variable("pi", initializer=tf.truncated_normal(shape=(k, 1))) log_pi = hlp.logsoftmax(pi) log_PDF = log_GaussPDF(x, mu, tf.sqrt(tf.exp(sigma))) log_post = log_posterior(log_PDF, log_pi) cluster = tf.argmax(log_post, axis=1) loss = -tf.reduce_sum( hlp.reduce_logsumexp(log_PDF + tf.transpose(log_pi), keep_dims=True)) optimizer = tf.train.AdamOptimizer(learning_rate=0.1, beta1=0.9, beta2=0.99, epsilon=1e-5) training_op = optimizer.minimize(loss=loss) return training_op, x, mu, cluster, loss, log_pi, tf.sqrt(tf.exp(sigma))
def buildGraph(): tf.reset_default_graph() # Clear any previous junk tf.set_random_seed(45689) trainingInput = tf.placeholder(tf.float32, shape=(None, dim)) # Data placeholder centroid, variance, logPi = initializeVars( ) # Initialize mean, variance, and weights logPDF = log_GaussPDF(trainingInput, centroid, variance) # Calculate P(X|Z) logPosterior = log_posterior(logPDF, logPi) # Calculates the Bayesian P(Z|X) loss = -1.0 * tf.reduce_sum( hlp.reduce_logsumexp(logPDF + logPi, keep_dims=True)) # Calculate the loss optimizer = tf.train.AdamOptimizer(learning_rate=0.01, beta1=0.9, beta2=0.99, epsilon=1e-5).minimize(loss) # The tf.exp term returns the probabilites and weights in a form humans can understand return optimizer, loss, tf.exp(logPosterior), centroid, variance, tf.exp( logPi), trainingInput
def loss_function(log_PDF, log_pi): loss = -tf.reduce_sum( hlp.reduce_logsumexp(log_PDF + log_pi, 1, keep_dims=True), axis=0) return loss
def neg_log_prob(x, mu, sigma, log_pi): log_pdf = log_gauss_pdf(x, mu, sigma) #pdf log_likelihood = hlp.reduce_logsumexp(log_pdf + tf.reshape(log_pi, [-1])) return -1 * log_likelihood
shape=(K, 1), initializer=tf.initializers.random_normal(seed=366901768)) prior = tf.get_variable( name='pi', dtype=tf.float64, shape=(K, 1), initializer=tf.initializers.random_normal(seed=1566557)) sigma = tf.exp(stdvec) log_gauss_pdf = log_GaussPDF(x, mu, sigma) log_prior = hlp.logsoftmax(prior) #To ensure that priors are normalized log_post = log_posterior(log_gauss_pdf, log_prior) #Defining loss function temp = hlp.reduce_logsumexp(tf.squeeze(log_prior) + log_gauss_pdf) loss = -tf.reduce_sum(temp) optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) train_loss = [] for epoch in range(epochs): sess.run(optimizer, feed_dict={x: data}) trainingloss = sess.run(loss, feed_dict={x: data}) validloss = sess.run(loss, feed_dict={x: val_data}) train_loss.append(trainingloss)