def buildGraph(K, D):
tf.compat.v1.set_random_seed(421)
MU = tf.Variable(tf.random.normal(shape=[K, D]))
psi = tf.Variable(tf.random.normal(shape=[K, 1]))
phi = tf.Variable(tf.random.normal(shape=[K, 1]))
X = tf.compat.v1.placeholder(tf.float32, [None, D])
sigma = tf.sqrt(tf.exp(phi))
log_pi = hlp.logsoftmax(psi)
pi = tf.math.exp(log_pi)
logPDF = log_GaussPDF(X, MU, sigma)
post = log_posterior(logPDF, log_pi)
assignments = tf.math.argmax(post, axis=1)
loss = -tf.reduce_sum(
hlp.reduce_logsumexp(tf.transpose(log_pi) + logPDF, keep_dims=True))
optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=0.1,
beta1=0.9,
beta2=0.99,
epsilon=1e-5)
train = optimizer.minimize(loss)
return MU, sigma, pi, X, assignments, loss, train
def GMM(K, D):
tf.set_random_seed(421)
X = tf.placeholder(tf.float32, shape=(None, D), name="trainData")
means = tf.Variable(tf.random_normal(shape=[K, D], stddev=1.0),
name="means")
phi = tf.Variable(tf.random_normal(shape=[K, 1], stddev=1.0), name="Phi")
psi = tf.Variable(tf.random_normal(shape=[K, 1], stddev=1.0), name="Psi")
sigma = tf.sqrt(tf.exp(phi))
psi_soft = hlp.logsoftmax(psi)
prob = tf.exp(psi_soft)
log_gauss = log_GaussPDF(X, means, sigma)
loss = -tf.reduce_sum(hlp.reduce_logsumexp(
log_gauss + tf.transpose(tf.log(prob)), 1),
axis=0)
# Returns a Nx1 vector of best_cluster assignments
best_cluster = tf.argmax(log_posterior(log_gauss, prob), axis=1)
optimizer = tf.train.AdamOptimizer(learning_rate=0.1,
beta1=0.9,
beta2=0.99,
epsilon=1e-5).minimize(loss)
return X, means, prob, best_cluster, loss, optimizer, log_gauss
def calculate_loss(log_PDF, log_pi):
P = log_PDF + tf.squeeze(log_pi)
P = hlp.reduce_logsumexp(P, 1, True)
loss = -1 * tf.reduce_mean(P)
return loss
def MoG():
X = tf.placeholder(tf.float32, [None, D], name="X")
MU = tf.get_variable('mean',
dtype=tf.float32,
shape=[K, D],
initializer=tf.initializers.random_normal())
Psi = tf.get_variable('variance',
dtype=tf.float32,
shape=[K, 1],
initializer=tf.initializers.random_normal())
Pi = tf.get_variable('posterior',
dtype=tf.float32,
shape=[K, 1],
initializer=tf.initializers.random_normal())
log_Pi = hlp.logsoftmax(Pi)
Sigma2 = tf.exp(Psi)
Gauss_PDF = log_GaussPDF(X, MU, Sigma2)
Log_Post = log_posterior(Gauss_PDF, log_Pi)
Belong = tf.arg_max(Log_Post, dimension=1)
lossfunc = -tf.reduce_sum(hlp.reduce_logsumexp(Log_Post))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01,
beta1=0.9,
beta2=0.99,
epsilon=1e-5)
train = optimizer.minimize(loss=lossfunc)
return X, MU, Psi, Pi, lossfunc, Belong, train
def compute_loss(X, mu, sigma, pi):
log_pi = tf.squeeze(hlp.logsoftmax(pi))
log_pdf = log_GaussPDF(X, mu, sigma)
log_loss = -1 * tf.reduce_sum(
hlp.reduce_logsumexp(log_pdf + log_pi, keep_dims=True))
return log_loss, log_pdf, log_pi
def main():
data = np.load('data2D.npy')
[num_pts, dim] = np.shape(data)
is_valid = False
K = 5
# For Validation set
if is_valid:
valid_batch = int(num_pts / 3.0)
np.random.seed(45689)
rnd_idx = np.arange(num_pts)
np.random.shuffle(rnd_idx)
val_data = data[rnd_idx[:valid_batch]]
data = data[rnd_idx[valid_batch:]]
X = tf.placeholder(tf.float32, [num_pts, dim], name='X')
mu = tf.Variable(tf.truncated_normal([K, dim],stddev=0.05),name="mu")
phi = tf.Variable(tf.zeros([K, 1]),name="sigma")
sigma = tf.exp(phi)
#clip_op = tf.assign(sigma, tf.clip_by_value(sigma, 0, np.infty))
log_PDF = log_GaussPDF(X, mu,sigma)
#temp_pi = np.float32(np.ones((K,1)))
#temp_pi[:] = 0
pi = tf.Variable(tf.truncated_normal([K, 1],stddev=0.05),name="pi")
log_pi = hlp.logsoftmax(pi)
log_Posterior = log_posterior(log_PDF, log_pi)
logPX = tf.reduce_sum(hlp.reduce_logsumexp(tf.add(tf.transpose(log_pi),log_PDF)))
print logPX.shape
Loss = -logPX
train_op = tf.train.AdamOptimizer(learning_rate = 0.01).minimize(Loss)
sess = tf.Session()
distances = distanceFunc(X, mu)
nearestIndices = tf.argmax(log_Posterior, 1)
partitions = tf.dynamic_partition(X,tf.to_int32(nearestIndices),K)
with sess.as_default():
init = tf.global_variables_initializer()
sess.run(init)
for i in range(800):
sess.run(train_op,feed_dict={X:np.float32(data)})
#sess.run(clip_op)
print sess.run(mu)
print sess.run(tf.exp(log_pi))
print(sess.run(Loss,feed_dict={X:np.float32(data)}))
updated_centroid_value = sess.run(mu)
part = sess.run(partitions,feed_dict={X:np.float32(data)})
for data in part:
print len(data)
plt.figure()
colour = plt.cm.rainbow(np.linspace(0,1,len(updated_centroid_value)))
for i, centroid in enumerate(updated_centroid_value):
print len(part[i])
for j,point in enumerate(part[i]):
plt.scatter(point[0], point[1], c=colour[i])
plt.plot(centroid[0], centroid[1], markersize=35, marker="x", color='k')
plt.savefig( 'cluster5' + '.png')
#plt.show()
plt.figure()
plt.scatter(data[:,0], data[:,1])
plt.savefig('Originaldata' + '.png')
def MoG(dataset, K, alpha):
N, D = num_pts, dim
X = tf.placeholder(tf.float32, shape=(N, D), name="X")
MU = tf.get_variable(name="MU", initializer=tf.random.normal(shape=[K, D]))
sigma = tf.get_variable(name="sigma",
initializer=tf.random.normal(shape=[K, 1]))
pi = tf.get_variable(name="pi", initializer=tf.random.normal(shape=[1, K]))
# Take the expononent of the sigma as per instructions
sexp = tf.exp(sigma)
# compute the P(xn | zn = K)
log_PDF = log_GaussPDF(X, MU, sexp)
#print(hlp.logsoftmax(pi).shape, log_PDF.shape)
sum_l = hlp.reduce_logsumexp(hlp.logsoftmax(pi) + log_PDF)
#print(sum_l.shape)
loss = -1 * tf.reduce_sum(sum_l)
opt = tf.train.AdamOptimizer(learning_rate=alpha,
beta1=0.9,
beta2=0.99,
epsilon=1e-5).minimize(loss)
# Find the log posterioer for plotting the clusters at the end
sm = hlp.logsoftmax(log_PDF)
return MU, X, loss, opt, sigma, pi, sm
def GMM(K):
# define the model parameters
MU = tf.Variable(
tf.truncated_normal(shape=[K, dim]),
name="MU",
)
sigma = tf.Variable(tf.truncated_normal(shape=[K, 1]), name="sigma")
sigma = tf.exp(sigma)
log_pi = tf.Variable(tf.truncated_normal(shape=[K, 1]), name="pi")
log_pi = hlp.logsoftmax(log_pi)
# input data
X = tf.placeholder(tf.float32, [None, dim], name='data')
# call the log_PDF
log_PDF = log_GaussPDF(X, MU, sigma)
# find the most possible cluster for each point
log_post = log_posterior(log_PDF, log_pi)
assigned = tf.argmax(log_post, 1)
# define the loss function #print(loss.get_shape().as_list())
loss = tf.add(log_PDF, tf.transpose(log_pi))
loss = hlp.reduce_logsumexp(loss)
loss = -1 * tf.reduce_sum(loss)
# train the model
train = tf.train.AdamOptimizer(learning_rate=0.1,
beta1=0.9,
beta2=0.99,
epsilon=1e-5).minimize(loss)
return MU, sigma, log_pi, X, assigned, loss, train
def NLL_loss(log_gauss_PDF, log_pi):
inner = tf.squeeze(log_pi) + log_gauss_PDF
summed_over_clusters = hlp.reduce_logsumexp(inner)
loss = -tf.reduce_sum(summed_over_clusters)
return loss
def log_posterior(log_PDF, log_pi):
# Input
# log_PDF: log Gaussian PDF N X K
# log_pi: K X 1
# Outputs
# log_post: N X K
# TODO
log_pi = tf.transpose(log_pi)
return log_PDF + log_pi - hlp.reduce_logsumexp(log_PDF + log_pi,
keep_dims=True)
def log_posterior(log_PDF, log_pi):
# Input
# log_PDF: log Gaussian PDF N X K
# log_pi: K X 1
log_pi = tf.reshape(log_pi, [1, K])
PDF = tf.exp(log_PDF)
pi = tf.exp(log_pi)
# Outputs
# log_post: N X K
return log_pi + log_PDF - hlp.reduce_logsumexp(PDF * pi, 1, True)
def log_posterior(log_pdf, log_pi):
# Input
# log_pdf: log Gaussian PDF N X K
# log_pi: K X 1
# Outputs
# log_post: N X K
anormalized = log_pdf + tf.reshape(log_pi, [-1])
return anormalized - hlp.reduce_logsumexp(
anormalized, reduction_indices=1, keep_dims=True)
def log_posterior(log_PDF, log_pi):
# Input
# log_PDF: log Gaussian PDF N X K
# log_pi: K X 1
# Outputs
# log_post: N X K
numerator_log = tf.add(log_PDF, tf.transpose(log_pi))
denominator_log = -hlp.reduce_logsumexp(numerator_log, 1, True)
return tf.add(numerator_log, denominator_log)
def log_posterior(log_PDF, log_pi):
# Input
# log_PDF: log Gaussian PDF N X K
# log_pi: K X 1
# Outputs
# log_post: N X K
log_density = log_PDF + tf.transpose(log_pi)
log_sums = hlp.reduce_logsumexp(log_density)
log_sums = tf.expand_dims(log_sums, 1)
return tf.subtract(log_density, log_sums)
def log_posterior(log_PDF, log_pi):
# Input
# log_PDF: log Gaussian PDF N X K
# log_pi: K X 1
# Outputs
# log_post: N X K
# TODO
coeff = tf.squeeze(log_pi) + log_PDF
return coeff - tf.expand_dims(hlp.reduce_logsumexp(coeff), 1)
def log_posterior(log_PDF, log_pi):
# Input
# log_PDF: log Gaussian PDF N X K
# log_pi: K X 1
# Outputs
# log_post: N X K
log_pi = tf.squeeze(log_pi)
sum_term = hlp.reduce_logsumexp(log_PDF + log_pi, keep_dims=True)
return tf.add(log_pi, log_PDF) - sum_term
def log_posterior(log_PDF, log_pi):
# Input
# log_PDF: log Gaussian PDF N X K
# log_pi: K X 1
# Outputs
# log_post: N X K
log_pi = tf.squeeze(log_pi)
log_prob = tf.add(log_pi,log_PDF)
log_sum = hlp.reduce_logsumexp(log_prob + log_pi,keep_dims=True)
output = log_prob - log_sum
return output
def log_posterior(log_PDF, log_pi):
# Input
# log_PDF: log Gaussian PDF N X K
# log_pi: K X 1
# Outputs
# log_post: N X K
# TODO
up = tf.add(tf.transpose(log_pi), log_PDF)
down = hlp.reduce_logsumexp(tf.transpose(log_pi) + log_PDF, keep_dims=True)
return up - down
def log_posterior(log_PDF, log_pi):
# Input
# log_PDF: log Gaussian PDF N X K
# log_pi: K X 1
# Outputs
# log_post: N X K
num = log_PDF + tf.reshape(log_pi, [1, -1])
denom = hlp.reduce_logsumexp(num, 1, True)
return num - denom
def log_posterior(log_PDF, log_pi):
# Input
# log_PDF: log Gaussian PDF N X K
# log_pi: K X 1
# Outputs
# log_post: N X K
# TODO
log_pi = tf.transpose(log_pi)
prob = log_pi + log_PDF
prob_sum = hlp.reduce_logsumexp(prob, keep_dims=True)
posterior = prob - prob_sum
return posterior
def log_posterior(log_PDF, log_pi):
# Input
# log_PDF: log Gaussian PDF N X K
# log_pi: K X 1
# Outputs
# log_post: N X K
log_pi = tf.squeeze(log_pi)
sum_log = hlp.reduce_logsumexp(tf.add(log_PDF, log_pi), keep_dims=True)
log_post = tf.subtract(tf.add(log_PDF, log_pi), sum_log)
return log_post
def log_posterior(log_PDF, log_pi):
# Input
# log_PDF: log Gaussian PDF N X K
# log_pi: K X 1
# Outputs
# log_post: N X K
# part 2.2
# log P(Z = k| x) = log P(x|mu, sigma) + log_pi - logsumexp(z)
term1 = tf.squeeze(log_pi) + log_PDF # N x K
log_post = term1 - tf.expand_dims(hlp.reduce_logsumexp(term1), 1)
return log_post
def buildGraph():
tf.reset_default_graph() # Clear any previous junk
tf.set_random_seed(45689)
trainingInput = tf.placeholder(tf.float32, shape=(None, dim))
centroid, variance, logPi = initializeVars()
logPDF = log_GaussPDF(trainingInput, centroid, variance)
logPosterior = log_posterior(logPDF,
logPi) #What dimension should this be?
loss = -1.0 * tf.reduce_sum(
hlp.reduce_logsumexp(logPDF + logPi, keep_dims=True))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01,
beta1=0.9,
beta2=0.99,
epsilon=1e-5).minimize(loss)
return optimizer, loss, tf.exp(logPosterior), centroid, variance, tf.exp(
logPi), trainingInput
def log_posterior(log_PDF, log_pi): #logP(z|X)
# Input
# log_PDF: log Gaussian PDF N X K
# log_pi: K X 1
# Outputs
# log_post: N X K
# TODO
N = log_PDF.shape[0]
part1 = log_PDF + log_pi # NxK
# print(part1.get_shape)
part2 = hlp.reduce_logsumexp(log_PDF + log_pi, reduction_indices=1) # Nx
part2 = tf.reshape(part2, (N, 1)) # Nx1
# print(part2.get_shape)
log_pst = part1 - part2 # NxK
# print(log_pst.get_shape)
return log_pst
def MOGLoss(K):
N, D = num_pts, dim
X = tf.placeholder(tf.float32, shape=(N, D), name="X_VAL")
MU = tf.get_variable(name="MU_VAL",
initializer=tf.random.normal(shape=[K, D]))
sigma = tf.get_variable(shape=(K, 1), name="sigma_VAL")
pi = tf.get_variable(shape=(1, K), name="pi_VAL")
# Take the expononent of the sigma as per instructions
sexp = tf.exp(sigma)
# compute the P(xn | zn = K)
log_PDF = log_GaussPDF(X, MU, sexp)
sum_l = hlp.reduce_logsumexp(hlp.logsoftmax(pi) + log_PDF)
loss = -1 * tf.reduce_sum(sum_l)
# Find the log posterioer for plotting the clusters at the end
sm = hlp.logsoftmax(log_PDF)
return MU, X, loss, sigma, pi, sm
def buildGraph():
k = 3
x = tf.placeholder(tf.float32, [None, dim])
mu = tf.get_variable("MU", initializer=tf.truncated_normal(shape=(k, dim)))
sigma = tf.get_variable("sigma",
initializer=tf.truncated_normal(shape=(k, 1)))
pi = tf.get_variable("pi", initializer=tf.truncated_normal(shape=(k, 1)))
log_pi = hlp.logsoftmax(pi)
log_PDF = log_GaussPDF(x, mu, tf.sqrt(tf.exp(sigma)))
log_post = log_posterior(log_PDF, log_pi)
cluster = tf.argmax(log_post, axis=1)
loss = -tf.reduce_sum(
hlp.reduce_logsumexp(log_PDF + tf.transpose(log_pi), keep_dims=True))
optimizer = tf.train.AdamOptimizer(learning_rate=0.1,
beta1=0.9,
beta2=0.99,
epsilon=1e-5)
training_op = optimizer.minimize(loss=loss)
return training_op, x, mu, cluster, loss, log_pi, tf.sqrt(tf.exp(sigma))
def buildGraph():
tf.reset_default_graph() # Clear any previous junk
tf.set_random_seed(45689)
trainingInput = tf.placeholder(tf.float32,
shape=(None, dim)) # Data placeholder
centroid, variance, logPi = initializeVars(
) # Initialize mean, variance, and weights
logPDF = log_GaussPDF(trainingInput, centroid,
variance) # Calculate P(X|Z)
logPosterior = log_posterior(logPDF,
logPi) # Calculates the Bayesian P(Z|X)
loss = -1.0 * tf.reduce_sum(
hlp.reduce_logsumexp(logPDF + logPi,
keep_dims=True)) # Calculate the loss
optimizer = tf.train.AdamOptimizer(learning_rate=0.01,
beta1=0.9,
beta2=0.99,
epsilon=1e-5).minimize(loss)
# The tf.exp term returns the probabilites and weights in a form humans can understand
return optimizer, loss, tf.exp(logPosterior), centroid, variance, tf.exp(
logPi), trainingInput
def loss_function(log_PDF, log_pi):
loss = -tf.reduce_sum(
hlp.reduce_logsumexp(log_PDF + log_pi, 1, keep_dims=True), axis=0)
return loss
def neg_log_prob(x, mu, sigma, log_pi):
log_pdf = log_gauss_pdf(x, mu, sigma) #pdf
log_likelihood = hlp.reduce_logsumexp(log_pdf + tf.reshape(log_pi, [-1]))
return -1 * log_likelihood
shape=(K, 1),
initializer=tf.initializers.random_normal(seed=366901768))
prior = tf.get_variable(
name='pi',
dtype=tf.float64,
shape=(K, 1),
initializer=tf.initializers.random_normal(seed=1566557))
sigma = tf.exp(stdvec)
log_gauss_pdf = log_GaussPDF(x, mu, sigma)
log_prior = hlp.logsoftmax(prior) #To ensure that priors are normalized
log_post = log_posterior(log_gauss_pdf, log_prior)
#Defining loss function
temp = hlp.reduce_logsumexp(tf.squeeze(log_prior) + log_gauss_pdf)
loss = -tf.reduce_sum(temp)
optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
train_loss = []
for epoch in range(epochs):
sess.run(optimizer, feed_dict={x: data})
trainingloss = sess.run(loss, feed_dict={x: data})
validloss = sess.run(loss, feed_dict={x: val_data})
train_loss.append(trainingloss)
