# Loading data

if is100D:

data = np.load('data100D.npy')

else:

data = np.load('data2D.npy')

[num_pts, dim] = np.shape(data)

# For Validation set

if isValid:

valid_batch = int(num_pts / 3.0)

np.random.seed(45689)

rnd_idx = np.arange(num_pts)

np.random.shuffle(rnd_idx)

val_data = data[rnd_idx[:valid_batch]]

data = data[rnd_idx[valid_batch:]]

return data, val_data

else:

"""Computes the sum of elements across dimensions of a tensor in log domain.

It uses a similar API to tf.reduce_sum.

Args:

input_tensor: The tensor to reduce. Should have numeric type.

reduction_indices: The dimensions to reduce.

keep_dims: If true, retains reduced dimensions with length 1.

Returns:

The reduced tensor.

"""

max_input_tensor1 = tf.reduce_max(

input_tensor, reduction_indices, keep_dims=keep_dims)

max_input_tensor2 = max_input_tensor1

if not keep_dims:

max_input_tensor2 = tf.expand_dims(max_input_tensor2, reduction_indices)

return tf.log(

tf.reduce_sum(

tf.exp(input_tensor - max_input_tensor2),

reduction_indices,

"""Computes normal softmax nonlinearity in log domain.

It can be used to normalize log probability.

The softmax is always computed along the second dimension of the input Tensor.

Args:

input_tensor: Unnormalized log probability.

Returns:

normalized log probability.

"""

# plot clusters with centroids

for i, K in enumerate(cluster_sizes):

for cluster in clusters[i]:

plt.scatter(cluster[:, 0], cluster[:, 1], s=1)

for centroid in centroids[i]:

plt.scatter(centroids[i][:, 0], centroids[i][:, 1], marker='x', c='black')

plt.title("Dataset K=" + str(K))

plt.xlabel('X')

plt.ylabel('Y')

plt.show()

# plot losses

for i, K in enumerate(cluster_sizes):

plt.title('Loss')

plt.xlabel('Iterations')

plt.ylabel('Loss')

plt.plot(loss[i], label='K='+str(K))

plt.legend(['K='+str(k) for k in cluster_sizes])

# Inputs

# X: is an NxD matrix (N observations and D dimensions)

# MU: is an KxD matrix (K means and D dimensions)

# Outputs

# pair_dist: is the squared pairwise distance matrix (NxK)

# TODO

# For the purposes of broadcasting to perform X - MU, change the dimensions

# of X and MU

# change dimension of X from (N,d) to (N,1,d)

_X = tf.expand_dims(input=X, axis=1)

# change dimension of MU from (K,d) to (1,K,d)

MU = tf.expand_dims(input=MU, axis=0)

# With new dimensions, broadcasting can be done as follows:

# N x 1 x d

# 1 x K x d

# ---------

# N x K x d

# With this dimension, subtracting the two tensors is now possible

# Square the result afterwards

dist = tf.math.square(tf.math.subtract(_X, MU))

# return intended Tensor with N x K dimensions by summing across 3rd dimension

# Number of iterations

iterations = 200

# Number of data points

N = data.shape[0]

# Size of dimension

d = data.shape[1]

# Number of clusters/centroids

K = numClusters

### Build Graph ###

# Create placeholder for data points

X = tf.placeholder(dtype=tf.float32, shape=(N,d), name="X")

# Initialize centre of clusters with standard normal distribution

MU = tf.Variable(initial_value=tf.random.normal(shape=[K, d], mean=0, stddev=math.sqrt(1), dtype=tf.float32),

trainable=True, name="MU")

# Calculate distance of each point to each cluster centre

distances = distanceFunc(X, MU)

# Calculate loss: L(MU) = sigma(n=1 to N) min(k=1 to K) ||X-MU||^2

loss = tf.math.reduce_sum(tf.math.reduce_min(distances, axis=1), name="loss")

# Adam Optimizer

opt = tf.train.AdamOptimizer(learning_rate=0.1, beta1=0.9, beta2=0.99, epsilon=1e-5).minimize(loss)

# Assign the index of the minimum distance centroid to each point in X

assign_to_cluster = tf.math.argmin(distances, axis=1, output_type=tf.int32)

# Transform data set by splitting into groups (for output)

output = tf.dynamic_partition(X, assign_to_cluster, num_partitions=numClusters)

# Initialize Tensorflow variables

init = tf.global_variables_initializer()

loss_history = []

clustered = None

with tf.Session() as sess:

sess.run(init)

# Training loop

for step in range(iterations):

_MU, _loss, _opt = sess.run([MU, loss, opt], feed_dict={X: data})

loss_history.append(_loss)

# get trained centroids

trained_centroids = MU.eval()

# Assign each point to cluster based on distance to closest cluster centre

clustered = sess.run(output, feed_dict={X:data})

# Inputs

# X: N X D

# mu: K X D

# sigma: K X 1

# log_pi: K X 1

# Outputs:

# log Gaussian PDF N X K

# TODO

# remove extra dimension of rank 1 to avoid broadcasting issues

sigma = tf.squeeze(sigma)

# calculate distance of each point to each center

dist = distanceFunc(X, mu)

# calculate log_pdf

log_pdf = (-0.5*tf.math.log(2*np.pi*sigma)) - (tf.math.square(dist) / (2*sigma))

# Input

# log_PDF: log Gaussian PDF N X K

# log_pi: K X 1

# Outputs

# log_post: N X K

# TODO

# remove extra dimension of rank 1 to avoid broadcasting issues

log_pi = tf.squeeze(log_pi)

# log likelihood of cluster: log(P(x|z=k)P(z=k)) = log(P(x|z=k)) + log(P(z))

log_likelihood = tf.math.add(log_PDF, log_pi)

# log posterior probability: log(likelihood / sum(k=1..K)(likelihood))

# = log(likelihood) - log(logsumexp(likelihood))

log_post = log_likelihood - reduce_logsumexp(log_likelihood, keep_dims=True)

# Number of iterations

iterations = 400

# Number of data points

N = data.shape[0]

# Size of dimension

d = data.shape[1]

# Number of clusters/centroids

K = numClusters

### Build Graph ###

# Create placeholder for data points

X = tf.placeholder(dtype=tf.float32, shape=(N,d), name="X")

# Initialize centre of clusters with sampling from standard normal distribution

MU = tf.Variable(initial_value=tf.random.normal(shape=[K, d], mean=0, stddev=math.sqrt(1), dtype=tf.float32),

trainable=True, name="MU")

# Initialize sigma with sampling from standard normal distribution

sigma = tf.Variable(initial_value=tf.random_normal(shape=[K, 1], mean=0, stddev=math.sqrt(1)),

trainable=True)

# pass sigma through exp() to avoid constraints

sigma = tf.math.exp(sigma)

# Initialize log_pi with sampling from standard normal distribution

log_pi = tf.Variable(initial_value=tf.random.normal(shape=[K, 1], mean=0, stddev=math.sqrt(1)),

trainable=True)

# pass log_pi through logsoftmax to avoid contraints

log_pi = logsoftmax(log_pi)

# calculate log probability: P(x,z)

log_PDF = log_GaussPDF(X, MU, sigma)

# Calculate loss: L = - logsumexp(log_PDF, log_pi)

loss = -1 * tf.reduce_sum(reduce_logsumexp(log_PDF + tf.squeeze(log_pi)))

# Adam Optimizer

opt = tf.train.AdamOptimizer(learning_rate=0.1, beta1=0.9, beta2=0.99, epsilon=1e-5).minimize(loss)

# Assign the index of the maximum probability cluster to each point in X

assign_to_cluster = tf.math.argmax(log_posterior(log_PDF, log_pi), axis=1, output_type=tf.int32)

# Transform data set by splitting into groups (for output)

output = tf.dynamic_partition(X, assign_to_cluster, num_partitions=numClusters)

# Initialize Tensorflow variables

init = tf.global_variables_initializer()

loss_history = []

clustered = None

with tf.Session() as sess:

sess.run(init)

# Training loop

for step in range(iterations):

_MU, _sigma, _log_pi, _loss, _opt = sess.run([MU, sigma, log_pi, loss, opt], feed_dict={X: data})

loss_history.append(_loss)

# get trained parameters

trained_centroids = MU.eval()

trained_log_pi = log_pi.eval()

trained_sigma = sigma.eval()

# Assign each point to cluster based on distance to closest cluster centre

clustered = sess.run(output, feed_dict={X:data})