def __init__(self, K=5, dist_threshold=14):
'''
Why such dist_threshold value?
See notebook: notebooks/experiments_with_classification.ipynb
:param K:
:param dist_threshold:
'''
# current training data
self.X_train = None
self.y_train = None
self.idx_to_lbl = None
self.lbl_to_idx = None
self.y_train_idx = None
# main params
self.dist_threshold_value = dist_threshold
self.K = K
# placeholders
self.xtr = tf.placeholder(tf.float32, [None, EMBEDDING_SIZE],
name='X_train')
self.ytr = tf.placeholder(tf.float32, [None], name='y_train')
self.xte = tf.placeholder(tf.float32, [EMBEDDING_SIZE], name='x_test')
self.dist_threshold = tf.placeholder(tf.float32,
shape=(),
name="dist_threshold")
############ build model ############
# model
distance = tf.reduce_sum(tf.abs(tf.subtract(self.xtr, self.xte)),
axis=1)
values, indices = tf.nn.top_k(tf.negative(distance),
k=self.K,
sorted=False)
nn_dist = tf.negative(values)
self.valid_nn_num = tf.reduce_sum(
tf.cast(nn_dist < self.dist_threshold, tf.float32))
nn = []
for i in range(self.K):
nn.append(self.ytr[indices[i]]) # taking the result indexes
# saving list in tensor variable
nearest_neighbors = nn
# this will return the unique neighbors the count will return the most common's index
self.y, idx, self.count = tf.unique_with_counts(nearest_neighbors)
self.pred = tf.slice(self.y,
begin=[tf.argmax(self.count, 0)],
size=tf.constant([1], dtype=tf.int64))[0]
def disable_some_fgs():
# We want to delete a randomly-selected subset of fg_inds of
# size `fg_inds.shape[0] - max_fg`.
# We shuffle along the dimension 0 and then we get the first
# num_fg_inds - max_fg indices and we disable them.
shuffled_inds = tf.random_shuffle(fg_inds, seed=self._seed)
disable_place = (tf.shape(fg_inds)[0] - max_fg)
# This function should never run if num_fg_inds <= max_fg, so we
# add an assertion to catch the wrong behaviour if it happens.
integrity_assertion = tf.assert_positive(
disable_place,
message="disable_place in disable_some_fgs is negative.")
with tf.control_dependencies([integrity_assertion]):
disable_inds = shuffled_inds[:disable_place]
is_disabled = tf.sparse_to_dense(
sparse_indices=disable_inds,
sparse_values=True,
default_value=False,
output_shape=tf.cast(proposals_label_shape, tf.int64),
# We are shuffling the indices, so they may not be ordered.
validate_indices=False)
return tf.where(
condition=is_disabled,
# We set it to -label for debugging purposes.
x=tf.negative(proposals_label),
y=proposals_label)
def extend_partial_state(self, JCK, r):
"""
Extends partial state by sampling two states to coalesce (Gumbel-max trick to sample without replacement)
JumpChain (JC_K) is a tensor formed from a numpy array of lists of strings, returns a new JumpChain tensor
"""
# Compute combinatorial term
# pdb.set_trace()
q = 1 / ncr(self.N - r, 2)
data = tf.reshape(tf.range((self.N - r) * self.K), (self.K, self.N - r))
data = tf.mod(data, (self.N - r))
data = tf.cast(data, dtype=tf.float32)
# Gumbel-max trick to sample without replacement
z = -tf.math.log(-tf.math.log(tf.random.uniform(tf.shape(data), 0, 1)))
top_values, coalesced_indices = tf.nn.top_k(data + z, 2)
bottom_values, remaining_indices = tf.nn.top_k(tf.negative(data + z), self.N - r - 2)
JC_keep = tf.gather(tf.reshape(JCK, [self.K * (self.N - r)]), remaining_indices)
particles = tf.gather(tf.reshape(JCK, [self.K * (self.N - r)]), coalesced_indices)
particle1 = particles[:, 0]
particle2 = particles[:, 1]
# Form new state
particle_coalesced = particle1 + '+' + particle2
# Form new Jump Chain
JCK = tf.concat([JC_keep, tf.expand_dims(particle_coalesced, axis=1)], axis=1)
#return particle1, particle2, particle_coalesced, coalesced_indices, remaining_indices, q, JCK
return coalesced_indices, remaining_indices, q, JCK
def update_neighbour(self, bmu):
learning_rate = 0.6
sigma = tf.to_float(tf.maximum(self.width, self.height) / 2)
square_difference = tf.square(self.location - bmu)
distance = tf.sqrt(tf.reduce_mean(square_difference, axis=1))
neighbour_strength = tf.exp(
tf.div(tf.negative(tf.square(distance)), 2 * tf.square(sigma)))
rate = neighbour_strength * learning_rate
total_node = self.width * self.height
rate_stacked = tf.stack([
tf.tile(tf.slice(rate, [i], [1]), [self.input_dimension])
for i in range(total_node)
])
input_weight_difference = tf.subtract(self.input, self.weight)
weight_difference = tf.multiply(rate_stacked, input_weight_difference)
weight_new = tf.add(self.weight, weight_difference)
return tf.assign(self.weight, weight_new), rate_stacked
def mdn_loss_func(y_true, y_pred):
# Reshape inputs in case this is used in a TimeDistribued layer
y_pred = tf.reshape(y_pred,
[-1, (2 * num_mixes * output_dim) + num_mixes],
name='reshape_ypreds')
y_true = tf.reshape(y_true, [-1, output_dim], name='reshape_ytrue')
# Split the inputs into paramaters
out_mu, out_sigma, out_pi = tf.split(y_pred,
num_or_size_splits=[
num_mixes * output_dim,
num_mixes * output_dim,
num_mixes
],
axis=-1,
name='mdn_coef_split')
# Construct the mixture models
cat = tfd.Categorical(logits=out_pi)
component_splits = [output_dim] * num_mixes
mus = tf.split(out_mu, num_or_size_splits=component_splits, axis=1)
sigs = tf.split(out_sigma, num_or_size_splits=component_splits, axis=1)
coll = [
tfd.MultivariateNormalDiag(loc=loc, scale_diag=scale)
for loc, scale in zip(mus, sigs)
]
mixture = tfd.Mixture(cat=cat, components=coll)
loss = mixture.log_prob(y_true)
loss = tf.negative(loss)
loss = tf.reduce_mean(loss)
return loss
def update_neigbours(self):
learning_rate = 0.1
#Formula calculate sigma / radius
sigma = tf.to_float(tf.maximum(self.width, self.height) / 2)
#Eucledian Distance between BMU and location
square_difference = tf.square(self.bmu - self.location)
distance = tf.sqrt(tf.reduce_sum(square_difference, axis=1))
#Calculate Neighbour Strength based on formula
# NS = tf.exp((- distance ** 2) / (2 * sigma ** 2))
NS = tf.exp(
tf.div(tf.negative(tf.square(distance)), 2 * tf.square(sigma)))
#Calculate rate before reshape
rate = NS * learning_rate
#Reshape to [width * height, input_dimension]
rate_stacked = tf.stack([
tf.tile(tf.slice(rate, [i], [1]), [self.input_dimension])
for i in range(self.width * self.height)
])
#Calculate New Weight
new_weight = self.weight + rate_stacked * (self.input - self.weight)
return tf.assign(self.weight, new_weight)
def build_graph(parameters):
"""Build the neg op testing graph."""
input_tensor = tf.compat.v1.placeholder(
dtype=parameters["input_dtype"],
name="input",
shape=parameters["input_shape"])
out = tf.negative(input_tensor)
return [input_tensor], [out]
def loop_body(mc_sample):
with tape() as variational_tape:
_ = variational_model(*model_args)
params = variational_tape.values()
energy = target(*params)
entropy = tf.negative(target_q(**variational_tape))
return energy + entropy
def __build_graph(self):
self.__graph = tf.Graph()
with self.__graph.as_default(), self.__graph.device(_device_for_node):
count_max = tf.constant([self.cooccurrence_cap], dtype=tf.float32,
name='max_cooccurrence_count')
scaling_factor = tf.constant([self.scaling_factor], dtype=tf.float32,
name="scaling_factor")
self.__focal_input = tf.placeholder(tf.int32, shape=[self.batch_size],
name="focal_words")
self.__context_input = tf.placeholder(tf.int32, shape=[self.batch_size],
name="context_words")
self.__cooccurrence_count = tf.placeholder(tf.float32, shape=[self.batch_size],
name="cooccurrence_count")
focal_embeddings = tf.Variable(
tf.random_uniform([self.vocab_size, self.embedding_size], 1.0, -1.0),
name="focal_embeddings")
context_embeddings = tf.Variable(
tf.random_uniform([self.vocab_size, self.embedding_size], 1.0, -1.0),
name="context_embeddings")
focal_biases = tf.Variable(tf.random_uniform([self.vocab_size], 1.0, -1.0),
name='focal_biases')
context_biases = tf.Variable(tf.random_uniform([self.vocab_size], 1.0, -1.0),
name="context_biases")
focal_embedding = tf.nn.embedding_lookup([focal_embeddings], self.__focal_input)
context_embedding = tf.nn.embedding_lookup([context_embeddings], self.__context_input)
focal_bias = tf.nn.embedding_lookup([focal_biases], self.__focal_input)
context_bias = tf.nn.embedding_lookup([context_biases], self.__context_input)
weighting_factor = tf.minimum(
1.0,
tf.pow(
tf.div(self.__cooccurrence_count, count_max),
scaling_factor))
embedding_product = tf.reduce_sum(tf.multiply(focal_embedding, context_embedding), 1)
log_cooccurrences = tf.log(tf.to_float(self.__cooccurrence_count))
distance_expr = tf.square(tf.add_n([
embedding_product,
focal_bias,
context_bias,
tf.negative(log_cooccurrences)]))
single_losses = tf.multiply(weighting_factor, distance_expr)
self.__total_loss = tf.reduce_sum(single_losses)
tf.summary.scalar("GloVe_loss", self.__total_loss)
self.__optimizer = tf.train.AdagradOptimizer(self.learning_rate).minimize(
self.__total_loss)
self.__summary = tf.summary.merge_all()
self.__combined_embeddings = tf.add(focal_embeddings, context_embeddings,
name="combined_embeddings")
def _create_bpr_loss(self, users, pos_items, neg_items):
pos_scores = tf.reduce_sum(tf.multiply(users, pos_items), axis=1)
neg_scores = tf.reduce_sum(tf.multiply(users, neg_items), axis=1)
regularizer = tf.nn.l2_loss(users) + tf.nn.l2_loss(
pos_items) + tf.nn.l2_loss(neg_items)
maxi = tf.log(tf.nn.sigmoid(pos_scores - neg_scores))
mf_loss = tf.negative(tf.reduce_mean(maxi))
reg_loss = self.regs[0] * regularizer
return mf_loss, reg_loss
def _get_cf_loss(self):
def _get_cf_score(u_e, i_e):
cf_score = tf.reduce_sum(tf.multiply(u_e, i_e), axis=1)
return cf_score
pos_cf_score = _get_cf_score(self.u_e, self.pos_i_e)
neg_cf_score = _get_cf_score(self.u_e, self.neg_i_e)
maxi = tf.log(1e-10 + tf.nn.sigmoid(pos_cf_score - neg_cf_score))
cf_loss = tf.negative(tf.reduce_mean(maxi))
cf_reg_loss = tf.nn.l2_loss(self.u_e) + tf.nn.l2_loss(self.pos_i_e) + tf.nn.l2_loss(self.neg_i_e)
return cf_loss, cf_reg_loss
def _get_kg_loss(self):
def _get_kg_score(h_e, r_e, t_e):
kg_score = tf.reduce_sum(tf.square((h_e + r_e - t_e)), 1, keepdims=True)
return kg_score
pos_kg_score = _get_kg_score(self.h_e, self.r_e, self.pos_t_e)
neg_kg_score = _get_kg_score(self.h_e, self.r_e, self.neg_t_e)
maxi = tf.log(tf.nn.sigmoid(neg_kg_score - pos_kg_score))
kg_loss = tf.negative(tf.reduce_mean(maxi))
kg_reg_loss = tf.nn.l2_loss(self.h_e) + tf.nn.l2_loss(self.r_e) + \
tf.nn.l2_loss(self.pos_t_e) + tf.nn.l2_loss(self.neg_t_e)
return kg_loss, kg_reg_loss
def GMM_M_Step(X, Gama, ClusterNo, name='GMM_Statistics', **kwargs):
D, h, s = tf.split(X, [1,1,1], axis=3)
WXd = tf.multiply(Gama, tf.tile(D ,[1,1,1,ClusterNo]))
WXa = tf.multiply(Gama, tf.tile(h ,[1,1,1,ClusterNo]))
WXb = tf.multiply(Gama, tf.tile(s ,[1,1,1,ClusterNo]))
S = tf.reduce_sum(tf.reduce_sum(Gama, axis=1), axis=1)
S = tf.add(S, tensorflow.keras.backend.epsilon())
S = tf.reshape(S,[1, ClusterNo])
M_d = tf.div(tf.reduce_sum(tf.reduce_sum(WXd, axis=1), axis=1) , S)
M_a = tf.div(tf.reduce_sum(tf.reduce_sum(WXa, axis=1), axis=1) , S)
M_b = tf.div(tf.reduce_sum(tf.reduce_sum(WXb, axis=1), axis=1) , S)
Mu = tf.split(tf.concat([M_d, M_a, M_b],axis=0), ClusterNo, 1)
Norm_d = tf.squared_difference(D, tf.reshape(M_d,[1, ClusterNo]))
Norm_h = tf.squared_difference(h, tf.reshape(M_a,[1, ClusterNo]))
Norm_s = tf.squared_difference(s, tf.reshape(M_b,[1, ClusterNo]))
WSd = tf.multiply(Gama, Norm_d)
WSh = tf.multiply(Gama, Norm_h)
WSs = tf.multiply(Gama, Norm_s)
S_d = tf.sqrt(tf.div(tf.reduce_sum(tf.reduce_sum(WSd, axis=1), axis=1) , S))
S_h = tf.sqrt(tf.div(tf.reduce_sum(tf.reduce_sum(WSh, axis=1), axis=1) , S))
S_s = tf.sqrt(tf.div(tf.reduce_sum(tf.reduce_sum(WSs, axis=1), axis=1) , S))
Std = tf.split(tf.concat([S_d, S_h, S_s],axis=0), ClusterNo, 1)
dist = list()
for k in range(0, ClusterNo):
dist = tfp.distributions.MultivariateNormalDiag(tf.reshape(Mu[k],[1,3]), tf.reshape(Std[k],[1,3]))
PI = tf.split(Gama, ClusterNo, axis=3)
Prob0 = list()
ds = tf.expand_dims(dataset_tf(X), -2)
for k in range(0, ClusterNo):
Prob0.append(tf.multiply(tf.squeeze(dist.prob(ds[:, 0, :])), tf.squeeze(PI[k])))
Prob = tf.convert_to_tensor(Prob0, dtype=tf.float32)
Prob = tf.minimum(tf.add(tf.reduce_sum(Prob, axis=0), tensorflow.keras.backend.epsilon()), tf.constant(1.0, tf.float32))
Log_Prob = tf.negative(tf.log(Prob))
Log_Likelihood = tf.reduce_mean(Log_Prob)
return Log_Likelihood, Mu, Std
def get_prediction_loss_cosine(input_tensor_1,
input_tensor_2,
labels,
similarity_score_amplifier=6.0,
neg_to_pos_example_ratio=1.0):
"""Get prediction based on pointwise version of cosine similarity function.
Compute the model predictions and losses based on cosine similarity functions.
This setting is useful for the binary classification task or regression task.
Args:
input_tensor_1: The Tensor with shape [batch_size, embed_size] to denote the
left input text.
input_tensor_2: The Tensor with shape [batch_size, embed_size] to denote
the right input text.
labels: Float tensor with shape [batch_size]. The ground truth labels to
denote whether two documents are matched.
similarity_score_amplifier: The amplifier to increase the logits value, so
that sigmoid(logits) is closer to 0 or 1. The default value is 6.0.
neg_to_pos_example_ratio: The ratio to compensate when we have more negative
examples.
Returns:
The loss, per example loss and similarities of two input texts.
"""
with tf.variable_scope("loss/text_pair_matching"):
logits = _pointwise_cosine(input_tensor_1, input_tensor_2)
labels = tf.reshape(labels, [-1])
labels = tf.cast(labels, tf.float32)
# To compensate when we have way more neg examples than pos examples and
# to compensate for larger masked lm loss.
# Note that we use weights_2 to make sure the weight for neg examples is
# 1, not 0.
weights_1 = tf.multiply(labels, neg_to_pos_example_ratio)
weights_2 = tf.add(tf.ones_like(labels), tf.negative(labels))
# When neg_to_pos_example_ratio = 1.0, weights will be all ones.
weights = tf.add(weights_1, weights_2)
logits *= similarity_score_amplifier
per_example_loss = tf.losses.sigmoid_cross_entropy(
multi_class_labels=labels,
logits=logits,
weights=weights,
reduction=tf.losses.Reduction.NONE)
loss = tf.reduce_mean(per_example_loss)
return (loss, per_example_loss, logits)
def negative_dkl(variational_params=None,
clip_alpha=None,
eps=common.EPSILON,
log_alpha=None):
R"""Compute the negative kl-divergence loss term.
Computes the negative kl-divergence between the log-uniform prior over the
weights and the variational posterior over the weights for each element
in the set of variational parameters. Each contribution is summed and the
sum is returned as a scalar Tensor.
The true kl-divergence is intractable, so we compute the tight approximation
from https://arxiv.org/abs/1701.05369.
Args:
variational_params: 2-tuple of Tensors, where the first tensor is the \theta
values and the second contains the log of the \sigma^2 values.
clip_alpha: Int or None. If integer, we clip the log \alpha values to
[-clip_alpha, clip_alpha]. If None, don't clip the values.
eps: Small constant value to use in log and sqrt operations to avoid NaNs.
log_alpha: float32 tensor of log alpha values.
Returns:
Output scalar Tensor containing the sum of all negative kl-divergence
contributions for each element in the input variational_params.
Raises:
RuntimeError: If the variational_params argument is not a 2-tuple.
"""
if variational_params is not None:
theta, log_sigma2 = _verify_variational_params(variational_params)
if log_alpha is None:
log_alpha = common.compute_log_alpha(log_sigma2, theta, eps,
clip_alpha)
# Constant values for approximating the kl divergence
k1, k2, k3 = 0.63576, 1.8732, 1.48695
c = -k1
# Compute each term of the KL and combine
term_1 = k1 * tf.nn.sigmoid(k2 + k3 * log_alpha)
term_2 = -0.5 * tf.log1p(tf.exp(tf.negative(log_alpha)))
eltwise_dkl = term_1 + term_2 + c
return -tf.reduce_sum(eltwise_dkl)
def aggregate(self, gradients):
with tf.name_scope("GAR_krum_tf"):
# Assertion
assert len(gradients) > 0, "Empty list of gradient to aggregate"
# Distance computations
distances = []
for i in range(self.__nbworkers - 1):
dists = list()
for j in range(i + 1, self.__nbworkers):
sqr_dst = tf.reduce_sum(
tf.squared_difference(gradients[i], gradients[j]))
dists.append(
tf.negative(
tf.where(tf.is_finite(sqr_dst), sqr_dst,
tf.constant(np.inf, dtype=sqr_dst.dtype)))
) # Use of 'negative' to get the smallest distances and score indexes in 'nn.top_k'
distances.append(dists)
# Score computations
scores = []
for i in range(self.__nbworkers):
dists = []
for j in range(self.__nbworkers):
if j == i:
continue
if j < i:
dists.append(distances[j][i - j - 1])
else:
dists.append(distances[i][j - i - 1])
dists = tf.parallel_stack(dists)
dists, _ = tf.nn.top_k(dists,
k=(self.__nbworkers - self.__nbbyzwrks -
2),
sorted=False)
scores.append(tf.reduce_sum(dists))
# Average of the 'nbselected' smallest scoring gradients
gradients = tf.parallel_stack(gradients)
scores = tf.parallel_stack(scores)
_, indexes = tf.nn.top_k(scores, k=self.__nbselected, sorted=False)
return tf.reduce_mean(tf.gather(gradients, indexes), axis=0)
def _build_loss(self):
pos_scores = self._get_bi_pooling_predictions(self.sp_pos_feats)
neg_scores = self._get_bi_pooling_predictions(self.sp_neg_feats)
maxi = tf.log(1e-10 + tf.nn.sigmoid(pos_scores - neg_scores))
cf_loss = tf.negative(tf.reduce_mean(maxi))
self.base_loss = cf_loss
self.reg_loss = self.regs[0] * tf.nn.l2_loss(self.weights['h'])
# self.reg_loss = self.regs[0] * tf.nn.l2_loss(self.weights['var_factor']) + \
# self.regs[1] * tf.nn.l2_loss(self.weights['h'])
#
# for k in range(self.n_layers):
# self.reg_loss += self.regs[-1] * (tf.nn.l2_loss(self.weights['W_%d' % k]))
self.kge_loss = tf.constant(0.0, tf.float32, [1])
self.loss = self.base_loss + self.kge_loss + self.reg_loss
# Optimization process.
self.opt = tf.train.AdamOptimizer(learning_rate=self.lr).minimize(
self.loss)
def tf_propagate1(self, center, error_range, equation, error_row,
bias_row):
#(784,512)*(100,784,1)->(100,784,512)->(100,512)
l1 = tf.reduce_sum(tf.abs(equation) *
tf.expand_dims(error_range, axis=-1),
axis=1)
upper = center+l1+bias_row+tf.reduce_sum(error_row*\
tf.cast(tf.greater(error_row,0), tf.float32),axis=1)
lower = center-l1+bias_row+tf.reduce_sum(error_row*\
tf.cast(tf.less(error_row,0), tf.float32), axis=1)
appr_condition = tf.cast(
tf.logical_and(tf.less(lower, 0), tf.greater(upper, 0)),
tf.float32)
mask = appr_condition * ((upper) /
(upper - lower + tf.math.exp(-10.0)))
mask = mask + 1 - appr_condition
mask = mask * tf.cast(tf.greater(upper, 0), tf.float32)
bias_row = bias_row * mask
center = center * mask
#mask=(100,1,512)
mask = tf.expand_dims(mask, axis=1)
#(784,512)*(100,1,512)
equation = equation * mask
#(1,512)*(100,1,512)
I = tf.eye(tf.shape(mask)[-1], dtype=tf.float32)
error_row = tf.concat([error_row,\
tf.expand_dims(tf.negative(lower), axis=1)*\
I*tf.expand_dims(appr_condition, axis=1)], axis=1)
error_row = error_row * mask
return upper, lower, center, equation, error_row, bias_row
def __init__(self, atom_types, **kwargs):
self.atom_types = atom_types
self.build_forces = kwargs.get('build_forces', False)
self.precision = kwargs.get('precision', _tf.float32)
self.atomic_contributions = {}
self.atom_indices = {}
self.feature_types = {'error_weights': self.precision}
self.feature_shapes = {
'error_weights': _tf.TensorShape([
None,
])
}
for t in self.atom_types:
self.feature_types['%s_indices' % t] = _tf.int32
self.feature_shapes['%s_indices' % t] = _tf.TensorShape([None, 1])
self.label_types = {'energy': self.precision}
self.label_shapes = {
'energy': _tf.TensorShape([
None,
])
}
if self.build_forces:
self.label_types['forces'] = self.precision
self.label_shapes['forces'] = _tf.TensorShape([None, None, 3])
# The individual atomic contributions and the data iterator is set up
# in this abstact method that has to be implemented for each potential
# type
self.configureAtomicContributions(**kwargs)
# Convenience handle for backwards compatibility
self.ANNs = self.atomic_contributions
self.target = self.labels['energy']
if self.build_forces:
self.target_forces = self.labels['forces']
for t in self.atom_types:
self.atom_indices[t] = self.features['%s_indices' % t]
with _tf.name_scope('Energy_Calculation'):
self.E_predict = _tf.scatter_nd(
_tf.concat([self.atom_indices[t] for t in self.atom_types], 0),
_tf.concat([
_tf.reshape(self.atomic_contributions[t].output, [-1])
for t in self.atom_types
], 0),
_tf.shape(self.target),
name='E_prediction')
with _tf.name_scope('Atom_Counting'):
self.num_atoms = _tf.reduce_sum(
[_tf.bincount(self.atom_indices[t]) for t in self.atom_types],
axis=0,
name='NumberOfAtoms')
with _tf.name_scope('MSE'):
self.error_weights = self.features['error_weights']
self.mse = _tf.reduce_mean(
(self.target - self.E_predict)**2 * self.error_weights)
self.mse_summ = _tf.summary.scalar('MSE',
self.mse,
family='performance')
with _tf.name_scope('RMSE'):
self.error_weights = self.features['error_weights']
self.rmse = _tf.sqrt(
_tf.reduce_mean(
(self.target - self.E_predict)**2 * self.error_weights))
self.rmse_summ = _tf.summary.scalar('RMSE',
self.rmse,
family='performance')
if self.build_forces:
with _tf.name_scope('Force_Calculation'):
self.gradients = {}
for t in self.atom_types:
self.gradients[t] = _tf.gradients(
self.atomic_contributions[t].output,
self.atomic_contributions[t].input,
name='%s_gradients' % t)[0]
self.F_predict = _tf.negative(
_tf.scatter_nd(
_tf.concat(
[self.atom_indices[t] for t in self.atom_types],
0),
_tf.concat([
_tf.einsum(
'ijkl,ij->ikl',
self.atomic_contributions[t].derivatives_input,
self.gradients[t],
name='%s_einsum' % t) for t in self.atom_types
], 0),
_tf.shape(self.target_forces),
name='F_prediction'))
with _tf.name_scope('MSE_Forces'):
# Following Behler. Int. J. Quant. Chem. 2015 115, 1032-1050
# equation (21). !! Not the same !! for varying number of atoms
self.mse_forces = _tf.reduce_mean(
_tf.reduce_mean((self.target_forces - self.F_predict)**2,
axis=[1, 2]) * self.error_weights)
self.mse_forces_summ = _tf.summary.scalar('MSE_Forces',
self.mse_forces,
family='performance')
with _tf.name_scope('RMSE_Forces'):
# Following Behler. Int. J. Quant. Chem. 2015 115, 1032-1050
# equation (21). !! Not the same !! for varying number of atoms
self.rmse_forces = _tf.sqrt(
_tf.reduce_mean(
_tf.reduce_mean(
(self.target_forces - self.F_predict)**2,
axis=[1, 2]) * self.error_weights))
self.rmse_forces_summ = _tf.summary.scalar(
'RMSE_Forces', self.rmse_forces, family='performance')
self.variables = _tf.get_collection(
_tf.GraphKeys.MODEL_VARIABLES,
scope=_tf.get_default_graph().get_name_scope())
self.saver = _tf.train.Saver(self.variables,
max_to_keep=None,
save_relative_paths=True)
import input_data
#Build the Training Set
mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
train_pixels, train_list_values = mnist.train.next_batch(100)
test_pixels, test_list_of_values = mnist.test.next_batch(10)
train_pixel_tensor = tf.placeholder("float", [None, 784])
test_pixel_tensor = tf.placeholder("float", [784])
#Cost Function and distance optimization
distance = tf.reduce_sum(tf.abs(
tf.add(train_pixel_tensor, tf.negative(test_pixel_tensor))),
reduction_indices=1)
pred = tf.arg_min(distance, 0)
# Testing and algorithm evaluation
accuracy = 0.
init = tf.initialize_all_variables()
with tf.Session() as sess:
sess.run(init)
for i in range(len(test_list_of_values)):
nn_index = sess.run(pred,
feed_dict={
train_pixel_tensor: train_pixels,
test_pixel_tensor: test_pixels[i, :]
def __init__(self,
predict: Union[Callable, tf.keras.Model, 'keras.Model'],
mode: str,
shape: tuple,
kappa: float = 0.,
beta: float = .1,
feature_range: tuple = (-1e10, 1e10),
gamma: float = 0.,
ae_model: Union[tf.keras.Model, 'keras.Model'] = None,
learning_rate_init: float = 1e-2,
max_iterations: int = 1000,
c_init: float = 10.,
c_steps: int = 10,
eps: tuple = (1e-3, 1e-3),
clip: tuple = (-100., 100.),
update_num_grad: int = 1,
no_info_val: Union[float, np.ndarray] = None,
write_dir: str = None,
sess: tf.Session = None) -> None:
"""
Initialize contrastive explanation method.
Paper: https://arxiv.org/abs/1802.07623
Parameters
----------
predict
Keras or TensorFlow model or any other model's prediction function returning class probabilities
mode
Find pertinant negatives ('PN') or pertinant positives ('PP')
shape
Shape of input data starting with batch size
kappa
Confidence parameter for the attack loss term
beta
Regularization constant for L1 loss term
feature_range
Tuple with min and max ranges to allow for perturbed instances. Min and max ranges can be floats or
numpy arrays with dimension (1x nb of features) for feature-wise ranges
gamma
Regularization constant for optional auto-encoder loss term
ae_model
Optional auto-encoder model used for loss regularization
learning_rate_init
Initial learning rate of optimizer
max_iterations
Maximum number of iterations for finding a PN or PP
c_init
Initial value to scale the attack loss term
c_steps
Number of iterations to adjust the constant scaling the attack loss term
eps
If numerical gradients are used to compute dL/dx = (dL/dp) * (dp/dx), then eps[0] is used to
calculate dL/dp and eps[1] is used for dp/dx. eps[0] and eps[1] can be a combination of float values and
numpy arrays. For eps[0], the array dimension should be (1x nb of prediction categories) and for
eps[1] it should be (1x nb of features)
clip
Tuple with min and max clip ranges for both the numerical gradients and the gradients
obtained from the TensorFlow graph
update_num_grad
If numerical gradients are used, they will be updated every update_num_grad iterations
no_info_val
Global or feature-wise value considered as containing no information
write_dir
Directory to write tensorboard files to
sess
Optional Tensorflow session that will be used if passed instead of creating or inferring one internally
"""
super().__init__(meta=copy.deepcopy(DEFAULT_META_CEM))
# get params for storage in meta
params = locals()
remove = ['self', 'predict', 'ae_model', 'sess', '__class__']
for key in remove:
params.pop(key)
self.meta['params'].update(params)
self.predict = predict
# check whether the model and the auto-encoder are Keras or TF models and get session
is_model, is_model_keras, model_sess = _check_keras_or_tf(predict)
is_ae, is_ae_keras, ae_sess = _check_keras_or_tf(ae_model)
# TODO: check ae and model are compatible
self.meta['params'].update(is_model=is_model, is_model_keras=is_model_keras, is_ae=is_ae,
is_ae_keras=is_ae_keras)
# if session provided, use it
if isinstance(sess, tf.Session):
self.sess = sess
else:
self.sess = model_sess
if is_model: # Keras or TF model
self.model = True
classes = self.sess.run(self.predict(tf.convert_to_tensor(np.zeros(shape), dtype=tf.float32))).shape[1]
else:
self.model = False
classes = self.predict(np.zeros(shape)).shape[1]
self.mode = mode
self.shape = shape
self.kappa = kappa
self.beta = beta
self.gamma = gamma
self.ae = ae_model
self.batch_size = shape[0]
self.max_iterations = max_iterations
self.c_init = c_init
self.c_steps = c_steps
self.update_num_grad = update_num_grad
self.eps = eps
self.clip = clip
self.write_dir = write_dir
if type(no_info_val) == float:
self.no_info_val = np.ones(shape) * no_info_val
else:
self.no_info_val = no_info_val
# values regarded as containing no information
# PNs will deviate away from these values while PPs will gravitate towards them
self.no_info = tf.Variable(np.zeros(shape), dtype=tf.float32, name='no_info')
# define tf variables for original and perturbed instances, and target labels
self.orig = tf.Variable(np.zeros(shape), dtype=tf.float32, name='orig')
self.adv = tf.Variable(np.zeros(shape), dtype=tf.float32, name='adv') # delta(k)
self.adv_s = tf.Variable(np.zeros(shape), dtype=tf.float32, name='adv_s') # y(k)
self.target = tf.Variable(np.zeros((self.batch_size, classes)), dtype=tf.float32, name='target')
# define tf variable for constant used in FISTA optimization
self.const = tf.Variable(np.zeros(self.batch_size), dtype=tf.float32, name='const')
self.global_step = tf.Variable(0.0, trainable=False, name='global_step')
# define placeholders that will be assigned to relevant variables
self.assign_orig = tf.placeholder(tf.float32, shape, name='assign_orig')
self.assign_adv = tf.placeholder(tf.float32, shape, name='assign_adv')
self.assign_adv_s = tf.placeholder(tf.float32, shape, name='assign_adv_s')
self.assign_target = tf.placeholder(tf.float32, (self.batch_size, classes), name='assign_target')
self.assign_const = tf.placeholder(tf.float32, [self.batch_size], name='assign_const')
self.assign_no_info = tf.placeholder(tf.float32, shape, name='assign_no_info')
# define conditions and values for element-wise shrinkage thresholding (eq.7)
with tf.name_scope('shrinkage_thresholding') as scope:
cond = [tf.cast(tf.greater(tf.subtract(self.adv_s, self.orig), self.beta), tf.float32),
tf.cast(tf.less_equal(tf.abs(tf.subtract(self.adv_s, self.orig)), self.beta), tf.float32),
tf.cast(tf.less(tf.subtract(self.adv_s, self.orig), tf.negative(self.beta)), tf.float32)]
upper = tf.minimum(tf.subtract(self.adv_s, self.beta), tf.cast(feature_range[1], tf.float32))
lower = tf.maximum(tf.add(self.adv_s, self.beta), tf.cast(feature_range[0], tf.float32))
self.assign_adv = tf.multiply(cond[0], upper) + tf.multiply(cond[1], self.orig) + tf.multiply(cond[2],
lower)
# perturbation update for delta and vector projection on correct set depending on PP or PN (eq.5)
# delta(k) = adv; delta(k+1) = assign_adv
with tf.name_scope('perturbation_delta') as scope:
proj_d = [tf.cast(tf.greater(tf.abs(tf.subtract(self.assign_adv, self.no_info)),
tf.abs(tf.subtract(self.orig, self.no_info))), tf.float32),
tf.cast(tf.less_equal(tf.abs(tf.subtract(self.assign_adv, self.no_info)),
tf.abs(tf.subtract(self.orig, self.no_info))), tf.float32)]
if self.mode == "PP":
self.assign_adv = tf.multiply(proj_d[1], self.assign_adv) + tf.multiply(proj_d[0], self.orig)
elif self.mode == "PN":
self.assign_adv = tf.multiply(proj_d[0], self.assign_adv) + tf.multiply(proj_d[1], self.orig)
# perturbation update and vector projection on correct set for y: y(k+1) = assign_adv_s (eq.6)
with tf.name_scope('perturbation_y') as scope:
self.zt = tf.divide(self.global_step, self.global_step + tf.cast(3, tf.float32)) # k/(k+3) in (eq.6)
self.assign_adv_s = self.assign_adv + tf.multiply(self.zt, self.assign_adv - self.adv)
proj_d_s = [tf.cast(tf.greater(tf.abs(tf.subtract(self.assign_adv_s, self.no_info)),
tf.abs(tf.subtract(self.orig, self.no_info))), tf.float32),
tf.cast(tf.less_equal(tf.abs(tf.subtract(self.assign_adv_s, self.no_info)),
tf.abs(tf.subtract(self.orig, self.no_info))), tf.float32)]
if self.mode == "PP":
self.assign_adv_s = tf.multiply(proj_d_s[1], self.assign_adv_s) + tf.multiply(proj_d_s[0], self.orig)
elif self.mode == "PN":
self.assign_adv_s = tf.multiply(proj_d_s[0], self.assign_adv_s) + tf.multiply(proj_d_s[1], self.orig)
# delta(k) <- delta(k+1); y(k) <- y(k+1)
with tf.name_scope('update_adv') as scope:
self.adv_updater = tf.assign(self.adv, self.assign_adv)
self.adv_updater_s = tf.assign(self.adv_s, self.assign_adv_s)
# from perturbed instance, derive deviation delta
with tf.name_scope('update_delta') as scope:
self.delta = self.orig - self.adv
self.delta_s = self.orig - self.adv_s
# define L1 and L2 loss terms; L1+L2 is later used as an optimization constraint for FISTA
ax_sum = list(np.arange(1, len(shape)))
with tf.name_scope('loss_l1_l2') as scope:
self.l2 = tf.reduce_sum(tf.square(self.delta), axis=ax_sum)
self.l2_s = tf.reduce_sum(tf.square(self.delta_s), axis=ax_sum)
self.l1 = tf.reduce_sum(tf.abs(self.delta), axis=ax_sum)
self.l1_s = tf.reduce_sum(tf.abs(self.delta_s), axis=ax_sum)
self.l1_l2 = self.l2 + tf.multiply(self.l1, self.beta)
self.l1_l2_s = self.l2_s + tf.multiply(self.l1_s, self.beta)
# sum losses
self.loss_l1 = tf.reduce_sum(self.l1)
self.loss_l1_s = tf.reduce_sum(self.l1_s)
self.loss_l2 = tf.reduce_sum(self.l2)
self.loss_l2_s = tf.reduce_sum(self.l2_s)
with tf.name_scope('loss_ae') as scope:
# gamma * AE loss
if self.mode == "PP" and callable(self.ae):
self.loss_ae = self.gamma * tf.square(tf.norm(self.ae(self.delta) - self.delta))
self.loss_ae_s = self.gamma * tf.square(tf.norm(self.ae(self.delta_s) - self.delta_s))
elif self.mode == "PN" and callable(self.ae):
self.loss_ae = self.gamma * tf.square(tf.norm(self.ae(self.adv) - self.adv))
self.loss_ae_s = self.gamma * tf.square(tf.norm(self.ae(self.adv_s) - self.adv_s))
else: # no auto-encoder available
self.loss_ae = tf.constant(0.)
self.loss_ae_s = tf.constant(0.)
with tf.name_scope('loss_attack') as scope:
if not self.model:
self.loss_attack = tf.placeholder(tf.float32)
else:
# make predictions on perturbed instance (PN) or delta (PP)
if self.mode == "PP":
self.pred_proba = self.predict(self.delta)
self.pred_proba_s = self.predict(self.delta_s)
elif self.mode == "PN":
self.pred_proba = self.predict(self.adv)
self.pred_proba_s = self.predict(self.adv_s)
# probability of target label prediction
self.target_proba = tf.reduce_sum(self.target * self.pred_proba, 1)
target_proba_s = tf.reduce_sum(self.target * self.pred_proba_s, 1)
# max probability of non target label prediction
self.nontarget_proba_max = tf.reduce_max((1 - self.target) * self.pred_proba - (self.target * 10000), 1)
nontarget_proba_max_s = tf.reduce_max((1 - self.target) * self.pred_proba_s - (self.target * 10000), 1)
# loss term f(x,d) for PP (eq.4) and PN (eq.2)
if self.mode == "PP":
loss_attack = tf.maximum(0.0, self.nontarget_proba_max - self.target_proba + self.kappa)
loss_attack_s = tf.maximum(0.0, nontarget_proba_max_s - target_proba_s + self.kappa)
elif self.mode == "PN":
loss_attack = tf.maximum(0.0, -self.nontarget_proba_max + self.target_proba + self.kappa)
loss_attack_s = tf.maximum(0.0, -nontarget_proba_max_s + target_proba_s + self.kappa)
# c * f(x,d)
self.loss_attack = tf.reduce_sum(self.const * loss_attack)
self.loss_attack_s = tf.reduce_sum(self.const * loss_attack_s)
with tf.name_scope('loss_combined') as scope:
# no need for L1 term in loss to optimize when using FISTA
if self.model:
self.loss_opt = self.loss_attack_s + self.loss_l2_s + self.loss_ae_s
else: # separate numerical computation of loss attack gradient
self.loss_opt = self.loss_l2_s + self.loss_ae_s
# add L1 term to overall loss; this is not the loss that will be directly optimized
self.loss_total = self.loss_attack + self.loss_l2 + self.loss_ae + tf.multiply(self.beta, self.loss_l1)
with tf.name_scope('training') as scope:
self.learning_rate = tf.train.polynomial_decay(learning_rate_init, self.global_step,
self.max_iterations, 0, power=0.5)
optimizer = tf.train.GradientDescentOptimizer(self.learning_rate)
start_vars = set(x.name for x in tf.global_variables())
# first compute, then apply grads
self.compute_grads = optimizer.compute_gradients(self.loss_opt, var_list=[self.adv_s])
self.grad_ph = tf.placeholder(tf.float32, name='grad_adv_s')
var = [tvar for tvar in tf.trainable_variables() if tvar.name.startswith('adv_s')][-1] # get the last in
# case explainer is re-initialized and a new graph is created
grad_and_var = [(self.grad_ph, var)]
self.apply_grads = optimizer.apply_gradients(grad_and_var, global_step=self.global_step)
end_vars = tf.global_variables()
new_vars = [x for x in end_vars if x.name not in start_vars]
# variables to initialize
self.setup = [] # type: list
self.setup.append(self.orig.assign(self.assign_orig))
self.setup.append(self.target.assign(self.assign_target))
self.setup.append(self.const.assign(self.assign_const))
self.setup.append(self.adv.assign(self.assign_adv))
self.setup.append(self.adv_s.assign(self.assign_adv_s))
self.setup.append(self.no_info.assign(self.assign_no_info))
self.init = tf.variables_initializer(var_list=[self.global_step] + [self.adv_s] + [self.adv] + new_vars)
if self.write_dir is not None:
writer = tf.summary.FileWriter(write_dir, tf.get_default_graph())
writer.add_graph(tf.get_default_graph())
def __init__(self, iSquaredMapDim, iInputLength, learning_rate, maxIter):
self.iSquaredMapDim = iSquaredMapDim
iNeurons = self.iSquaredMapDim * self.iSquaredMapDim
self.maxIter = maxIter
self.dRadius = self.iSquaredMapDim * .6
self.tfGraph = tf.Graph()
# Initialize a TensorFlow computation graph
with self.tfGraph.as_default():
# Initializing variables
tf.set_random_seed(0)
self.NeuronWeights = tf.Variable(
tf.random_normal([iNeurons, iInputLength]))
self.NeuronLocation = self.generateIndexMatrix()
# Input placeholders
self.inputPlaceholder = tf.placeholder("float", [iInputLength])
self.iterInputPlaceholder = tf.placeholder("float")
# Calculating best mapping unit (BMU) and its location
input_matix = tf.stack(
[self.inputPlaceholder for i in range(iNeurons)])
euclidenDistances = tf.sqrt(
tf.reduce_sum(
tf.pow(tf.subtract(self.NeuronWeights, input_matix), 2),
1))
bmu = tf.argmin(euclidenDistances, 0)
mask = tf.pad(tf.reshape(bmu, [1]), np.array([[0, 1]]))
size = tf.cast(tf.constant(np.array([1, 2])), dtype=tf.int64)
bmu_location = tf.reshape(
tf.slice(self.NeuronLocation, mask, size), [2])
# Calculate learning rate and radius
decay_function = tf.subtract(
1.0, tf.div(self.iterInputPlaceholder, self.maxIter))
_current_learning_rate = tf.multiply(learning_rate, decay_function)
_current_radius = tf.multiply(self.dRadius, decay_function)
# Adapt learning rate to each neuron based on position
bmu_matrix = tf.stack([bmu_location for i in range(iNeurons)])
bmu_distance = tf.reduce_sum(
tf.pow(tf.subtract(self.NeuronLocation, bmu_matrix), 2), 1)
# Gaussian distrbution
gaussianNeighbourhood = tf.exp(
tf.negative(
tf.div(tf.cast(bmu_distance, "float32"),
tf.pow(_current_radius, 2))))
learning_rate_matrix = tf.multiply(_current_learning_rate,
gaussianNeighbourhood)
# Update all the weights
multiplytiplier = tf.stack([
tf.tile(
tf.slice(learning_rate_matrix, np.array([i]),
np.array([1])), [iInputLength])
for i in range(iNeurons)
])
delta = tf.multiply(
multiplytiplier,
tf.subtract(
tf.stack([self.inputPlaceholder for i in range(iNeurons)]),
self.NeuronWeights))
new_weights = tf.add(self.NeuronWeights, delta)
self._training = tf.assign(self.NeuronWeights, new_weights)
#Initilize session and run it
self._sess = tf.Session()
self._sess.run(tf.global_variables_initializer())
return
(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()
x_train = x_train / 255
x_test = x_test / 255
# k-param
k_max = 30
# number of test samples - to be able to reduce total execution time
n = 5000
# TensorFlow Graph calculation model
x_train_ph = tf.placeholder(tf.float32, shape=x_train.shape)
y_train_ph = tf.placeholder(tf.float32, shape=y_train.shape)
x_test_ph = tf.placeholder(tf.float32, shape=x_test.shape[1:])
# Calculate L1-distances as negative to allow picking first top K entries after DESC sorting
distances = tf.negative(
tf.reduce_sum(tf.reduce_sum(tf.abs(tf.subtract(x_train_ph, x_test_ph)),
axis=1),
axis=1))
# Find top K entries after DESC sorting
top_k_values, top_k_indices = tf.nn.top_k(distances, k=k_max + 1, sorted=True)
top_k_max_labels = tf.gather(y_train_ph, top_k_indices)
predictions = []
# Calculate predictions for different k - [1, k_max]
for k in range(1, k_max + 1):
top_k_labels = tf.slice(top_k_max_labels, begin=[0], size=[k])
unique_classes, ids, top_k_labels_counts = tf.unique_with_counts(
top_k_labels)
prediction = tf.gather(unique_classes, tf.argmax(top_k_labels_counts))
predictions.append(prediction)
predictions = tf.stack(predictions)
# Start TensorFlow Session
def __init__(self, board_width, board_height, model_file=None):
self.board_width = board_width
self.board_height = board_height
# Define the tensorflow neural network
# 1. Input:
self.input_states = tf.placeholder(
tf.float32, shape=[None, 4, board_height, board_width])
self.input_state = tf.transpose(self.input_states, [0, 2, 3, 1])
# 2. Common Networks Layers
self.conv1 = tf.layers.conv2d(inputs=self.input_state,
filters=32,
kernel_size=[3, 3],
padding="same",
data_format="channels_last",
activation=tf.nn.relu)
self.conv2 = tf.layers.conv2d(inputs=self.conv1,
filters=64,
kernel_size=[3, 3],
padding="same",
data_format="channels_last",
activation=tf.nn.relu)
self.conv3 = tf.layers.conv2d(inputs=self.conv2,
filters=128,
kernel_size=[3, 3],
padding="same",
data_format="channels_last",
activation=tf.nn.relu)
# 3-1 Action Networks
self.action_conv = tf.layers.conv2d(inputs=self.conv3,
filters=4,
kernel_size=[1, 1],
padding="same",
data_format="channels_last",
activation=tf.nn.relu)
# Flatten the tensor
self.action_conv_flat = tf.reshape(
self.action_conv, [-1, 4 * board_height * board_width])
# 3-2 Full connected layer, the output is the log probability of moves
# on each slot on the board
self.action_fc = tf.layers.dense(inputs=self.action_conv_flat,
units=board_height * board_width,
activation=tf.nn.log_softmax)
# 4 Evaluation Networks
self.evaluation_conv = tf.layers.conv2d(inputs=self.conv3,
filters=2,
kernel_size=[1, 1],
padding="same",
data_format="channels_last",
activation=tf.nn.relu)
self.evaluation_conv_flat = tf.reshape(
self.evaluation_conv, [-1, 2 * board_height * board_width])
self.evaluation_fc1 = tf.layers.dense(inputs=self.evaluation_conv_flat,
units=64,
activation=tf.nn.relu)
# output the score of evaluation on current state
self.evaluation_fc2 = tf.layers.dense(inputs=self.evaluation_fc1,
units=1,
activation=tf.nn.tanh)
# Define the Loss function
# 1. Label: the array containing if the game wins or not for each state
self.labels = tf.placeholder(tf.float32, shape=[None, 1])
# 2. Predictions: the array containing the evaluation score of each state
# which is self.evaluation_fc2
# 3-1. Value Loss function
self.value_loss = tf.losses.mean_squared_error(self.labels,
self.evaluation_fc2)
# 3-2. Policy Loss function
self.mcts_probs = tf.placeholder(
tf.float32, shape=[None, board_height * board_width])
self.policy_loss = tf.negative(
tf.reduce_mean(
tf.reduce_sum(tf.multiply(self.mcts_probs, self.action_fc),
1)))
# 3-3. L2 penalty (regularization)
l2_penalty_beta = 1e-4
vars = tf.trainable_variables()
l2_penalty = l2_penalty_beta * tf.add_n(
[tf.nn.l2_loss(v) for v in vars if 'bias' not in v.name.lower()])
# 3-4 Add up to be the Loss function
self.loss = self.value_loss + self.policy_loss + l2_penalty
# Define the optimizer we use for training
self.learning_rate = tf.placeholder(tf.float32)
self.optimizer = tf.train.AdamOptimizer(
learning_rate=self.learning_rate).minimize(self.loss)
# Make a session
self.session = tf.Session()
# calc policy entropy, for monitoring only
self.entropy = tf.negative(
tf.reduce_mean(
tf.reduce_sum(tf.exp(self.action_fc) * self.action_fc, 1)))
# Initialize variables
init = tf.global_variables_initializer()
self.session.run(init)
# For saving and restoring
self.saver = tf.train.Saver()
if model_file is not None:
self.restore_model(model_file)
K = 4
mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
# In this example, we limit mnist data
Xtr, Ytr = mnist.train.next_batch(55000) # whole training set
Xte, Yte = mnist.test.next_batch(100) # whole test set
# tf Graph Input
xtr = tf.placeholder("float", [None, 784])
ytr = tf.placeholder("float", [None, 10])
xte = tf.placeholder("float", [784])
# Euclidean Distance
distance = tf.negative(
tf.sqrt(
tf.reduce_sum(tf.square(tf.subtract(xtr, xte)), reduction_indices=1)))
# Prediction: Get min distance neighbors
values, indices = tf.nn.top_k(distance, k=K, sorted=False)
nearest_neighbors = []
for i in range(K):
nearest_neighbors.append(tf.argmax(ytr[indices[i]], 0))
neighbors_tensor = tf.stack(nearest_neighbors)
y, idx, count = tf.unique_with_counts(neighbors_tensor)
pred = tf.slice(y,
begin=[tf.argmax(count, 0)],
size=tf.constant([1], dtype=tf.int64))[0]
accuracy = 0.
my_kernel = tf.exp(tf.multiply(gamma, tf.abs(sq_dists)))
def reshape_matmul(mat):
v1 = tf.expand_dims(mat, 1)
v2 = tf.reshape(v1, [5, batch_size, 1])
return (tf.matmul(v2, v1))
first_term = tf.reduce_sum(b)
b_vec_cross = tf.matmul(tf.transpose(b), b)
y_target_cross = reshape_matmul(y_target)
second_term = tf.reduce_sum(
tf.multiply(my_kernel, tf.multiply(b_vec_cross, y_target_cross)))
loss = tf.negative(tf.subtract(first_term, second_term))
rA = tf.reshape(tf.reduce_sum(tf.square(x_data), 1), [-1, 1])
rB = tf.reshape(tf.reduce_sum(tf.square(prediction_grid), 1), [-1, 1])
pred_sq_dist = tf.add(
tf.subtract(
rA, tf.multiply(2., tf.matmul(x_data, tf.transpose(prediction_grid)))),
tf.transpose(rB))
pred_kernel = tf.exp(tf.multiply(gamma, tf.abs(pred_sq_dist)))
prediction_output = tf.matmul(tf.multiply(y_target, b), pred_kernel)
prediction = tf.arg_max(
prediction_output -
tf.expand_dims(tf.reduce_mean(prediction_output, 1), 1), 0)
accuracy = tf.reduce_mean(
tf.cast(tf.equal(prediction, tf.argmax(y_target, 0)), tf.float32))
tr_encode = y_train[5000:10000]
te_encode = y_train[10000:10200]
Ytr = tf.one_hot(tr_encode, 10)
Yte = tf.one_hot(te_encode, 10)
Xtr = x_train.iloc[5000:10000, :]
Xte = x_train.iloc[10000:10200, :]
Xtr = np.array(Xtr)
Xte = np.array(Xte)
xtr = tf.placeholder("float", [None, 784])
xte = tf.placeholder("float", [784])
# Nearest Neighbor calculation using L1 Distance
# Calculate L1 Distance
distance = tf.reduce_sum(tf.abs(tf.add(xtr, tf.negative(xte))),
reduction_indices=1)
# Prediction: Get min distance index (Nearest neighbor)
pred = tf.arg_min(distance, 0)
accuracy = 0
# Initialize the variables (i.e. assign their default value)
init = tf.global_variables_initializer()
# Start training
with tf.Session() as sess:
# Run the initializer
sess.run(init)
def _flip_gradients(op, grad):
return [tf.negative(grad) * l]
def _tower_som(self):
""" Build a single SOM tower on the TensorFlow graph """
# Randomly initialized weights for all neurons, stored together
# as a matrix Variable of shape [num_neurons, input_dims]
with tf.name_scope('Weights'):
# Each tower will get its own copy of the weights variable. Since the towers are constructed sequentially,
# the handle to the Tensors will be different for each tower even if we reference "self"
self._weights = tf.get_variable(
name='weights',
shape=[self._m * self._n, self._dim],
initializer=tf.random_uniform_initializer(maxval=1))
with tf.name_scope('summaries'):
# All summary ops are added to a list and then the merge() function is called at the end of
# this method
mean = tf.reduce_mean(self._weights)
self._summary_list.append(tf.summary.scalar('mean', mean))
with tf.name_scope('stdev'):
stdev = tf.sqrt(
tf.reduce_mean(
tf.squared_difference(self._weights, mean)))
self._summary_list.append(tf.summary.scalar('stdev', stdev))
self._summary_list.append(
tf.summary.scalar('max', tf.reduce_max(self._weights)))
self._summary_list.append(
tf.summary.scalar('min', tf.reduce_min(self._weights)))
self._summary_list.append(
tf.summary.histogram('histogram', self._weights))
# Matrix of size [m*n, 2] for SOM grid locations of neurons.
# Maps an index to an (x,y) coordinate of a neuron in the map for calculating the neighborhood distance
self._location_vects = tf.constant(np.array(
list(self._neuron_locations())),
name='Location_Vectors')
with tf.name_scope('Input'):
self._input = tf.identity(self._input_tensor)
# Start by computing the best matching units / winning units for each input vector in the batch.
# Basically calculates the Euclidean distance between every
# neuron's weight vector and the inputs, and returns the index of the neurons which give the least value
# Since we are doing batch processing of the input, we need to calculate a BMU for each of the individual
# inputs in the batch. Will have the shape [batch_size]
# Oh also any time we call expand_dims it's almost always so we can make TF broadcast stuff properly
with tf.name_scope('BMU_Indices'):
# Distance between weights and the input vector
# Note we are reducing along 2nd axis so we end up with a tensor of [batch_size, num_neurons]
# corresponding to the distance between a particular input and each neuron in the map
# Also note we are getting the squared distance because there's no point calling sqrt or tf.norm
# if we're just doing a strict comparison
squared_distance = tf.reduce_sum(
tf.pow(
tf.subtract(tf.expand_dims(self._weights, axis=0),
tf.expand_dims(self._input, axis=1)), 2), 2)
# Get the index of the minimum distance for each input item, shape will be [batch_size],
bmu_indices = tf.argmin(squared_distance, axis=1)
# This will extract the location of the BMU in the map for each input based on the BMU's indices
with tf.name_scope('BMU_Locations'):
# Using tf.gather we can use `bmu_indices` to index the location vectors directly
bmu_locs = tf.reshape(tf.gather(self._location_vects, bmu_indices),
[-1, 2])
with tf.name_scope('Learning_Rate'):
# With each epoch, the initial sigma value decreases linearly
radius = tf.subtract(
self._initial_radius,
tf.multiply(
self._epoch,
tf.divide(
tf.cast(tf.subtract(self._initial_radius, 1),
tf.float32),
tf.cast(tf.subtract(self._max_epochs, 1),
tf.float32))))
alpha = tf.multiply(
self._initial_learning_rate,
tf.subtract(
1.0,
tf.divide(tf.cast(self._epoch, tf.float32),
tf.cast(self._max_epochs, tf.float32))))
# Construct the op that will generate a matrix with learning rates for all neurons and all inputs,
# based on iteration number and location to BMU
# Start by getting the squared difference between each BMU location and every other unit in the map
# bmu_locs is [batch_size, 2], i.e. the coordinates of the BMU for each input vector.
# location vects shape should be [1, num_neurons, 2]
# bmu_locs should be [batch_size, 1, 2]
# Output needs to be [batch_size, num_neurons], i.e. a row vector of distances for each input item
bmu_distance_squares = tf.reduce_sum(
tf.pow(
tf.subtract(tf.expand_dims(self._location_vects, axis=0),
tf.expand_dims(bmu_locs, axis=1)), 2), 2)
# Using the distances between each BMU, construct the Gaussian neighborhood function.
# Basically, neurons which are close to the winner will move more than those further away.
# The radius tensor decreases the width of the Gaussian over time, so early in training more
# neurons will be affected by the winner and by the end of training only the winner will move.
# This tensor will be of shape [batch_size, num_neurons] as well and will be the value multiplied to
# each neuron based on its distance from the BMU for each input vector
neighbourhood_func = tf.exp(
tf.divide(
tf.negative(tf.cast(bmu_distance_squares, "float32")),
tf.multiply(
tf.square(tf.multiply(radius, self._std_coeff)), 2)))
# Finally multiply by the learning rate to decrease overall neuron movement over time
learning_rate_op = tf.multiply(neighbourhood_func, alpha)
# The batch formula for SOMs multiplies a neuron's neighborhood by all of the input vectors in the batch,
# then divides that by just the sum of the neighborhood function for each of the inputs.
# We are writing this in a way that performs that operation for each of the neurons in the map.
with tf.name_scope('Update_Weights'):
# The numerator needs to be shaped [num_neurons, dimensions] to represent the new weights
# for each of the neurons. At this point, the learning rate tensor will be
# shaped [batch_size, neurons].
# The end result is that, for each neuron in the network, we use the learning
# rate between it and each of the input vectors, to calculate a new set of weights.
numerator = tf.reduce_sum(tf.multiply(
tf.expand_dims(learning_rate_op, axis=-1),
tf.expand_dims(self._input, axis=1)),
axis=0)
# The denominator is just the sum of the neighborhood functions for each neuron, so we get the sum
# along axis 1 giving us an output shape of [num_neurons]. We then expand the dims so we can
# broadcast for the division op. Again we transpose the learning rate tensor so it's
# [num_neurons, batch_size] representing the learning rate of each neuron for each input vector
denominator = tf.expand_dims(
tf.reduce_sum(learning_rate_op, axis=0) + float(1e-12),
axis=-1)
# We on;y really care about summaries from one of the tower SOMs, so assign the merge op to
# the last tower we make. Otherwise there's way too many on Tensorboard.
self._merged = tf.summary.merge(self._summary_list)
# With multi-gpu training we collect the results and do the weight assignment on the CPU
return numerator, denominator
sess.run(tf.global_variables_initializer())
A_3_eval = sess.run(A_3, feed_dict={ X: X_train })
# Predicted Output
print(A_3_eval)
################################################################################
# Metrics #
################################################################################
# Target Placeholders
Y = tf.placeholder(tf.float32, shape=(2, None), name='Y')
# Negative Log-Likelihood Multi-Class Loss
L = tf.negative(tf.reduce_sum(tf.multiply(Y, tf.log(A_3))))
# Accuracy
accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(Y, axis=0), tf.argmax(A_3, axis=0)), tf.float32))
# Execution
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
L_eval, accuracy_eval = sess.run([L, accuracy], feed_dict={ X: X_train, Y: Y_train })
# Evaluated Metrics
print(L_eval, accuracy_eval)
################################################################################
