def simple_center_loss(y_true, y_pred):
# y_true has no fixed shape, but we require that the second dimension size is already known. Reshape it.
n = self.input_count
if self.include_self_comparison:
y_len = n * (n + 1) // 2
else:
y_len = n * (n - 1) // 2
y_true = Lambda(lambda x: y_true)(center_loss_vectors[0])
y_true = Reshape((y_len, ))(y_true)
# # Dummy values
# y_true = Lambda(lambda y_true: 0. * y_true)(y_true)
# Calculate the centers for each vector and then the loss (just MSE)
centers = reweight_values(center_loss_vectors, y_true)
return mean_squared_error(concat(centers, axis=1),
concat(center_loss_vectors, axis=1))
def __build_comparisons_arrays(self, embeddings, comparator_f, weight):
assert self.input_count == len(embeddings)
cmp_eq = [] # All comparisons assuming the elements do equal
cmp_ne = [] # All comparisons assuming the elements do not equal
comparisons = 0
for i_source in range(self.input_count):
e_source = embeddings[i_source]
# Should i_source be compared with itself?
if self._include_self_comparison:
target_range = range(i_source, self.input_count)
else:
target_range = range(i_source + 1, self.input_count)
for i_target in target_range:
e_target = embeddings[i_target]
cmp_eq.append(comparator_f(e_source, e_target, 1.))
cmp_ne.append(comparator_f(e_source, e_target, 0.))
comparisons += 1
# Create two long vectors for cmp_eq and cmp_ne
cmp_eq = concat(cmp_eq, axis=1)
cmp_ne = concat(cmp_ne, axis=1)
# # Reshape and merge them
# cmp_eq = Reshape((1, comparisons))(cmp_eq)
# cmp_ne = Reshape((1, comparisons))(cmp_ne)
#
# # Concat both
# cmp = concat([cmp_eq, cmp_ne], axis=1)
cmp = concat([cmp_eq, cmp_ne], axis=1)
# Weight the result
cmp = cmp * weight
# If this is not already the case: Convert the 'cmp' obj to a keras tensor (this command is a bit "dummy")
cmp = Lambda(lambda x: cmp)(embeddings[0])
return cmp
def _build_network(self, network_input, network_output, additional_network_outputs):
cluster_counts = list(self.data_provider.get_cluster_counts())
# The simple loss cluster NN requires a specific output: a list of softmax distributions
# First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each
# object etc. At the end, there is the cluster count output.
# First we get an embedding for the network inputs
embeddings = self._get_embedding(network_input)
# Reshape all embeddings to 1d vectors
# embedding_shape = self._embedding_nn.model.layers[-1].output_shape
# embedding_size = np.prod(embedding_shape[1:])
embedding_shape = embeddings[0].shape
embedding_size = int(str(np.prod(embedding_shape[1:])))
embedding_reshaper = self._s_layer('embedding_reshape', lambda name: Reshape((1, embedding_size), name=name))
embeddings_reshaped = [embedding_reshaper(embedding) for embedding in embeddings]
cohesion_d_layer = self._s_layer('cohesion_d_layer', lambda name: Dense(2, activation='tanh', name=name))
cohesion_embeddings = [cohesion_d_layer(embedding) for embedding in embeddings_reshaped]
self._add_additional_prediction_output(
concat(cohesion_embeddings, axis=1),
'cohesion_embeddings'
)
self._register_additional_embedding_comparison_regularisation(
'cluster_cohesion',
# If the values are no longer used (as this is the case here), we need a dissimilarity weight, because
# otherwise there is the trivial solution to map all inputs to 0
lambda x, y, similar: meier_cluster_cohesion(x, y, similar, dissimilar_distance_weight=-1),
cohesion_embeddings
)
# We need now the internal representation of the embeddings. This means we have to resize them.
internal_embedding_size = self.__internal_embedding_size // 2 * 2
embedding_internal_resizer = self._s_layer('internal_embedding_resize', lambda name: Dense(internal_embedding_size, name=name))
embeddings_reshaped = [embedding_internal_resizer(embedding) for embedding in embeddings_reshaped]
embedding_internal_resizer_act = LeakyReLU()
embeddings_reshaped = [embedding_internal_resizer_act(embedding) for embedding in embeddings_reshaped]
# Merge all embeddings to one tensor
embeddings_merged = self._s_layer('embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped)
# Use now some lstm-layers
processed = embeddings_merged
for i in range(self.__lstm_layers):
tmp = self._s_layer(
'LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(internal_embedding_size // 2, return_sequences=True), name=name)
)(processed)
processed = Add()([processed, tmp])
# Split the tensor to seperate layers
embeddings_processed = [self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input))]
# Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution.
# These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start
# with the cluster count / object combinations.
# First prepare some generally required layers
layers = []
for i in range(self.__output_dense_layers):
layers += [
self._s_layer('output_dense{}'.format(i), lambda name: Dense(self.__output_dense_units, name=name)),
self._s_layer('output_batch'.format(i), lambda name: BatchNormalization(name=name)),
LeakyReLU()
# self._s_layer('output_relu'.format(i), lambda name: Activation(LeakyReLU(), name=name))
]
cluster_softmax = {
k: self._s_layer('softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts
}
# Create now the outputs
clusters_output = additional_network_outputs['clusters'] = {}
for i in range(len(embeddings_processed)):
embedding_proc = embeddings_processed[i]
# Add the required layers
for layer in layers:
embedding_proc = layer(embedding_proc)
input_clusters_output = clusters_output['input{}'.format(i)] = {}
for k in cluster_counts:
# Create now the required softmax distributions
output_classifier = cluster_softmax[k](embedding_proc)
input_clusters_output['cluster{}'.format(k)] = output_classifier
network_output.append(output_classifier)
# Calculate the real cluster count
assert self.__cluster_count_lstm_layers >= 1
cluster_count = embeddings_merged
for i in range(self.__cluster_count_lstm_layers - 1):
cluster_count = self._s_layer('cluster_count_LSTM{}'.format(i), lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units, return_sequences=True), name=name)(cluster_count))
cluster_count = self._s_layer('cluster_count_LSTM{}_batch'.format(i), lambda name: BatchNormalization(name=name))(cluster_count)
cluster_count = self._s_layer('cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units), name=name)(cluster_count))
cluster_count = self._s_layer('cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count)
for i in range(self.__cluster_count_dense_layers):
cluster_count = self._s_layer('cluster_count_dense{}'.format(i), lambda name: Dense(self.__cluster_count_dense_units, name=name))(cluster_count)
cluster_count = self._s_layer('cluster_count_batch{}'.format(i), lambda name: BatchNormalization(name=name))(cluster_count)
cluster_count = LeakyReLU()(cluster_count)
# cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation(LeakyReLU(), name=name))(cluster_count)
# The next layer is an output-layer, therefore the name must not be formatted
cluster_count = self._s_layer(
'cluster_count_output',
lambda name: Dense(len(cluster_counts), activation='softmax', name=name),
format_name=False
)(cluster_count)
additional_network_outputs['cluster_count_output'] = cluster_count
network_output.append(cluster_count)
return True
def _build_network(self, network_input, network_output,
additional_network_outputs):
cluster_counts = self._get_cluster_counts()
# The simple loss cluster NN requires a specific output: a list of softmax distributions
# First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each
# object etc. At the end, there is the cluster count output.
# First we get an embedding for the network inputs
embeddings = self._get_embedding(network_input)
# Reshape all embeddings to 1d vectors
# embedding_shape = self._embedding_nn.model.layers[-1].output_shape
# embedding_size = np.prod(embedding_shape[1:])
embedding_shape = embeddings[0].shape
embedding_size = int(str(np.prod(embedding_shape[1:])))
embedding_reshaper = self._s_layer(
'embedding_reshape', lambda name: Reshape(
(1, embedding_size), name=name))
embeddings_reshaped = [
embedding_reshaper(embedding) for embedding in embeddings
]
# Merge all embeddings to one tensor
embeddings_merged = self._s_layer(
'embeddings_merge',
lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped)
# self._add_additional_prediction_output(embeddings_merged, 'Embeddings')
# Use now some LSTM-layer to process all embeddings
processed = embeddings_merged
# Split the tensor to seperate layers
embeddings_processed = [
self._s_layer('slice_{}'.format(i),
lambda name: slice_layer(processed, i, name))
for i in range(len(network_input))
]
# Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution.
# These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start
# with the cluster count / object combinations.
# First prepare some generally required layers
layers = []
cluster_softmax = {
k: self._s_layer(
'softmax_cluster_{}'.format(k),
lambda name: Dense(k, activation='softmax', name=name))
for k in cluster_counts
}
# Create now the outputs
clusters_output = additional_network_outputs['clusters'] = {}
for i in range(len(embeddings_processed)):
embedding_proc = embeddings_processed[i]
# Add the required layers
for layer in layers:
embedding_proc = layer(embedding_proc)
input_clusters_output = clusters_output['input{}'.format(i)] = {}
for k in cluster_counts:
# Create now the required softmax distributions
output_classifier = cluster_softmax[k](embedding_proc)
input_clusters_output['cluster{}'.format(
k)] = output_classifier
network_output.append(output_classifier)
# Calculate the real cluster count
cluster_count = embeddings_merged
cluster_count = self._s_layer(
'cluster_count_LSTM_merge',
lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units),
name=name)(cluster_count))
cluster_count = self._s_layer(
'cluster_count_LSTM_merge_batch',
lambda name: BatchNormalization(name=name))(cluster_count)
# The next layer is an output-layer, therefore the name must not be formatted
cluster_count = self._s_layer(
'cluster_count_output',
lambda name: Dense(
len(cluster_counts), activation='softmax', name=name),
format_name=False)(cluster_count)
additional_network_outputs['cluster_count_output'] = cluster_count
# Add a regularizer that is based on the "Deep Divergence-Based CLustering" paper
# We require these inputs:
# 1) A list of embeddings
# 2) A list of softmaxs
# We have for each cluster count possibility a list of softmaxs and we calculate the regularization for all
# possibilities, then they are weighted by their cluster probability.
x_inputs = embeddings
# ddbc_losses = []
d_a_losses = []
d_m_losses = []
triuAAt_losses = []
for k in cluster_counts:
s_inputs = list(
map(
lambda i: Reshape((k, ))
(additional_network_outputs['clusters']['input{}'.format(
i)]['cluster{}'.format(k)]), range(len(embeddings))))
# ddbc_losses.append(get_ddbc_loss_function(x_inputs, s_inputs)[0])
_, d_a, triuAAt, d_m = get_ddbc_loss_function(x_inputs, s_inputs)
d_a_losses.append(d_a)
d_m_losses.append(d_m)
triuAAt_losses.append(triuAAt)
# ddbc_loss = concat_layer(input_count=len(ddbc_losses), axis=1)(ddbc_losses) # Concatenate(axis=1)(ddbc_losses)
# self._add_debug_output(ddbc_loss, "ddbc_loss")
# self._add_debug_output(cluster_count, "cluster_count")
# ddbc_loss = Dot(axes=1)([ddbc_loss, cluster_count])
# self._add_debug_output(ddbc_loss, "ddbc_loss2")
# self._register_additional_regularisation(ddbc_loss, "Ddbc_Loss")
d_a_losses = concat(inputs=d_a_losses, axis=1)
d_m_losses = concat(inputs=d_m_losses, axis=1)
triuAAt_losses = concat(inputs=triuAAt_losses, axis=1)
self._add_debug_output(d_a_losses, "d_a_losses")
self._add_debug_output(d_m_losses, "d_m_losses")
self._add_debug_output(triuAAt_losses, "triuAAt_losses")
self._add_debug_output(cluster_count, "cluster_count")
d_a_losses = Dot(axes=1)([d_a_losses, cluster_count])
d_m_losses = Dot(axes=1)([d_m_losses, cluster_count])
triuAAt_losses = Dot(axes=1)([triuAAt_losses, cluster_count])
self._add_debug_output(d_a_losses, "d_a_losses2")
self._add_debug_output(d_m_losses, "d_m_losses2")
self._add_debug_output(triuAAt_losses, "triuAAt_losses2")
self._register_additional_regularisation(d_a_losses, "d_a_loss")
self._register_additional_regularisation(d_m_losses, "d_m_loss")
self._register_additional_regularisation(triuAAt_losses,
"triuAAt_losses")
network_output.append(cluster_count)
return True
def py_matrix_to_keras_matrix(M):
return concat(axis=1, inputs=list(map(
lambda x: concat(axis=2, inputs=x),
M
)))
def _build_network(self, network_input, network_output,
additional_network_outputs):
cluster_counts = list(self.data_provider.get_cluster_counts())
# The simple loss cluster NN requires a specific output: a list of softmax distributions
# First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each
# object etc. At the end, there is the cluster count output.
# First we get an embedding for the network inputs
embeddings = self._get_embedding(network_input)
# Do some dropout
d_out = self._s_layer('embedding_dout',
lambda name: Dropout(0.5, name=name))
embeddings = [d_out(embedding) for embedding in embeddings]
# Reshape all embeddings to 1d vectors
# embedding_shape = self._embedding_nn.model.layers[-1].output_shape
# embedding_size = np.prod(embedding_shape[1:])
embedding_shape = embeddings[0].shape
embedding_size = int(str(np.prod(embedding_shape[1:])))
embedding_reshaper = self._s_layer(
'embedding_reshape', lambda name: Reshape(
(1, embedding_size), name=name))
embeddings_reshaped = [
embedding_reshaper(embedding) for embedding in embeddings
]
self._add_debug_output(concat(embeddings_reshaped, axis=1),
'embeddings_reshaped')
embeddings_reshaped = reweight_values(embeddings_reshaped,
self._get_clustering_hint())
self._add_debug_output(concat(embeddings_reshaped, axis=1),
'embeddings_processed_new')
# # Implement the KL-divergence on this layer. First, like lukic et al., do another fully connedted layer
# kl_embeddings = embeddings_reshaped
# kl_dense0 = self._s_layer('kl_dense0', lambda name: Dense(self.__kl_embedding_size, name=name, activation='relu'))
# kl_embeddings = [kl_dense0(kl_embedding) for kl_embedding in kl_embeddings]
# kl_softmax = self._s_layer('kl_softmax', lambda name: Dense(self.__kl_embedding_size, name=name, activation='softmax'))
# kl_embeddings = [kl_softmax(kl_embedding) for kl_embedding in kl_embeddings]
# self._register_additional_embedding_comparison_regularisation(
# 'KL_divergence',
# lukic_kl_divergence,
# kl_embeddings,
# weight=self.__kl_divergence_factor
# )
# We need now the internal representation of the embeddings. This means we have to resize them.
internal_embedding_size = self.__internal_embedding_size // 2 * 2
embedding_internal_resizer = self._s_layer(
'internal_embedding_resize',
lambda name: Dense(internal_embedding_size, name=name))
embeddings_reshaped = [
embedding_internal_resizer(embedding)
for embedding in embeddings_reshaped
]
embedding_internal_resizer_act = LeakyReLU()
embeddings_reshaped = [
embedding_internal_resizer_act(embedding)
for embedding in embeddings_reshaped
]
# Merge all embeddings to one tensor
embeddings_merged = self._s_layer(
'embeddings_merge',
lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped)
# Use now some lstm-layers
processed = embeddings_merged
for i in range(self.__lstm_layers):
tmp = self._s_layer(
'LSTM_proc_{}'.format(i),
lambda name: Bidirectional(LSTM(internal_embedding_size // 2,
return_sequences=True),
name=name))(processed)
processed = Add()([processed, tmp])
# Split the tensor to seperate layers
embeddings_processed = [
self._s_layer('slice_{}'.format(i),
lambda name: slice_layer(processed, i, name))
for i in range(len(network_input))
]
# Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution.
# These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start
# with the cluster count / object combinations.
# First prepare some generally required layers
layers = []
for i in range(self.__output_dense_layers):
layers += [
self._s_layer(
'output_dense{}'.format(i),
lambda name: Dense(self.__output_dense_units, name=name)),
self._s_layer('output_batch'.format(i),
lambda name: BatchNormalization(name=name)),
LeakyReLU(),
Dropout(0.5)
# self._s_layer('output_relu'.format(i), lambda name: Activation(LeakyReLU(), name=name))
]
cluster_softmax = {
k: self._s_layer(
'softmax_cluster_{}'.format(k),
lambda name: Dense(k, activation='softmax', name=name))
for k in cluster_counts
}
# Create now the outputs
clusters_output = additional_network_outputs['clusters'] = {}
for i in range(len(embeddings_processed)):
embedding_proc = embeddings_processed[i]
# Add the required layers
for layer in layers:
embedding_proc = layer(embedding_proc)
input_clusters_output = clusters_output['input{}'.format(i)] = {}
for k in cluster_counts:
# Create now the required softmax distributions
output_classifier = cluster_softmax[k](embedding_proc)
input_clusters_output['cluster{}'.format(
k)] = output_classifier
network_output.append(output_classifier)
# Calculate the real cluster count
assert self.__cluster_count_lstm_layers >= 1
cluster_count = embeddings_merged
for i in range(self.__cluster_count_lstm_layers - 1):
cluster_count = self._s_layer(
'cluster_count_LSTM{}'.format(i),
lambda name: Bidirectional(LSTM(
self.__cluster_count_lstm_units, return_sequences=True),
name=name)(cluster_count))
cluster_count = self._s_layer(
'cluster_count_LSTM{}_batch'.format(i),
lambda name: BatchNormalization(name=name))(cluster_count)
cluster_count = self._s_layer(
'cluster_count_LSTM_merge',
lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units),
name=name)(cluster_count))
cluster_count = self._s_layer(
'cluster_count_LSTM_merge_batch',
lambda name: BatchNormalization(name=name))(cluster_count)
for i in range(self.__cluster_count_dense_layers):
cluster_count = self._s_layer(
'cluster_count_dense{}'.format(i),
lambda name: Dense(self.__cluster_count_dense_units, name=name
))(cluster_count)
cluster_count = self._s_layer(
'cluster_count_batch{}'.format(i),
lambda name: BatchNormalization(name=name))(cluster_count)
cluster_count = LeakyReLU()(cluster_count)
# cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation(LeakyReLU(), name=name))(cluster_count)
# The next layer is an output-layer, therefore the name must not be formatted
cluster_count = Dropout(0.5)(cluster_count)
cluster_count = self._s_layer(
'cluster_count_output',
lambda name: Dense(
len(cluster_counts), activation='softmax', name=name),
format_name=False)(cluster_count)
additional_network_outputs['cluster_count_output'] = cluster_count
network_output.append(cluster_count)
return True
