def _build_network(self, network_input, network_output,
additional_network_outputs):
cluster_counts = list(self.data_provider.get_cluster_counts())
# The simple loss cluster NN requires a specific output: a list of softmax distributions
# First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each
# object etc. At the end, there is the cluster count output.
# First we get an embedding for the network inputs
embeddings = self._get_embedding(network_input)
# Reshape all embeddings to 1d vectors
# embedding_shape = self._embedding_nn.model.layers[-1].output_shape
# embedding_size = np.prod(embedding_shape[1:])
embedding_shape = embeddings[0].shape
embedding_size = int(str(np.prod(embedding_shape[1:])))
embedding_reshaper = self._s_layer(
'embedding_reshape', lambda name: Reshape(
(1, embedding_size), name=name))
embeddings_reshaped = [
embedding_reshaper(embedding) for embedding in embeddings
]
# The KL-divergence only makes sense if an embedding is used (otherwise no embedding can be optimized)
if self._uses_embedding_layer() and self.__kl_divergence_factor > 0.:
# Implement the KL-divergence on this layer. First, like lukic et al., do another fully connedted layer
kl_embeddings = embeddings_reshaped
kl_dense0 = self._s_layer(
'kl_dense0',
lambda name: LeakyReLU(self.__kl_embedding_size, name=name))
# kl_dense0 = self._s_layer('kl_dense0', lambda name: Activation(self.__kl_embedding_size, name=name, activation='relu'))
kl_embeddings = [
kl_dense0(kl_embedding) for kl_embedding in kl_embeddings
]
kl_softmax = self._s_layer(
'kl_softmax', lambda name: Dense(
self.__kl_embedding_size, name=name, activation='softmax'))
kl_embeddings = [
kl_softmax(kl_embedding) for kl_embedding in kl_embeddings
]
self._register_additional_embedding_comparison_regularisation(
'KL_divergence',
lukic_kl_divergence,
kl_embeddings,
weight=self.__kl_divergence_factor)
# We need now the internal representation of the embeddings. This means we have to resize them.
internal_embedding_size = self.__internal_embedding_size // 2 * 2
embedding_internal_resizer = self._s_layer(
'internal_embedding_resize',
lambda name: Dense(internal_embedding_size, name=name))
embeddings_reshaped = [
embedding_internal_resizer(embedding)
for embedding in embeddings_reshaped
]
embedding_internal_resizer_act = LeakyReLU()
embeddings_reshaped = [
embedding_internal_resizer_act(embedding)
for embedding in embeddings_reshaped
]
# Merge all embeddings to one tensor
embeddings_merged = self._s_layer(
'embeddings_merge',
lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped)
# Use now some lstm-layers
processed = embeddings_merged
second_last_processed = None
for i in range(self.__lstm_layers):
second_last_processed = processed
tmp = self._s_layer(
'LSTM_proc_{}'.format(i),
lambda name: Bidirectional(LSTM(internal_embedding_size // 2,
return_sequences=True),
name=name))(processed)
processed = Add()([processed, tmp])
# Split the tensor to seperate layers
embeddings_processed = [
self._s_layer('slice_{}'.format(i),
lambda name: slice_layer(processed, i, name))
for i in range(len(network_input))
]
# Implement the simplified center loss
center_loss_vectors = embeddings_processed
def simple_center_loss(y_true, y_pred):
# y_true has no fixed shape, but we require that the second dimension size is already known. Reshape it.
n = self.input_count
if self.include_self_comparison:
y_len = n * (n + 1) // 2
else:
y_len = n * (n - 1) // 2
y_true = Lambda(lambda x: y_true)(center_loss_vectors[0])
y_true = Reshape((y_len, ))(y_true)
# # Dummy values
# y_true = Lambda(lambda y_true: 0. * y_true)(y_true)
# Calculate the centers for each vector and then the loss (just MSE)
centers = reweight_values(center_loss_vectors, y_true)
return mean_squared_error(concat(centers, axis=1),
concat(center_loss_vectors, axis=1))
if self.__simplified_center_loss_factor > 0.:
self._register_additional_grouping_similarity_loss(
'simple_center_loss',
simple_center_loss,
False,
weight=self.__simplified_center_loss_factor)
# Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution.
# These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start
# with the cluster count / object combinations.
# First prepare some generally required layers
layers = []
for i in range(self.__output_dense_layers):
layers += [
self._s_layer(
'output_dense{}'.format(i),
lambda name: Dense(self.__output_dense_units, name=name)),
self._s_layer('output_batch'.format(i),
lambda name: BatchNormalization(name=name)),
LeakyReLU(),
self._s_layer('output_relu'.format(i),
lambda name: Activation(LeakyReLU(), name=name)),
Dropout(0.5)
]
# Add forward pass dropout
if self._try_get_additional_build_config_value(
'forward_pass_dropout', default_value=False):
layers.append(
ExtendedDropout(0.5,
train_phase_active=False,
test_phase_active=True))
cluster_softmax = {
k: self._s_layer(
'softmax_cluster_{}'.format(k),
lambda name: Dense(k, activation='softmax', name=name))
for k in cluster_counts
}
# Create now the outputs
clusters_output = additional_network_outputs['clusters'] = {}
for i in range(len(embeddings_processed)):
embedding_proc = embeddings_processed[i]
# Add the required layers
for layer in layers:
embedding_proc = layer(embedding_proc)
input_clusters_output = clusters_output['input{}'.format(i)] = {}
for k in cluster_counts:
# Create now the required softmax distributions
output_classifier = cluster_softmax[k](embedding_proc)
input_clusters_output['cluster{}'.format(
k)] = output_classifier
network_output.append(output_classifier)
# Calculate the real cluster count
assert self.__cluster_count_lstm_layers >= 1
cluster_count = second_last_processed
for i in range(self.__cluster_count_lstm_layers - 1):
cluster_count = self._s_layer(
'cluster_count_LSTM{}'.format(i),
lambda name: Bidirectional(LSTM(
self.__cluster_count_lstm_units, return_sequences=True),
name=name)(cluster_count))
cluster_count = self._s_layer(
'cluster_count_LSTM{}_batch'.format(i),
lambda name: BatchNormalization(name=name))(cluster_count)
cluster_count = self._s_layer(
'cluster_count_LSTM_merge',
lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units),
name=name)(cluster_count))
cluster_count = self._s_layer(
'cluster_count_LSTM_merge_batch',
lambda name: BatchNormalization(name=name))(cluster_count)
for i in range(self.__cluster_count_dense_layers):
cluster_count = self._s_layer(
'cluster_count_dense{}'.format(i),
lambda name: Dense(self.__cluster_count_dense_units, name=name
))(cluster_count)
cluster_count = LeakyReLU()(cluster_count)
cluster_count = self._s_layer(
'cluster_count_batch{}'.format(i),
lambda name: BatchNormalization(name=name))(cluster_count)
cluster_count = Dropout(0.5)(cluster_count)
# Add forward pass dropout
if self._try_get_additional_build_config_value(
'forward_pass_dropout', default_value=False):
cluster_count = ExtendedDropout(
0.5, train_phase_active=False,
test_phase_active=True)(cluster_count)
# cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation(LeakyReLU(), name=name))(cluster_count)
# The next layer is an output-layer, therefore the name must not be formatted
cluster_count = self._s_layer(
'cluster_count_output',
lambda name: Dense(
len(cluster_counts), activation='softmax', name=name),
format_name=False)(cluster_count)
additional_network_outputs['cluster_count_output'] = cluster_count
network_output.append(cluster_count)
return True
def _build_network(self, network_input, network_output,
additional_network_outputs):
cluster_counts = list(self.data_provider.get_cluster_counts())
# The simple loss cluster NN requires a specific output: a list of softmax distributions
# First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each
# object etc. At the end, there is the cluster count output.
# First we get an embedding for the network inputs
embeddings = self._get_embedding(network_input)
# Reshape all embeddings to 1d vectors
# embedding_shape = self._embedding_nn.model.layers[-1].output_shape
# embedding_size = np.prod(embedding_shape[1:])
embedding_shape = embeddings[0].shape
embedding_size = int(str(np.prod(embedding_shape[1:])))
embedding_reshaper = self._s_layer(
'embedding_reshape', lambda name: Reshape(
(1, embedding_size), name=name))
embeddings_reshaped = [
embedding_reshaper(embedding) for embedding in embeddings
]
# Merge all embeddings to one tensor
embeddings_merged = self._s_layer(
'embeddings_merge',
lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped)
self._add_additional_prediction_output(embeddings_merged, 'Embeddings')
# Use now some LSTM-layer to process all embeddings
processed = embeddings_merged
for i in range(self.__lstm_layers):
processed = self._s_layer(
'LSTM_proc_{}'.format(i),
lambda name: Bidirectional(LSTM(self.__lstm_units,
return_sequences=True),
name=name))(processed)
processed = self._s_layer(
'LSTM_proc_{}_batch'.format(i),
lambda name: BatchNormalization(name=name))(processed)
# Split the tensor to seperate layers
embeddings_processed = [
self._s_layer('slice_{}'.format(i),
lambda name: slice_layer(processed, i, name))
for i in range(len(network_input))
]
# Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution.
# These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start
# with the cluster count / object combinations.
# First prepare some generally required layers
layers = []
for i in range(self.__output_dense_layers):
layers += [
self._s_layer(
'output_dense{}'.format(i),
lambda name: Dense(self.__output_dense_units, name=name)),
self._s_layer('output_batch'.format(i),
lambda name: BatchNormalization(name=name)),
LeakyReLU()
# self._s_layer('output_relu'.format(i), lambda name: Activation(LeakyReLU(), name=name))
]
cluster_softmax = {
k: self._s_layer(
'softmax_cluster_{}'.format(k),
lambda name: Dense(k, activation='softmax', name=name))
for k in cluster_counts
}
# Create now the outputs
clusters_output = additional_network_outputs['clusters'] = {}
for i in range(len(embeddings_processed)):
embedding_proc = embeddings_processed[i]
# Add the required layers
for layer in layers:
embedding_proc = layer(embedding_proc)
input_clusters_output = clusters_output['input{}'.format(i)] = {}
for k in cluster_counts:
# Create now the required softmax distributions
output_classifier = cluster_softmax[k](embedding_proc)
input_clusters_output['cluster{}'.format(
k)] = output_classifier
network_output.append(output_classifier)
# Calculate the real cluster count
assert self.__cluster_count_lstm_layers >= 1
for i in range(self.__cluster_count_lstm_layers - 1):
cluster_count = self._s_layer(
'cluster_count_LSTM{}'.format(i),
lambda name: Bidirectional(LSTM(
self.__cluster_count_lstm_units, return_sequences=True),
name=name)(embeddings_merged))
cluster_count = self._s_layer(
'cluster_count_LSTM{}_batch'.format(i),
lambda name: BatchNormalization(name=name))(cluster_count)
cluster_count = self._s_layer(
'cluster_count_LSTM_merge',
lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units),
name=name)(cluster_count))
cluster_count = self._s_layer(
'cluster_count_LSTM_merge_batch',
lambda name: BatchNormalization(name=name))(cluster_count)
for i in range(self.__cluster_count_dense_layers):
cluster_count = self._s_layer(
'cluster_count_dense{}'.format(i),
lambda name: Dense(self.__cluster_count_dense_units, name=name
))(cluster_count)
cluster_count = self._s_layer(
'cluster_count_batch{}'.format(i),
lambda name: BatchNormalization(name=name))(cluster_count)
cluster_count = LeakyReLU()(cluster_count)
# cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation(LeakyReLU(), name=name))(cluster_count)
# The next layer is an output-layer, therefore the name must not be formatted
cluster_count = self._s_layer(
'cluster_count_output',
lambda name: Dense(
len(cluster_counts), activation='softmax', name=name),
format_name=False)(cluster_count)
additional_network_outputs['cluster_count_output'] = cluster_count
network_output.append(cluster_count)
return True
def _build_network(self, network_input, network_output, additional_network_outputs):
cluster_counts = list(self.data_provider.get_cluster_counts())
# The simple loss cluster NN requires a specific output: a list of softmax distributions
# First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each
# object etc. At the end, there is the cluster count output.
# First we get an embedding for the network inputs
embeddings = self._get_embedding(network_input)
# Reshape all embeddings to 1d vectors
# embedding_shape = self._embedding_nn.model.layers[-1].output_shape
# embedding_size = np.prod(embedding_shape[1:])
embedding_shape = embeddings[0].shape
embedding_size = int(str(np.prod(embedding_shape[1:])))
embedding_reshaper = self._s_layer('embedding_reshape', lambda name: Reshape((1, embedding_size), name=name))
embeddings_reshaped = [embedding_reshaper(embedding) for embedding in embeddings]
# Merge all embeddings to one tensor
embeddings_merged = self._s_layer('embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped)
self._add_additional_prediction_output(embeddings_merged, 'Embeddings')
# The value range of the embeddings must be [-1, 1] to allow an efficient execution of the cluster evaluations,
# so be sure the embeddings are processed by tanh or some similar activation
# Use now some LSTM-layer to process all embeddings
processed = embeddings_merged
for i in range(self.__lstm_layers):
processed = self._s_layer(
'LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(self.__lstm_units, return_sequences=True), name=name)
)(processed)
processed = self._s_layer(
'LSTM_proc_{}_batch'.format(i), lambda name: BatchNormalization(name=name)
)(processed)
# Split the tensor to seperate layers
embeddings_processed = [self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input))]
# Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution.
# These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start
# with the cluster count / object combinations.
# First prepare some generally required layers
layers = []
for i in range(self.__output_dense_layers):
layers += [
self._s_layer('output_dense{}'.format(i), lambda name: Dense(self.__output_dense_units, name=name)),
self._s_layer('output_batch'.format(i), lambda name: BatchNormalization(name=name)),
self._s_layer('output_relu'.format(i), lambda name: Activation('relu', name=name))
]
cluster_softmax = {
k: self._s_layer('softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts
}
# Create now the outputs
clusters_output = additional_network_outputs['clusters'] = {}
cluster_classifiers = {k: [] for k in cluster_counts}
for i in range(len(embeddings_processed)):
embedding_proc = embeddings_processed[i]
# Add the required layers
for layer in layers:
embedding_proc = layer(embedding_proc)
input_clusters_output = clusters_output['input{}'.format(i)] = {}
for k in cluster_counts:
# Create now the required softmax distributions
output_classifier = cluster_softmax[k](embedding_proc)
input_clusters_output['cluster{}'.format(k)] = output_classifier
network_output.append(output_classifier)
cluster_classifiers[k].append(output_classifier)
cluster_quality_loss = 0
alpha = 0.5
beta = 0.25
self._add_debug_output(Concatenate(axis=1)(embeddings_reshaped), 'eval_embeddings')
for k in cluster_counts:
# Create evaluation metrics
self._add_debug_output(Concatenate(axis=1)(cluster_classifiers[k]), 'eval_classifications_k{}'.format(k))
cluster_centers = get_cluster_centers(embeddings_reshaped, cluster_classifiers[k])
self._add_debug_output(Concatenate(axis=1)(cluster_centers), 'eval_cluster_centers_k{}'.format(k))
cohesion = get_cluster_cohesion(cluster_centers, embeddings_reshaped, cluster_classifiers[k])
self._add_debug_output(cohesion, 'eval_cohesion_k{}'.format(k))
separation = get_cluster_separation(cluster_centers, cluster_classifiers[k])
self._add_debug_output(separation, 'eval_separation_k{}'.format(k))
cluster_quality_loss = Lambda(lambda cohesion:
# Update the loss
cluster_quality_loss + (alpha * cohesion - beta * separation)
)(cohesion)
cluster_quality_loss = Lambda(lambda x: x / len(cluster_counts))(cluster_quality_loss)
additional_network_outputs['additional_loss'] = Activation('linear', name='additional_loss')(cluster_quality_loss)
# Calculate the real cluster count
cluster_count = self._s_layer('cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__lstm_units), name=name)(embeddings_merged))
cluster_count = self._s_layer('cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count)
for i in range(self.__cluster_count_dense_layers):
cluster_count = self._s_layer('cluster_count_dense{}'.format(i), lambda name: Dense(self.__cluster_count_dense_units, name=name))(cluster_count)
cluster_count = self._s_layer('cluster_count_batch{}'.format(i), lambda name: BatchNormalization(name=name))(cluster_count)
cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation('relu', name=name))(cluster_count)
# The next layer is an output-layer, therefore the name must not be formatted
cluster_count = self._s_layer(
'cluster_count_output',
lambda name: Dense(len(cluster_counts), activation='softmax', name=name),
format_name=False
)(cluster_count)
additional_network_outputs['cluster_count_output'] = cluster_count
network_output.append(cluster_count)
return True
def _build_network(self, network_input, network_output, additional_network_outputs):
cluster_counts = list(self.data_provider.get_cluster_counts())
# The simple loss cluster NN requires a specific output: a list of softmax distributions
# First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each
# object etc. At the end, there is the cluster count output.
# First we get an embedding for the network inputs
embeddings = self._get_embedding(network_input)
# Reshape all embeddings to 1d vectors
# embedding_shape = self._embedding_nn.model.layers[-1].output_shape
# embedding_size = np.prod(embedding_shape[1:])
embedding_shape = embeddings[0].shape
embedding_size = int(str(np.prod(embedding_shape[1:])))
embedding_reshaper = self._s_layer('embedding_reshape', lambda name: Reshape((1, embedding_size), name=name))
embeddings_reshaped = [embedding_reshaper(embedding) for embedding in embeddings]
cohesion_d_layer = self._s_layer('cohesion_d_layer', lambda name: Dense(2, activation='tanh', name=name))
cohesion_embeddings = [cohesion_d_layer(embedding) for embedding in embeddings_reshaped]
self._add_additional_prediction_output(
concat(cohesion_embeddings, axis=1),
'cohesion_embeddings'
)
self._register_additional_embedding_comparison_regularisation(
'cluster_cohesion',
# If the values are no longer used (as this is the case here), we need a dissimilarity weight, because
# otherwise there is the trivial solution to map all inputs to 0
lambda x, y, similar: meier_cluster_cohesion(x, y, similar, dissimilar_distance_weight=-1),
cohesion_embeddings
)
# We need now the internal representation of the embeddings. This means we have to resize them.
internal_embedding_size = self.__internal_embedding_size // 2 * 2
embedding_internal_resizer = self._s_layer('internal_embedding_resize', lambda name: Dense(internal_embedding_size, name=name))
embeddings_reshaped = [embedding_internal_resizer(embedding) for embedding in embeddings_reshaped]
embedding_internal_resizer_act = LeakyReLU()
embeddings_reshaped = [embedding_internal_resizer_act(embedding) for embedding in embeddings_reshaped]
# Merge all embeddings to one tensor
embeddings_merged = self._s_layer('embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped)
# Use now some lstm-layers
processed = embeddings_merged
for i in range(self.__lstm_layers):
tmp = self._s_layer(
'LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(internal_embedding_size // 2, return_sequences=True), name=name)
)(processed)
processed = Add()([processed, tmp])
# Split the tensor to seperate layers
embeddings_processed = [self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input))]
# Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution.
# These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start
# with the cluster count / object combinations.
# First prepare some generally required layers
layers = []
for i in range(self.__output_dense_layers):
layers += [
self._s_layer('output_dense{}'.format(i), lambda name: Dense(self.__output_dense_units, name=name)),
self._s_layer('output_batch'.format(i), lambda name: BatchNormalization(name=name)),
LeakyReLU()
# self._s_layer('output_relu'.format(i), lambda name: Activation(LeakyReLU(), name=name))
]
cluster_softmax = {
k: self._s_layer('softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts
}
# Create now the outputs
clusters_output = additional_network_outputs['clusters'] = {}
for i in range(len(embeddings_processed)):
embedding_proc = embeddings_processed[i]
# Add the required layers
for layer in layers:
embedding_proc = layer(embedding_proc)
input_clusters_output = clusters_output['input{}'.format(i)] = {}
for k in cluster_counts:
# Create now the required softmax distributions
output_classifier = cluster_softmax[k](embedding_proc)
input_clusters_output['cluster{}'.format(k)] = output_classifier
network_output.append(output_classifier)
# Calculate the real cluster count
assert self.__cluster_count_lstm_layers >= 1
cluster_count = embeddings_merged
for i in range(self.__cluster_count_lstm_layers - 1):
cluster_count = self._s_layer('cluster_count_LSTM{}'.format(i), lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units, return_sequences=True), name=name)(cluster_count))
cluster_count = self._s_layer('cluster_count_LSTM{}_batch'.format(i), lambda name: BatchNormalization(name=name))(cluster_count)
cluster_count = self._s_layer('cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units), name=name)(cluster_count))
cluster_count = self._s_layer('cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count)
for i in range(self.__cluster_count_dense_layers):
cluster_count = self._s_layer('cluster_count_dense{}'.format(i), lambda name: Dense(self.__cluster_count_dense_units, name=name))(cluster_count)
cluster_count = self._s_layer('cluster_count_batch{}'.format(i), lambda name: BatchNormalization(name=name))(cluster_count)
cluster_count = LeakyReLU()(cluster_count)
# cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation(LeakyReLU(), name=name))(cluster_count)
# The next layer is an output-layer, therefore the name must not be formatted
cluster_count = self._s_layer(
'cluster_count_output',
lambda name: Dense(len(cluster_counts), activation='softmax', name=name),
format_name=False
)(cluster_count)
additional_network_outputs['cluster_count_output'] = cluster_count
network_output.append(cluster_count)
return True
def _build_network(self, network_input, network_output,
additional_network_outputs):
cluster_counts = list(self.data_provider.get_cluster_counts())
# f_update_global_state
def f_update_global_state(x, s):
xs = self._s_layer('f_update_global_state_CONCAT',
lambda name: Concatenate(axis=2, name=name))(
[x, s])
res = self._s_layer(
'f_update_global_state_BDLSTM', lambda name: Bidirectional(
LSTM(self.__f_update_global_state_lstm_units), name=name))(
xs)
res = self._s_layer(
'f_update_global_state_BDLSTM_BNORM',
lambda name: BatchNormalization(name=name))(res)
for i in range(self.__f_update_global_state_dense_layer_count):
res = self._s_layer(
'f_update_global_state_DENSE{}'.format(i),
lambda name: Dense(self.__global_state_size, name=name))(
res)
res = self._s_layer(
'f_update_global_state_DENSE_BNORM{}'.format(i),
lambda name: BatchNormalization(name=name))(res)
res = self._s_layer('f_update_global_state_RELU{}'.format(i),
lambda name: Activation('relu'))(res)
return res
# f_update_local_state
def f_update_local_state(x, g):
g = self._s_layer(
'f_update_local_state_REPEAT',
lambda name: RepeatVector(self.input_count, name=name))(g)
xg = self._s_layer('f_update_local_state_CONCAT',
lambda name: Concatenate(axis=2, name=name))(
[x, g])
for i in range(self.__f_update_local_state_dense_layer_count):
xg = self._s_layer(
'f_update_local_state_DENSE{}'.format(i),
lambda name: TimeDistributed(
Dense(self.__f_update_local_state_units), name=name))(
xg)
xg = self._s_layer(
'f_update_local_state_BNORM{}'.format(i),
lambda name: BatchNormalization(name=name))(xg)
xg = self._s_layer('f_update_local_state_RELU{}'.format(i),
lambda name: Activation('relu'))(xg)
xg = self._s_layer(
'f_update_local_state_OUT_DENSE',
lambda name: TimeDistributed(Dense(self.__local_state_size),
name=name))(xg)
xg = self._s_layer('f_update_local_state_OUT_BNORM',
lambda name: BatchNormalization(name=name))(xg)
xg = self._s_layer('f_update_local_state_OUT_RELU',
lambda name: Activation('relu'))(xg)
return xg
# f_cluster_count
def f_cluster_count(g):
for i in range(self.__f_cluster_count_dense_layer_count):
g = self._s_layer(
'f_cluster_count_DENSE{}'.format(i), lambda name: Dense(
self.__f_cluster_count_units, name=name))(g)
g = self._s_layer(
'f_cluster_count_BNORM{}'.format(i),
lambda name: BatchNormalization(name=name))(g)
g = self._s_layer('f_cluster_count_RELU{}'.format(i),
lambda name: Activation('relu'))(g)
return g
# f_cluster_assignment
def f_cluster_assignment(x, g):
g = self._s_layer(
'f_cluster_assignment_gRESHAPE', lambda name: Reshape(
(1, self.__global_state_size), name=name))(g)
xg = self._s_layer('f_cluster_assignment_CONCAT',
lambda name: Concatenate(axis=2, name=name))(
[x, g])
for i in range(self.__f_cluster_assignment_dense_layer_count):
xg = self._s_layer(
'f_cluster_assignment_DENSE{}'.format(i),
lambda name: Dense(self.__f_cluster_assignment_units,
name=name))(xg)
xg = self._s_layer(
'f_cluster_assignment_BNORM{}'.format(i),
lambda name: BatchNormalization(name=name))(xg)
xg = self._s_layer('f_cluster_assignment_RELU{}'.format(i),
lambda name: Activation('relu'))(xg)
return xg
# First we get an embedding for the network inputs
embeddings = self._get_embedding(network_input)
# Reshape all embeddings to 1d vectors
embedding_shape = self._embedding_nn.model.layers[-1].output_shape
embedding_size = np.prod(embedding_shape[1:])
embedding_reshaper = self._s_layer(
'embedding_reshape', lambda name: Reshape(
(1, embedding_size), name=name))
embeddings_reshaped = [
embedding_reshaper(embedding) for embedding in embeddings
]
# Merge all embeddings to one tensor
embeddings = self._s_layer(
'embeddings_merge',
lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped)
# Create empty initial local states
s = self._s_layer(
'local_states_init',
lambda name: TimeDistributed(Dense(self.__local_state_size,
kernel_initializer='zeros',
bias_initializer='zeros',
trainable=False),
name=name))(embeddings)
# Do the iterations
for i in range(self.__iterations):
# Update the gloabl state
g = f_update_global_state(embeddings, s)
# Update the local states
s = f_update_local_state(embeddings, g)
# Get the processed embeddings (=states) as a list
embeddings_processed = [
self._s_layer('slice_{}'.format(i),
lambda name: slice_layer(s, i, name))
for i in range(len(network_input))
]
# Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution.
# These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start
# with the cluster count / object combinations.
# First prepare some generally required layers
cluster_softmax = {
k: self._s_layer(
'softmax_cluster_{}'.format(k),
lambda name: Dense(k, activation='softmax', name=name))
for k in cluster_counts
}
# Create now the outputs
clusters_output = additional_network_outputs['clusters'] = {}
for i in range(len(embeddings_processed)):
embedding_proc = embeddings_processed[i]
embedding_proc = f_cluster_assignment(embedding_proc, g)
input_clusters_output = clusters_output['input{}'.format(i)] = {}
for k in cluster_counts:
# Create now the required softmax distributions
output_classifier = cluster_softmax[k](embedding_proc)
input_clusters_output['cluster{}'.format(
k)] = output_classifier
network_output.append(output_classifier)
# Calculate the real cluster count
# cluster_count = self._s_layer('cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__lstm_units), name=name)(embeddings_merged))
# cluster_count = self._s_layer('cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count)
cluster_count = f_cluster_count(g)
# The next layer is an output-layer, therefore the name must not be formatted
cluster_count = self._s_layer(
'cluster_count_output',
lambda name: Dense(
len(cluster_counts), activation='softmax', name=name),
format_name=False)(cluster_count)
additional_network_outputs['cluster_count_output'] = cluster_count
network_output.append(cluster_count)
return True
def _build_network(self, network_input, network_output, additional_network_outputs):
cluster_counts = list(self.data_provider.get_cluster_counts())
# The simple loss cluster NN requires a specific outputlayer: a list of softmax distributions
# First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each
# object etc. At the end, there is the cluster count output.
def point_preprocessor(p):
return Lambda(lambda x: K.tanh(x))(p)
# Obsolete
def add_dbg_output(name, layer):
self._add_debug_output(layer, name, create_wrapper_layer=True)
# debug_output.append(Activation('linear', name=name)(layer))
# First we get an embedding for the network inputs
embeddings = self._get_embedding(network_input)
# Reshape all embeddings to 1d vectors
# embedding_shape = self._embedding_nn.model.layers[-1].output_shape
# embedding_size = np.prod(embedding_shape[1:])
embedding_shape = embeddings[0].shape
embedding_size = int(str(np.prod(embedding_shape[1:])))
embedding_reshaper = self._s_layer('embedding_reshape', lambda name: Reshape((1, embedding_size), name=name))
embeddings_reshaped = [embedding_reshaper(embedding) for embedding in embeddings]
# Merge all embeddings to one tensor
embeddings_merged = self._s_layer('embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped)
# Use now some BDLSTM-layer to process all embeddings.
processed = embeddings_merged
for i in range(self.__lstm_layers):
processed = self._s_layer(
'LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(self.__lstm_units, return_sequences=True), name=name)
)(processed)
processed = self._s_layer(
'LSTM_proc_{}_batch'.format(i), lambda name: BatchNormalization(name=name)
)(processed)
# Batch normalize everything if not already done
if self.__lstm_layers == 0:
add_dbg_output('EMBEDDINGS', processed)
processed = BatchNormalization()(processed)
add_dbg_output('EMBEDDINGS_NORMALIZED', processed)
# Split the tensor to seperate layers
embeddings_processed = [self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input))]
# Prepare the embeddings for the kmeans input
if self.__kmeans_input_dimension is not None:
# We just update the input dimensions for the kmeans, therefore no activation function is used
layers = [
self._s_layer('kmeans_dimension_changer_dense', lambda name: Dense(self.__kmeans_input_dimension, name=name)),
self._s_layer('kmeans_dimension_changer_batch', lambda name: BatchNormalization(name=name))
]
for layer in layers:
embeddings_processed = [layer(e) for e in embeddings_processed]
# Apply the preprocessing for all embeddings (we force the network to have all values in the range [-1, 1]);
# this makes kmeans easier
embeddings_processed = [point_preprocessor(e) for e in embeddings_processed]
add_dbg_output('EMBEDDINGS_PREPROCESSED', Concatenate(axis=1)(embeddings_processed))
self._add_additional_prediction_output(Concatenate(axis=1)(embeddings_processed), 'Processed_Embeddings')
# Define a distance-function
d = lambda x, y: K.sqrt(K.sum(K.square(x - y))) # euclidean distance
def euclideanDistance(inputs, squared=False):
if (len(inputs) != 2):
raise 'oops'
# For better gradient flow: remove the sqrt
output = K.sum(K.square(inputs[0] - inputs[1]), axis=-1)
if not squared:
output = K.sqrt(output)
output = K.expand_dims(output, 1)
return output
# Apply k-means for each possible k
assert self.__kmeans_itrs > 0
cluster_vector_size = self.__kmeans_input_dimension #(self.__lstm_units * 2) if self.__lstm_layers > 0 else embedding_size
cluster_assignements = {}
def guess_initial_cluster(name):
def get_name(layer_name):
return "{}_{}".format(name, layer_name)
def res(input_layer):
nw = input_layer
nw = self._s_layer(
get_name('guess_initial_cluster_bdlstm0'),
lambda name: Bidirectional(LSTM(cluster_vector_size * cluster_counts[-1], name=name))
)(nw)
nw = self._s_layer(
get_name('guess_initial_cluster_dense0'),
lambda name: Dense(cluster_vector_size)
)(nw)
nw = self._s_layer(
get_name('guess_initial_cluster_reshape0'),
lambda name: Reshape((1, cluster_vector_size))
)(nw)
return nw
return res
for k in cluster_counts:
# Create initial cluster centers
clusters = [self._s_layer(
'k_{}_init_{}'.format(k, i), lambda name: gaussian_random_layer((1, cluster_vector_size), name=name, only_execute_for_training=False)
)(embeddings[0]) for i in range(k)]
# # Create initial cluster guesses
con_proc_embd = Concatenate(axis=1)(embeddings_processed)
clusters = [guess_initial_cluster("k{}_{}".format(k, i))(con_proc_embd) for i in range(k)]
# Apply the preprocessing
clusters = [point_preprocessor(c) for c in clusters]
self._add_additional_prediction_output(
Concatenate(axis=1)(clusters),
'Initial cluster guesses (k={})'.format(k)
)
# # # Use the first n input points
# clusters = embeddings_processed[:k]
# # Create initial cluster centers:
# # 1) The first cluster center is the mean of all points
# def l_mean(inputs):
# inputs_len = len(inputs)
# l = add(inputs)
# l = Lambda(lambda x: x / inputs_len)(l)
# return l
# clusters = [l_mean(embeddings_processed)]
# # 2) Create now all other cluster centers
# if k > 1:
# for i in range(1, k):
# # Sum over all points
# c_s = 0
# s_s = 0
#
# for e in embeddings_processed:
# # Get the distances to all points from the current embedding
# dists = []
# for c in clusters:
# dists.append(Lambda(lambda x: euclideanDistance(x, False), output_shape=lambda x: (x[0][0], 1))([c, e]))
#
# if len(dists) == 1:
# dists = dists[0]
# else:
# dists = Concatenate()(dists)
#
# # Calculate a weight for this data point
# w = Lambda(lambda x: K.exp(10 * K.min(x, axis=0)))(dists)
#
# # Update c_s and s_s
# c_s = Lambda(lambda x: c_s + w * x)(e)
# s_s = Lambda(lambda x: s_s + w)(e)
#
# # Calculate the new cluster center
# c_s = Lambda(lambda x: c_s / s_s)(c_s)
#
# # Append it to the clusters list
# clusters.append(c_s)
# # Choose k random points and use them as initial clusters
# c_i_embeddings = Concatenate(axis=1)(embeddings_processed)
# c_i_embeddings = Lambda(lambda x: tf.random_shuffle(x))(c_i_embeddings) # non-differentiable;
# clusters = [slice_layer(c_i_embeddings, i) for i in range(k)]
for i in range(len(clusters)):
add_dbg_output('INIT_CLUSTER_{}'.format(i), clusters[i])
# Cluster-assignements
current_cluster_assignements = None
# Idee zum neuen Mittelwert finden:
# - Nicht mit Softmax rechnen sondern nur noch mit der Distanz
# - Clusterzentrum mit Punkten Gewichten, falls die Distanz zum Cluster vom Punkt minimal ist (also kein anderer Cluster näher ist)
# Do all iterations
for i in range(self.__kmeans_itrs):
def get_val_at(input, i_i):
def at(val, indices):
for index in indices:
val = val[index]
return val
return Lambda(lambda x: at(x, i_i), output_shape=(1,))(input)
# Recalculate the cluster centers (if required)
if i > 0:
clusters_old = clusters
clusters = []
for c_i in range(k):
c = 0
s = 1e-8 # Avoid zero-divison
for e_i in range(len(embeddings_processed)):
t = get_val_at(current_cluster_assignements[e_i], [1, c_i])
c = Lambda(lambda x: c + x * t, output_shape=(1, cluster_vector_size))(embeddings_processed[e_i])
# c = Lambda(lambda x: c + x * 10. * K.relu(t + 0.1 - K.max(current_cluster_assignements[e_i])), output_shape=(1, cluster_vector_size))(embeddings_processed[e_i])
s = Lambda(lambda x: x + s, output_shape=(1,))(t)
# t = current_cluster_assignements[e_i][c_i]
# c += t * embeddings_processed[e_i]
# s += t
c = Lambda(lambda x: x / s, output_shape=(1, cluster_vector_size))(c)
# c = c / s
c = Lambda(lambda x: 0 + x, output_shape=(1, cluster_vector_size))(c)
add_dbg_output("k{}_ITR{}_ci{}".format(k, i, c_i), c)
clusters.append(c)
# Recalculate the assigned cluster centers for each embedding
cluster_assignements[k] = []
current_cluster_assignements = cluster_assignements[k]
e_i = 0
for e in embeddings_processed:
# Calculate all distances to all cluster centers
k_distances = [
Lambda(lambda x: euclideanDistance(x, True), output_shape=lambda x: (x[0][0], 1))([e, cluster])
# merge([e, cluster], mode=euclideanDistance, output_shape=lambda x: (x[0][0], 1))
for cluster in clusters
# self._s_layer('kmeans_d', lambda name: Merge([]))
]
# Merge the distances and calculate a softmax
k_distances = Concatenate(axis=1)(k_distances)
k_distances = Reshape((k,))(k_distances)
add_dbg_output("k{}_ITR{}_e_i{}_DISTANCES_PLAIN".format(k, i, e_i), k_distances)
# k_distances = Lambda(lambda x: 1 / (0.01 + x))(k_distances)
# k_distances = Lambda(lambda x: -(1+x)**2)(k_distances)
def kmax(c, x):
return K.relu(x - c) + c
def kmin(c, x):
return -kmax(-c, -x)
k_distances = Lambda(lambda x: -(1 + x)**2)(k_distances)
# k_distances = Lambda(lambda x: -(1 + 3 * K.sqrt(x)) ** 3)(k_distances)
# cluster_assignements[p_i][c_i] = -min(500, (1 + 3 * np.sqrt(d)) ** 3) # + 3/(1.+d)
k_distances = Activation('softmax')(k_distances)
# # Dirty tune the softmax a bit
# k_distances = Lambda(lambda x: (x / K.max(x))**4)(k_distances)
# Save the new distances
current_cluster_assignements.append(k_distances)
add_dbg_output("k{}_ITR{}_e_i{}_DISTANCES".format(k, i, e_i), k_distances)
e_i += 1
self._add_additional_prediction_output(
Concatenate(axis=1)(clusters),
'Final cluster guesses (k={})'.format(k)
)
# Reshape all softmax layers
for k in cluster_counts:
cluster_assignements[k] = [
Reshape((1, k))(s) for s in cluster_assignements[k]
]
# Append all softmax layers to the network output
network_output += chain.from_iterable(map(
lambda k: cluster_assignements[k],
cluster_counts
))
# Create the additional cluster output
clusters_output = additional_network_outputs['clusters'] = {}
for i in range(self.input_count):
input_clusters_output = clusters_output['input{}'.format(i)] = {}
for k in cluster_counts:
input_clusters_output['cluster{}'.format(k)] = cluster_assignements[k][i]
add_dbg_output('RESULT_INPUT{}_K{}'.format(i, k), cluster_assignements[k][i])
# Calculate the real cluster count
cluster_count = self._s_layer('cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__lstm_units), name=name)(embeddings_merged))
cluster_count = self._s_layer('cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count)
for i in range(self.__cluster_count_dense_layers):
cluster_count = self._s_layer('cluster_count_dense{}'.format(i), lambda name: Dense(self.__cluster_count_dense_units, name=name))(cluster_count)
cluster_count = self._s_layer('cluster_count_batch{}'.format(i), lambda name: BatchNormalization(name=name))(cluster_count)
cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation('relu', name=name))(cluster_count)
# The next layer is an output-layer, therefore the name must not be formatted
cluster_count = self._s_layer(
'cluster_count_output',
lambda name: Dense(len(cluster_counts), activation='softmax', name=name),
format_name=False
)(cluster_count)
additional_network_outputs['cluster_count_output'] = cluster_count
network_output.append(cluster_count)
return True
def _build_network(self, network_input, network_output,
additional_network_outputs):
cluster_counts = self._get_cluster_counts()
# The simple loss cluster NN requires a specific output: a list of softmax distributions
# First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each
# object etc. At the end, there is the cluster count output.
# First we get an embedding for the network inputs
embeddings = self._get_embedding(network_input)
# Reshape all embeddings to 1d vectors
# embedding_shape = self._embedding_nn.model.layers[-1].output_shape
# embedding_size = np.prod(embedding_shape[1:])
embedding_shape = embeddings[0].shape
embedding_size = int(str(np.prod(embedding_shape[1:])))
embedding_reshaper = self._s_layer(
'embedding_reshape', lambda name: Reshape(
(1, embedding_size), name=name))
embeddings_reshaped = [
embedding_reshaper(embedding) for embedding in embeddings
]
# Merge all embeddings to one tensor
embeddings_merged = self._s_layer(
'embeddings_merge',
lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped)
# self._add_additional_prediction_output(embeddings_merged, 'Embeddings')
# Use now some LSTM-layer to process all embeddings
processed = embeddings_merged
# Split the tensor to seperate layers
embeddings_processed = [
self._s_layer('slice_{}'.format(i),
lambda name: slice_layer(processed, i, name))
for i in range(len(network_input))
]
# Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution.
# These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start
# with the cluster count / object combinations.
# First prepare some generally required layers
layers = []
cluster_softmax = {
k: self._s_layer(
'softmax_cluster_{}'.format(k),
lambda name: Dense(k, activation='softmax', name=name))
for k in cluster_counts
}
# Create now the outputs
clusters_output = additional_network_outputs['clusters'] = {}
for i in range(len(embeddings_processed)):
embedding_proc = embeddings_processed[i]
# Add the required layers
for layer in layers:
embedding_proc = layer(embedding_proc)
input_clusters_output = clusters_output['input{}'.format(i)] = {}
for k in cluster_counts:
# Create now the required softmax distributions
output_classifier = cluster_softmax[k](embedding_proc)
input_clusters_output['cluster{}'.format(
k)] = output_classifier
network_output.append(output_classifier)
# Calculate the real cluster count
cluster_count = embeddings_merged
cluster_count = self._s_layer(
'cluster_count_LSTM_merge',
lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units),
name=name)(cluster_count))
cluster_count = self._s_layer(
'cluster_count_LSTM_merge_batch',
lambda name: BatchNormalization(name=name))(cluster_count)
# The next layer is an output-layer, therefore the name must not be formatted
cluster_count = self._s_layer(
'cluster_count_output',
lambda name: Dense(
len(cluster_counts), activation='softmax', name=name),
format_name=False)(cluster_count)
additional_network_outputs['cluster_count_output'] = cluster_count
# Add a regularizer that is based on the "Deep Divergence-Based CLustering" paper
# We require these inputs:
# 1) A list of embeddings
# 2) A list of softmaxs
# We have for each cluster count possibility a list of softmaxs and we calculate the regularization for all
# possibilities, then they are weighted by their cluster probability.
x_inputs = embeddings
# ddbc_losses = []
d_a_losses = []
d_m_losses = []
triuAAt_losses = []
for k in cluster_counts:
s_inputs = list(
map(
lambda i: Reshape((k, ))
(additional_network_outputs['clusters']['input{}'.format(
i)]['cluster{}'.format(k)]), range(len(embeddings))))
# ddbc_losses.append(get_ddbc_loss_function(x_inputs, s_inputs)[0])
_, d_a, triuAAt, d_m = get_ddbc_loss_function(x_inputs, s_inputs)
d_a_losses.append(d_a)
d_m_losses.append(d_m)
triuAAt_losses.append(triuAAt)
# ddbc_loss = concat_layer(input_count=len(ddbc_losses), axis=1)(ddbc_losses) # Concatenate(axis=1)(ddbc_losses)
# self._add_debug_output(ddbc_loss, "ddbc_loss")
# self._add_debug_output(cluster_count, "cluster_count")
# ddbc_loss = Dot(axes=1)([ddbc_loss, cluster_count])
# self._add_debug_output(ddbc_loss, "ddbc_loss2")
# self._register_additional_regularisation(ddbc_loss, "Ddbc_Loss")
d_a_losses = concat(inputs=d_a_losses, axis=1)
d_m_losses = concat(inputs=d_m_losses, axis=1)
triuAAt_losses = concat(inputs=triuAAt_losses, axis=1)
self._add_debug_output(d_a_losses, "d_a_losses")
self._add_debug_output(d_m_losses, "d_m_losses")
self._add_debug_output(triuAAt_losses, "triuAAt_losses")
self._add_debug_output(cluster_count, "cluster_count")
d_a_losses = Dot(axes=1)([d_a_losses, cluster_count])
d_m_losses = Dot(axes=1)([d_m_losses, cluster_count])
triuAAt_losses = Dot(axes=1)([triuAAt_losses, cluster_count])
self._add_debug_output(d_a_losses, "d_a_losses2")
self._add_debug_output(d_m_losses, "d_m_losses2")
self._add_debug_output(triuAAt_losses, "triuAAt_losses2")
self._register_additional_regularisation(d_a_losses, "d_a_loss")
self._register_additional_regularisation(d_m_losses, "d_m_loss")
self._register_additional_regularisation(triuAAt_losses,
"triuAAt_losses")
network_output.append(cluster_count)
return True
def _build_network(self, network_input, network_output,
additional_network_outputs):
cluster_counts = list(self.data_provider.get_cluster_counts())
# The simple loss cluster NN requires a specific output: a list of softmax distributions
# First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each
# object etc. At the end, there is the cluster count output.
# First we get an embedding for the network inputs
embeddings = self._get_embedding(network_input)
# Reshape all embeddings to 1d vectors
# embedding_shape = self._embedding_nn.model.layers[-1].output_shape
# embedding_size = np.prod(embedding_shape[1:])
embedding_shape = embeddings[0].shape
embedding_size = int(str(np.prod(embedding_shape[1:])))
embedding_reshaper = self._s_layer(
'embedding_reshape', lambda name: Reshape(
(1, embedding_size), name=name))
embeddings_reshaped = [
embedding_reshaper(embedding) for embedding in embeddings
]
# Merge all embeddings to one tensor
embeddings_merged = self._s_layer(
'embeddings_merge',
lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped)
# Use now one LSTM-layer to process all embeddings
lstm_units = embedding_size * 4
processed = self._s_layer(
'LSTM_proc_0',
lambda name: Bidirectional(LSTM(lstm_units, return_sequences=True),
name=name))(embeddings_merged)
# Split the tensor to seperate layers
embeddings_processed = [
self._s_layer('slice_{}'.format(i),
lambda name: slice_layer(processed, i, name))
for i in range(len(network_input))
]
# Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution.
# These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start
# with the cluster count / object combinations.
# First prepare some generally required layers
output_dense_size = embedding_size * 10
layers = [
self._s_layer('output_dense0',
lambda name: Dense(output_dense_size, name=name)),
self._s_layer('output_batch0',
lambda name: BatchNormalization(name=name)),
self._s_layer('output_relu0',
lambda name: Activation('relu', name=name))
]
cluster_softmax = {
k: self._s_layer(
'softmax_cluster_{}'.format(k),
lambda name: Dense(k, activation='softmax', name=name))
for k in cluster_counts
}
# Create now the outputs
clusters_output = additional_network_outputs['clusters'] = {}
for i in range(len(embeddings_processed)):
embedding_proc = embeddings_processed[i]
# Add the required layers
for layer in layers:
embedding_proc = layer(embedding_proc)
input_clusters_output = clusters_output['input{}'.format(i)] = {}
for k in cluster_counts:
# Create now the required softmax distributions
output_classifier = cluster_softmax[k](embedding_proc)
input_clusters_output['cluster{}'.format(
k)] = output_classifier
network_output.append(output_classifier)
# Calculate the real cluster count
cluster_count_dense_units = embedding_size * 8
cluster_count = self._s_layer(
'cluster_count_LSTM_merge',
lambda name: Bidirectional(LSTM(lstm_units), name=name)
(embeddings_merged))
cluster_count = self._s_layer(
'cluster_count_dense0',
lambda name: Dense(cluster_count_dense_units, name=name))(
cluster_count)
cluster_count = self._s_layer(
'cluster_count_batch0',
lambda name: BatchNormalization(name=name))(cluster_count)
cluster_count = self._s_layer(
'cluster_count_relu0',
lambda name: Activation('relu', name=name))(cluster_count)
# Use the forward pass dropout
self._add_debug_output(cluster_count,
'00_cluster_count_before_forward_pass_dropout')
if self._try_get_additional_build_config_value('forward_pass_dropout',
default_value=False):
cluster_count = ExtendedDropout(
0.5, train_phase_active=False,
test_phase_active=True)(cluster_count)
self._add_debug_output(cluster_count,
'01_cluster_count_after_forward_pass_dropout')
# The next layer is an output-layer, therefore the name must not be formatted
cluster_count = self._s_layer(
'cluster_count_output',
lambda name: Dense(
len(cluster_counts), activation='softmax', name=name),
format_name=False)(cluster_count)
self._add_debug_output(cluster_count, '02_cluster_count_result')
additional_network_outputs['cluster_count_output'] = cluster_count
network_output.append(cluster_count)
return True
def _build_network(self, network_input, network_output,
additional_network_outputs):
cluster_counts = list(self.data_provider.get_cluster_counts())
# The simple loss cluster NN requires a specific output: a list of softmax distributions
# First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each
# object etc. At the end, there is the cluster count output.
# Obsolete
def add_dbg_output(name, layer):
self._add_debug_output(layer, name)
# debug_output.append(Activation('linear', name=name)(layer))
# First we get an embedding for the network inputs
embeddings = self._get_embedding(network_input)
# Reshape all embeddings to 1d vectors
# embedding_shape = self._embedding_nn.model.layers[-1].output_shape
# embedding_size = np.prod(embedding_shape[1:])
embedding_shape = embeddings[0].shape
embedding_size = int(str(np.prod(embedding_shape[1:])))
embedding_reshaper = self._s_layer(
'embedding_reshape', lambda name: Reshape(
(1, embedding_size), name=name))
embeddings_reshaped = [
embedding_reshaper(embedding) for embedding in embeddings
]
# Merge all embeddings to one tensor
embeddings_merged = self._s_layer(
'embeddings_merge',
lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped)
# Use now some BDLSTM-layer to process all embeddings.
processed = embeddings_merged
for i in range(self.__lstm_layers):
processed = self._s_layer(
'LSTM_proc_{}'.format(i),
lambda name: Bidirectional(LSTM(self.__lstm_units,
return_sequences=True),
name=name))(processed)
processed = self._s_layer(
'LSTM_proc_{}_batch'.format(i),
lambda name: BatchNormalization(name=name))(processed)
# Batch normalize everything if not already done
add_dbg_output('EMBEDDINGS', processed)
if self.__lstm_layers == 0:
processed = BatchNormalization()(processed)
add_dbg_output('EMBEDDINGS_NORMALIZED', processed)
# Split the tensor to seperate layers
embeddings_processed = [
self._s_layer('slice_{}'.format(i),
lambda name: slice_layer(processed, i, name))
for i in range(len(network_input))
]
# Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution.
# These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start
# with the cluster count / object combinations.
# # First prepare some generally required layers
# layers = []
# for i in range(self.__output_dense_layers):
# layers += [
# self._s_layer('output_dense{}'.format(i), lambda name: Dense(self.__output_dense_units, name=name)),
# self._s_layer('output_batch'.format(i), lambda name: BatchNormalization(name=name)),
# self._s_layer('output_relu'.format(i), lambda name: Activation('relu', name=name))
# ]
# cluster_softmax = {
# k: self._s_layer('softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts
# }
#
# # Create now the outputs
# clusters_output = additional_network_outputs['clusters'] = {}
# for i in range(len(embeddings_processed)):
# embedding_proc = embeddings_processed[i]
#
# # Add the required layers
# for layer in layers:
# embedding_proc = layer(embedding_proc)
#
# input_clusters_output = clusters_output['input{}'.format(i)] = {}
# for k in cluster_counts:
#
# # Create now the required softmax distributions
# output_classifier = cluster_softmax[k](embedding_proc)
# input_clusters_output['cluster{}'.format(k)] = output_classifier
# network_output.append(output_classifier)
# Define a distance-function
d = lambda x, y: K.sqrt(K.sum(K.square(x - y))) # euclidean distance
def euclideanDistance(inputs, squared=False):
if (len(inputs) != 2):
raise 'oops'
# For better gradient flow: remove the sqrt
output = K.sum(K.square(inputs[0] - inputs[1]), axis=-1)
if not squared:
output = K.sqrt(output)
output = K.expand_dims(output, 1)
return output
# Apply k-means for each possible k
assert self.__kmeans_itrs > 0
cluster_vector_size = (self.__lstm_units *
2) if self.__lstm_layers > 0 else embedding_size
cluster_assignements = {}
for k in cluster_counts:
# Create initial cluster centers
# clusters = [self._s_layer(
# 'k_{}_init_{}'.format(k, i), lambda name: gaussian_random_layer((1, cluster_vector_size), name=name, only_execute_for_training=False)
# )(embeddings[0]) for i in range(k)]
# # Use the first n input points
clusters = embeddings_processed[:k]
for i in range(len(clusters)):
add_dbg_output('INIT_CLUSTER_{}'.format(i), clusters[i])
# Cluster-assignements
current_cluster_assignements = None
# Idee zum neuen Mittelwert finden:
# - Nicht mit Softmax rechnen sondern nur noch mit der Distanz
# - Clusterzentrum mit Punkten Gewichten, falls die Distanz zum Cluster vom Punkt minimal ist (also kein anderer Cluster näher ist)
# Do all iterations
for i in range(self.__kmeans_itrs):
def get_val_at(input, i_i):
def at(val, indices):
for index in indices:
val = val[index]
return val
return Lambda(lambda x: at(x, i_i),
output_shape=(1, ))(input)
# Recalculate the cluster centers (if required)
if i > 0:
clusters_old = clusters
clusters = []
for c_i in range(k):
c = 0
s = 0
for e_i in range(len(embeddings_processed)):
t = get_val_at(current_cluster_assignements[e_i],
[1, c_i])
c = Lambda(lambda x: c + x * t,
output_shape=(1, cluster_vector_size))(
embeddings_processed[e_i])
# c = Lambda(lambda x: c + x * 10. * K.relu(t + 0.1 - K.max(current_cluster_assignements[e_i])), output_shape=(1, cluster_vector_size))(embeddings_processed[e_i])
s = Lambda(lambda x: x + s, output_shape=(1, ))(t)
# t = current_cluster_assignements[e_i][c_i]
# c += t * embeddings_processed[e_i]
# s += t
c = Lambda(lambda x: x / s,
output_shape=(1, cluster_vector_size))(c)
# c = c / s
c = Lambda(lambda x: 0 + x,
output_shape=(1, cluster_vector_size))(c)
add_dbg_output("k{}_ITR{}_ci{}".format(k, i, c_i), c)
clusters.append(c)
# Recalculate the assigned cluster centers for each embedding
cluster_assignements[k] = []
current_cluster_assignements = cluster_assignements[k]
e_i = 0
for e in embeddings_processed:
# Calculate all distances to all cluster centers
k_distances = [
Lambda(lambda x: euclideanDistance(x, True),
output_shape=lambda x:
(x[0][0], 1))([e, cluster])
# merge([e, cluster], mode=euclideanDistance, output_shape=lambda x: (x[0][0], 1))
for cluster in clusters
# self._s_layer('kmeans_d', lambda name: Merge([]))
]
# Merge the distances and calculate a softmax
k_distances = Concatenate()(k_distances)
k_distances = Reshape((k, ))(k_distances)
add_dbg_output(
"k{}_ITR{}_e_i{}_DISTANCES_PLAIN".format(k, i, e_i),
k_distances)
# k_distances = Lambda(lambda x: 1 / (0.01 + x))(k_distances)
k_distances = Lambda(lambda x: -(1 + x)**2)(k_distances)
k_distances = Activation('softmax')(k_distances)
# # Dirty tune the softmax a bit
# k_distances = Lambda(lambda x: (x / K.max(x))**4)(k_distances)
# Save the new distances
current_cluster_assignements.append(k_distances)
add_dbg_output(
"k{}_ITR{}_e_i{}_DISTANCES".format(k, i, e_i),
k_distances)
e_i += 1
# Reshape all softmax layers
for k in cluster_counts:
cluster_assignements[k] = [
Reshape((1, k))(s) for s in cluster_assignements[k]
]
# Append all softmax layers to the network output
network_output += chain.from_iterable(
map(lambda k: cluster_assignements[k], cluster_counts))
# Create the additional cluster output
clusters_output = additional_network_outputs['clusters'] = {}
for i in range(self.input_count):
input_clusters_output = clusters_output['input{}'.format(i)] = {}
for k in cluster_counts:
input_clusters_output['cluster{}'.format(
k)] = cluster_assignements[k][i]
add_dbg_output('RESULT_INPUT{}_K{}'.format(i, k),
cluster_assignements[k][i])
# Calculate the real cluster count
cluster_count = self._s_layer(
'cluster_count_LSTM_merge',
lambda name: Bidirectional(LSTM(self.__lstm_units), name=name)
(embeddings_merged))
cluster_count = self._s_layer(
'cluster_count_LSTM_merge_batch',
lambda name: BatchNormalization(name=name))(cluster_count)
for i in range(self.__cluster_count_dense_layers):
cluster_count = self._s_layer(
'cluster_count_dense{}'.format(i),
lambda name: Dense(self.__cluster_count_dense_units, name=name
))(cluster_count)
cluster_count = self._s_layer(
'cluster_count_batch{}'.format(i),
lambda name: BatchNormalization(name=name))(cluster_count)
cluster_count = self._s_layer(
'cluster_count_relu{}'.format(i),
lambda name: Activation('relu', name=name))(cluster_count)
# The next layer is an output-layer, therefore the name must not be formatted
cluster_count = self._s_layer(
'cluster_count_output',
lambda name: Dense(
len(cluster_counts), activation='softmax', name=name),
format_name=False)(cluster_count)
additional_network_outputs['cluster_count_output'] = cluster_count
network_output.append(cluster_count)
return True
def _build_network(self, network_input, network_output, additional_network_outputs):
cluster_counts = list(self.data_provider.get_cluster_counts())
# The simple loss cluster NN requires a specific output: a list of softmax distributions
# First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each
# object etc. At the end, there is the cluster count output.
# First we get an embedding for the network inputs
embeddings = self._get_embedding(network_input)
# Reshape all embeddings to 1d vectors
# embedding_shape = self._embedding_nn.model.layers[-1].output_shape
# embedding_size = np.prod(embedding_shape[1:])
embedding_shape = embeddings[0].shape
embedding_size = int(str(np.prod(embedding_shape[1:])))
embedding_reshaper = self._s_layer('embedding_reshape', lambda name: Reshape((1, embedding_size), name=name))
embeddings_reshaped = [embedding_reshaper(embedding) for embedding in embeddings]
# Merge all embeddings to one tensor
embeddings_merged = self._s_layer('embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped)
self._add_additional_prediction_output(embeddings_merged, '0_Embeddings')
# The value range of the embeddings must be [-1, 1] to allow an efficient execution of the cluster evaluations,
# so be sure the embeddings are processed by tanh or some similar activation
# Use some LSTMs for some kind of preprocessing. After these layers a regularisation is applied
processed = embeddings_merged
for i in range(self.__pre_lstm_layers):
processed = self._s_layer(
'PRE_LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(self.__lstm_units, return_sequences=True), name=name)
)(processed)
processed = self._s_layer(
'PRE_LSTM_proc_{}_batch'.format(i), lambda name: BatchNormalization(name=name)
)(processed)
# Reshape the data now to a embeddings sized representation representation
processed = TimeDistributed(Dense(embedding_size, activation='tanh'))(processed)
# # Store these processed embeddings
# preprocessed_embeddings = [slice_layer(processed, i) for i in range(len(network_input))]
self._add_additional_prediction_output(processed, "1_LSTM_Preprocessed_Embeddings")
# Use now some LSTM-layer to process all embeddings
# processed = embeddings_merged
for i in range(self.__lstm_layers):
processed = self._s_layer(
'LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(self.__lstm_units, return_sequences=True), name=name)
)(processed)
processed = self._s_layer(
'LSTM_proc_{}_batch'.format(i), lambda name: BatchNormalization(name=name)
)(processed)
# Split the tensor to seperate layers
embeddings_processed = [self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input))]
# Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution.
# These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start
# with the cluster count / object combinations.
# First prepare some generally required layers
layers = []
for i in range(self.__output_dense_layers):
layers += [
self._s_layer('output_dense{}'.format(i), lambda name: Dense(self.__output_dense_units, name=name)),
self._s_layer('output_batch'.format(i), lambda name: BatchNormalization(name=name)),
# self._s_layer('output_relu'.format(i), lambda name: Activation('relu', name=name))
LeakyReLU()
]
cluster_softmax = {
k: self._s_layer('softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts
}
# Create now the outputs
clusters_output = additional_network_outputs['clusters'] = {}
cluster_classifiers = {k: [] for k in cluster_counts}
for i in range(len(embeddings_processed)):
embedding_proc = embeddings_processed[i]
# Add the required layers
for layer in layers:
embedding_proc = layer(embedding_proc)
input_clusters_output = clusters_output['input{}'.format(i)] = {}
for k in cluster_counts:
# Create now the required softmax distributions
output_classifier = cluster_softmax[k](embedding_proc)
input_clusters_output['cluster{}'.format(k)] = output_classifier
network_output.append(output_classifier)
cluster_classifiers[k].append(output_classifier)
# clustering_quality = 0
# sum_cohesion = 0
# sum_separation = 0
# alpha = 0.5
# beta = 0.25
# self._add_debug_output(Concatenate(axis=1)(preprocessed_embeddings), 'eval_embeddings')
# # Squared euclidean distance
# distance_f = lambda x, y: K.sum(K.square(x - y), axis=2)
# # Euclidean distance
# distance_f = lambda x, y: K.sqrt(K.sum(K.square(x - y), axis=2))
# for k in cluster_counts:
#
# # Create evaluation metrics
# self._add_debug_output(Concatenate(axis=1)(cluster_classifiers[k]), 'eval_classifications_k{}'.format(k))
# cluster_centers = get_cluster_centers(preprocessed_embeddings, cluster_classifiers[k])
# self._add_debug_output(Concatenate(axis=1)(cluster_centers), 'eval_cluster_centers_k{}'.format(k))
# cohesion = get_cluster_cohesion(cluster_centers, preprocessed_embeddings, cluster_classifiers[k])
# self._add_debug_output(cohesion, 'eval_cohesion_k{}'.format(k))
# separation = get_cluster_separation(cluster_centers, cluster_classifiers[k])
# self._add_debug_output(separation, 'eval_separation_k{}'.format(k))
#
# self._add_additional_prediction_output(
# Concatenate(axis=1, name='2_cluster_centers_k{}'.format(k))(cluster_centers),
# 'cluster_centers_k{}'.format(k)
# )
#
# sum_cohesion = Lambda(lambda cohesion: sum_cohesion + cohesion)(cohesion)
# sum_separation = Lambda(lambda separation: sum_separation + separation)(separation)
# # clustering_quality = Lambda(lambda cohesion:
# #
# # # Update the loss
# # clustering_quality + (alpha * cohesion - beta * separation)
# # )(cohesion)
# # Add alpha and beta to the cohesion and the separation
# sum_cohesion = Lambda(lambda sum_cohesion: alpha * sum_cohesion)(sum_cohesion)
# sum_separation = Lambda(lambda sum_separation: beta * sum_separation)(sum_separation)
#
# # Normalize the cohesion and the separation by the cluster_counts
# sum_cohesion = Lambda(lambda sum_cohesion: sum_cohesion / len(cluster_counts))(sum_cohesion)
# sum_separation = Lambda(lambda sum_separation: sum_separation / len(cluster_counts))(sum_separation)
#
# # Add the losses for the cohesion and the separation. Use for both the same loss function
# cluster_quality_loss = lambda x: lambda similiarty_loss, x=x: K.exp(- similiarty_loss * similiarty_loss * 4) * x
# self._register_additional_grouping_similarity_loss(
# 'cluster_cohesion',
# cluster_quality_loss(sum_cohesion)
# )
# self._register_additional_grouping_similarity_loss(
# 'cluster_separation',
# cluster_quality_loss(- K.log(1 + 2 * sum_separation))
# )
# # Normalize the cluster quality
# clustering_quality = Lambda(lambda x: x / len(cluster_counts))(clustering_quality)
#
# # What to do with the cluster quality? We use it for an additional loss, this loss should optimize
# # the cluster quality as soon as the clustering works relatively well.
# self._register_additional_grouping_similarity_loss(
# 'cluster_quality',
# lambda similiarty_loss: K.exp(- similiarty_loss * similiarty_loss) * clustering_quality
# )
# Calculate the real cluster count
cluster_count = self._s_layer('cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__lstm_units), name=name)(embeddings_merged))
cluster_count = self._s_layer('cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count)
for i in range(self.__cluster_count_dense_layers):
cluster_count = self._s_layer('cluster_count_dense{}'.format(i), lambda name: Dense(self.__cluster_count_dense_units, name=name))(cluster_count)
cluster_count = self._s_layer('cluster_count_batch{}'.format(i), lambda name: BatchNormalization(name=name))(cluster_count)
# cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation('relu', name=name))(cluster_count)
cluster_count = LeakyReLU()(cluster_count)
# The next layer is an output-layer, therefore the name must not be formatted
cluster_count = self._s_layer(
'cluster_count_output',
lambda name: Dense(len(cluster_counts), activation='softmax', name=name),
format_name=False
)(cluster_count)
additional_network_outputs['cluster_count_output'] = cluster_count
network_output.append(cluster_count)
return True
def _build_network(self, network_input, network_output,
additional_network_outputs):
cluster_counts = list(self.data_provider.get_cluster_counts())
# The simple loss cluster NN requires a specific output: a list of softmax distributions
# First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each
# object etc. At the end, there is the cluster count output.
# First we get an embedding for the network inputs
embeddings = self._get_embedding(network_input)
# Do some dropout
d_out = self._s_layer('embedding_dout',
lambda name: Dropout(0.5, name=name))
embeddings = [d_out(embedding) for embedding in embeddings]
# Reshape all embeddings to 1d vectors
# embedding_shape = self._embedding_nn.model.layers[-1].output_shape
# embedding_size = np.prod(embedding_shape[1:])
embedding_shape = embeddings[0].shape
embedding_size = int(str(np.prod(embedding_shape[1:])))
embedding_reshaper = self._s_layer(
'embedding_reshape', lambda name: Reshape(
(1, embedding_size), name=name))
embeddings_reshaped = [
embedding_reshaper(embedding) for embedding in embeddings
]
self._add_debug_output(concat(embeddings_reshaped, axis=1),
'embeddings_reshaped')
embeddings_reshaped = reweight_values(embeddings_reshaped,
self._get_clustering_hint())
self._add_debug_output(concat(embeddings_reshaped, axis=1),
'embeddings_processed_new')
# # Implement the KL-divergence on this layer. First, like lukic et al., do another fully connedted layer
# kl_embeddings = embeddings_reshaped
# kl_dense0 = self._s_layer('kl_dense0', lambda name: Dense(self.__kl_embedding_size, name=name, activation='relu'))
# kl_embeddings = [kl_dense0(kl_embedding) for kl_embedding in kl_embeddings]
# kl_softmax = self._s_layer('kl_softmax', lambda name: Dense(self.__kl_embedding_size, name=name, activation='softmax'))
# kl_embeddings = [kl_softmax(kl_embedding) for kl_embedding in kl_embeddings]
# self._register_additional_embedding_comparison_regularisation(
# 'KL_divergence',
# lukic_kl_divergence,
# kl_embeddings,
# weight=self.__kl_divergence_factor
# )
# We need now the internal representation of the embeddings. This means we have to resize them.
internal_embedding_size = self.__internal_embedding_size // 2 * 2
embedding_internal_resizer = self._s_layer(
'internal_embedding_resize',
lambda name: Dense(internal_embedding_size, name=name))
embeddings_reshaped = [
embedding_internal_resizer(embedding)
for embedding in embeddings_reshaped
]
embedding_internal_resizer_act = LeakyReLU()
embeddings_reshaped = [
embedding_internal_resizer_act(embedding)
for embedding in embeddings_reshaped
]
# Merge all embeddings to one tensor
embeddings_merged = self._s_layer(
'embeddings_merge',
lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped)
# Use now some lstm-layers
processed = embeddings_merged
for i in range(self.__lstm_layers):
tmp = self._s_layer(
'LSTM_proc_{}'.format(i),
lambda name: Bidirectional(LSTM(internal_embedding_size // 2,
return_sequences=True),
name=name))(processed)
processed = Add()([processed, tmp])
# Split the tensor to seperate layers
embeddings_processed = [
self._s_layer('slice_{}'.format(i),
lambda name: slice_layer(processed, i, name))
for i in range(len(network_input))
]
# Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution.
# These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start
# with the cluster count / object combinations.
# First prepare some generally required layers
layers = []
for i in range(self.__output_dense_layers):
layers += [
self._s_layer(
'output_dense{}'.format(i),
lambda name: Dense(self.__output_dense_units, name=name)),
self._s_layer('output_batch'.format(i),
lambda name: BatchNormalization(name=name)),
LeakyReLU(),
Dropout(0.5)
# self._s_layer('output_relu'.format(i), lambda name: Activation(LeakyReLU(), name=name))
]
cluster_softmax = {
k: self._s_layer(
'softmax_cluster_{}'.format(k),
lambda name: Dense(k, activation='softmax', name=name))
for k in cluster_counts
}
# Create now the outputs
clusters_output = additional_network_outputs['clusters'] = {}
for i in range(len(embeddings_processed)):
embedding_proc = embeddings_processed[i]
# Add the required layers
for layer in layers:
embedding_proc = layer(embedding_proc)
input_clusters_output = clusters_output['input{}'.format(i)] = {}
for k in cluster_counts:
# Create now the required softmax distributions
output_classifier = cluster_softmax[k](embedding_proc)
input_clusters_output['cluster{}'.format(
k)] = output_classifier
network_output.append(output_classifier)
# Calculate the real cluster count
assert self.__cluster_count_lstm_layers >= 1
cluster_count = embeddings_merged
for i in range(self.__cluster_count_lstm_layers - 1):
cluster_count = self._s_layer(
'cluster_count_LSTM{}'.format(i),
lambda name: Bidirectional(LSTM(
self.__cluster_count_lstm_units, return_sequences=True),
name=name)(cluster_count))
cluster_count = self._s_layer(
'cluster_count_LSTM{}_batch'.format(i),
lambda name: BatchNormalization(name=name))(cluster_count)
cluster_count = self._s_layer(
'cluster_count_LSTM_merge',
lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units),
name=name)(cluster_count))
cluster_count = self._s_layer(
'cluster_count_LSTM_merge_batch',
lambda name: BatchNormalization(name=name))(cluster_count)
for i in range(self.__cluster_count_dense_layers):
cluster_count = self._s_layer(
'cluster_count_dense{}'.format(i),
lambda name: Dense(self.__cluster_count_dense_units, name=name
))(cluster_count)
cluster_count = self._s_layer(
'cluster_count_batch{}'.format(i),
lambda name: BatchNormalization(name=name))(cluster_count)
cluster_count = LeakyReLU()(cluster_count)
# cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation(LeakyReLU(), name=name))(cluster_count)
# The next layer is an output-layer, therefore the name must not be formatted
cluster_count = Dropout(0.5)(cluster_count)
cluster_count = self._s_layer(
'cluster_count_output',
lambda name: Dense(
len(cluster_counts), activation='softmax', name=name),
format_name=False)(cluster_count)
additional_network_outputs['cluster_count_output'] = cluster_count
network_output.append(cluster_count)
return True
