def simple_center_loss(y_true, y_pred): # y_true has no fixed shape, but we require that the second dimension size is already known. Reshape it. n = self.input_count if self.include_self_comparison: y_len = n * (n + 1) // 2 else: y_len = n * (n - 1) // 2 y_true = Lambda(lambda x: y_true)(center_loss_vectors[0]) y_true = Reshape((y_len, ))(y_true) # # Dummy values # y_true = Lambda(lambda y_true: 0. * y_true)(y_true) # Calculate the centers for each vector and then the loss (just MSE) centers = reweight_values(center_loss_vectors, y_true) return mean_squared_error(concat(centers, axis=1), concat(center_loss_vectors, axis=1))
def __build_comparisons_arrays(self, embeddings, comparator_f, weight): assert self.input_count == len(embeddings) cmp_eq = [] # All comparisons assuming the elements do equal cmp_ne = [] # All comparisons assuming the elements do not equal comparisons = 0 for i_source in range(self.input_count): e_source = embeddings[i_source] # Should i_source be compared with itself? if self._include_self_comparison: target_range = range(i_source, self.input_count) else: target_range = range(i_source + 1, self.input_count) for i_target in target_range: e_target = embeddings[i_target] cmp_eq.append(comparator_f(e_source, e_target, 1.)) cmp_ne.append(comparator_f(e_source, e_target, 0.)) comparisons += 1 # Create two long vectors for cmp_eq and cmp_ne cmp_eq = concat(cmp_eq, axis=1) cmp_ne = concat(cmp_ne, axis=1) # # Reshape and merge them # cmp_eq = Reshape((1, comparisons))(cmp_eq) # cmp_ne = Reshape((1, comparisons))(cmp_ne) # # # Concat both # cmp = concat([cmp_eq, cmp_ne], axis=1) cmp = concat([cmp_eq, cmp_ne], axis=1) # Weight the result cmp = cmp * weight # If this is not already the case: Convert the 'cmp' obj to a keras tensor (this command is a bit "dummy") cmp = Lambda(lambda x: cmp)(embeddings[0]) return cmp
def _build_network(self, network_input, network_output, additional_network_outputs): cluster_counts = list(self.data_provider.get_cluster_counts()) # The simple loss cluster NN requires a specific output: a list of softmax distributions # First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each # object etc. At the end, there is the cluster count output. # First we get an embedding for the network inputs embeddings = self._get_embedding(network_input) # Reshape all embeddings to 1d vectors # embedding_shape = self._embedding_nn.model.layers[-1].output_shape # embedding_size = np.prod(embedding_shape[1:]) embedding_shape = embeddings[0].shape embedding_size = int(str(np.prod(embedding_shape[1:]))) embedding_reshaper = self._s_layer('embedding_reshape', lambda name: Reshape((1, embedding_size), name=name)) embeddings_reshaped = [embedding_reshaper(embedding) for embedding in embeddings] cohesion_d_layer = self._s_layer('cohesion_d_layer', lambda name: Dense(2, activation='tanh', name=name)) cohesion_embeddings = [cohesion_d_layer(embedding) for embedding in embeddings_reshaped] self._add_additional_prediction_output( concat(cohesion_embeddings, axis=1), 'cohesion_embeddings' ) self._register_additional_embedding_comparison_regularisation( 'cluster_cohesion', # If the values are no longer used (as this is the case here), we need a dissimilarity weight, because # otherwise there is the trivial solution to map all inputs to 0 lambda x, y, similar: meier_cluster_cohesion(x, y, similar, dissimilar_distance_weight=-1), cohesion_embeddings ) # We need now the internal representation of the embeddings. This means we have to resize them. internal_embedding_size = self.__internal_embedding_size // 2 * 2 embedding_internal_resizer = self._s_layer('internal_embedding_resize', lambda name: Dense(internal_embedding_size, name=name)) embeddings_reshaped = [embedding_internal_resizer(embedding) for embedding in embeddings_reshaped] embedding_internal_resizer_act = LeakyReLU() embeddings_reshaped = [embedding_internal_resizer_act(embedding) for embedding in embeddings_reshaped] # Merge all embeddings to one tensor embeddings_merged = self._s_layer('embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped) # Use now some lstm-layers processed = embeddings_merged for i in range(self.__lstm_layers): tmp = self._s_layer( 'LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(internal_embedding_size // 2, return_sequences=True), name=name) )(processed) processed = Add()([processed, tmp]) # Split the tensor to seperate layers embeddings_processed = [self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input))] # Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution. # These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start # with the cluster count / object combinations. # First prepare some generally required layers layers = [] for i in range(self.__output_dense_layers): layers += [ self._s_layer('output_dense{}'.format(i), lambda name: Dense(self.__output_dense_units, name=name)), self._s_layer('output_batch'.format(i), lambda name: BatchNormalization(name=name)), LeakyReLU() # self._s_layer('output_relu'.format(i), lambda name: Activation(LeakyReLU(), name=name)) ] cluster_softmax = { k: self._s_layer('softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts } # Create now the outputs clusters_output = additional_network_outputs['clusters'] = {} for i in range(len(embeddings_processed)): embedding_proc = embeddings_processed[i] # Add the required layers for layer in layers: embedding_proc = layer(embedding_proc) input_clusters_output = clusters_output['input{}'.format(i)] = {} for k in cluster_counts: # Create now the required softmax distributions output_classifier = cluster_softmax[k](embedding_proc) input_clusters_output['cluster{}'.format(k)] = output_classifier network_output.append(output_classifier) # Calculate the real cluster count assert self.__cluster_count_lstm_layers >= 1 cluster_count = embeddings_merged for i in range(self.__cluster_count_lstm_layers - 1): cluster_count = self._s_layer('cluster_count_LSTM{}'.format(i), lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units, return_sequences=True), name=name)(cluster_count)) cluster_count = self._s_layer('cluster_count_LSTM{}_batch'.format(i), lambda name: BatchNormalization(name=name))(cluster_count) cluster_count = self._s_layer('cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units), name=name)(cluster_count)) cluster_count = self._s_layer('cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count) for i in range(self.__cluster_count_dense_layers): cluster_count = self._s_layer('cluster_count_dense{}'.format(i), lambda name: Dense(self.__cluster_count_dense_units, name=name))(cluster_count) cluster_count = self._s_layer('cluster_count_batch{}'.format(i), lambda name: BatchNormalization(name=name))(cluster_count) cluster_count = LeakyReLU()(cluster_count) # cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation(LeakyReLU(), name=name))(cluster_count) # The next layer is an output-layer, therefore the name must not be formatted cluster_count = self._s_layer( 'cluster_count_output', lambda name: Dense(len(cluster_counts), activation='softmax', name=name), format_name=False )(cluster_count) additional_network_outputs['cluster_count_output'] = cluster_count network_output.append(cluster_count) return True
def _build_network(self, network_input, network_output, additional_network_outputs): cluster_counts = self._get_cluster_counts() # The simple loss cluster NN requires a specific output: a list of softmax distributions # First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each # object etc. At the end, there is the cluster count output. # First we get an embedding for the network inputs embeddings = self._get_embedding(network_input) # Reshape all embeddings to 1d vectors # embedding_shape = self._embedding_nn.model.layers[-1].output_shape # embedding_size = np.prod(embedding_shape[1:]) embedding_shape = embeddings[0].shape embedding_size = int(str(np.prod(embedding_shape[1:]))) embedding_reshaper = self._s_layer( 'embedding_reshape', lambda name: Reshape( (1, embedding_size), name=name)) embeddings_reshaped = [ embedding_reshaper(embedding) for embedding in embeddings ] # Merge all embeddings to one tensor embeddings_merged = self._s_layer( 'embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped) # self._add_additional_prediction_output(embeddings_merged, 'Embeddings') # Use now some LSTM-layer to process all embeddings processed = embeddings_merged # Split the tensor to seperate layers embeddings_processed = [ self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input)) ] # Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution. # These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start # with the cluster count / object combinations. # First prepare some generally required layers layers = [] cluster_softmax = { k: self._s_layer( 'softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts } # Create now the outputs clusters_output = additional_network_outputs['clusters'] = {} for i in range(len(embeddings_processed)): embedding_proc = embeddings_processed[i] # Add the required layers for layer in layers: embedding_proc = layer(embedding_proc) input_clusters_output = clusters_output['input{}'.format(i)] = {} for k in cluster_counts: # Create now the required softmax distributions output_classifier = cluster_softmax[k](embedding_proc) input_clusters_output['cluster{}'.format( k)] = output_classifier network_output.append(output_classifier) # Calculate the real cluster count cluster_count = embeddings_merged cluster_count = self._s_layer( 'cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units), name=name)(cluster_count)) cluster_count = self._s_layer( 'cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count) # The next layer is an output-layer, therefore the name must not be formatted cluster_count = self._s_layer( 'cluster_count_output', lambda name: Dense( len(cluster_counts), activation='softmax', name=name), format_name=False)(cluster_count) additional_network_outputs['cluster_count_output'] = cluster_count # Add a regularizer that is based on the "Deep Divergence-Based CLustering" paper # We require these inputs: # 1) A list of embeddings # 2) A list of softmaxs # We have for each cluster count possibility a list of softmaxs and we calculate the regularization for all # possibilities, then they are weighted by their cluster probability. x_inputs = embeddings # ddbc_losses = [] d_a_losses = [] d_m_losses = [] triuAAt_losses = [] for k in cluster_counts: s_inputs = list( map( lambda i: Reshape((k, )) (additional_network_outputs['clusters']['input{}'.format( i)]['cluster{}'.format(k)]), range(len(embeddings)))) # ddbc_losses.append(get_ddbc_loss_function(x_inputs, s_inputs)[0]) _, d_a, triuAAt, d_m = get_ddbc_loss_function(x_inputs, s_inputs) d_a_losses.append(d_a) d_m_losses.append(d_m) triuAAt_losses.append(triuAAt) # ddbc_loss = concat_layer(input_count=len(ddbc_losses), axis=1)(ddbc_losses) # Concatenate(axis=1)(ddbc_losses) # self._add_debug_output(ddbc_loss, "ddbc_loss") # self._add_debug_output(cluster_count, "cluster_count") # ddbc_loss = Dot(axes=1)([ddbc_loss, cluster_count]) # self._add_debug_output(ddbc_loss, "ddbc_loss2") # self._register_additional_regularisation(ddbc_loss, "Ddbc_Loss") d_a_losses = concat(inputs=d_a_losses, axis=1) d_m_losses = concat(inputs=d_m_losses, axis=1) triuAAt_losses = concat(inputs=triuAAt_losses, axis=1) self._add_debug_output(d_a_losses, "d_a_losses") self._add_debug_output(d_m_losses, "d_m_losses") self._add_debug_output(triuAAt_losses, "triuAAt_losses") self._add_debug_output(cluster_count, "cluster_count") d_a_losses = Dot(axes=1)([d_a_losses, cluster_count]) d_m_losses = Dot(axes=1)([d_m_losses, cluster_count]) triuAAt_losses = Dot(axes=1)([triuAAt_losses, cluster_count]) self._add_debug_output(d_a_losses, "d_a_losses2") self._add_debug_output(d_m_losses, "d_m_losses2") self._add_debug_output(triuAAt_losses, "triuAAt_losses2") self._register_additional_regularisation(d_a_losses, "d_a_loss") self._register_additional_regularisation(d_m_losses, "d_m_loss") self._register_additional_regularisation(triuAAt_losses, "triuAAt_losses") network_output.append(cluster_count) return True
def py_matrix_to_keras_matrix(M): return concat(axis=1, inputs=list(map( lambda x: concat(axis=2, inputs=x), M )))
def _build_network(self, network_input, network_output, additional_network_outputs): cluster_counts = list(self.data_provider.get_cluster_counts()) # The simple loss cluster NN requires a specific output: a list of softmax distributions # First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each # object etc. At the end, there is the cluster count output. # First we get an embedding for the network inputs embeddings = self._get_embedding(network_input) # Do some dropout d_out = self._s_layer('embedding_dout', lambda name: Dropout(0.5, name=name)) embeddings = [d_out(embedding) for embedding in embeddings] # Reshape all embeddings to 1d vectors # embedding_shape = self._embedding_nn.model.layers[-1].output_shape # embedding_size = np.prod(embedding_shape[1:]) embedding_shape = embeddings[0].shape embedding_size = int(str(np.prod(embedding_shape[1:]))) embedding_reshaper = self._s_layer( 'embedding_reshape', lambda name: Reshape( (1, embedding_size), name=name)) embeddings_reshaped = [ embedding_reshaper(embedding) for embedding in embeddings ] self._add_debug_output(concat(embeddings_reshaped, axis=1), 'embeddings_reshaped') embeddings_reshaped = reweight_values(embeddings_reshaped, self._get_clustering_hint()) self._add_debug_output(concat(embeddings_reshaped, axis=1), 'embeddings_processed_new') # # Implement the KL-divergence on this layer. First, like lukic et al., do another fully connedted layer # kl_embeddings = embeddings_reshaped # kl_dense0 = self._s_layer('kl_dense0', lambda name: Dense(self.__kl_embedding_size, name=name, activation='relu')) # kl_embeddings = [kl_dense0(kl_embedding) for kl_embedding in kl_embeddings] # kl_softmax = self._s_layer('kl_softmax', lambda name: Dense(self.__kl_embedding_size, name=name, activation='softmax')) # kl_embeddings = [kl_softmax(kl_embedding) for kl_embedding in kl_embeddings] # self._register_additional_embedding_comparison_regularisation( # 'KL_divergence', # lukic_kl_divergence, # kl_embeddings, # weight=self.__kl_divergence_factor # ) # We need now the internal representation of the embeddings. This means we have to resize them. internal_embedding_size = self.__internal_embedding_size // 2 * 2 embedding_internal_resizer = self._s_layer( 'internal_embedding_resize', lambda name: Dense(internal_embedding_size, name=name)) embeddings_reshaped = [ embedding_internal_resizer(embedding) for embedding in embeddings_reshaped ] embedding_internal_resizer_act = LeakyReLU() embeddings_reshaped = [ embedding_internal_resizer_act(embedding) for embedding in embeddings_reshaped ] # Merge all embeddings to one tensor embeddings_merged = self._s_layer( 'embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped) # Use now some lstm-layers processed = embeddings_merged for i in range(self.__lstm_layers): tmp = self._s_layer( 'LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(internal_embedding_size // 2, return_sequences=True), name=name))(processed) processed = Add()([processed, tmp]) # Split the tensor to seperate layers embeddings_processed = [ self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input)) ] # Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution. # These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start # with the cluster count / object combinations. # First prepare some generally required layers layers = [] for i in range(self.__output_dense_layers): layers += [ self._s_layer( 'output_dense{}'.format(i), lambda name: Dense(self.__output_dense_units, name=name)), self._s_layer('output_batch'.format(i), lambda name: BatchNormalization(name=name)), LeakyReLU(), Dropout(0.5) # self._s_layer('output_relu'.format(i), lambda name: Activation(LeakyReLU(), name=name)) ] cluster_softmax = { k: self._s_layer( 'softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts } # Create now the outputs clusters_output = additional_network_outputs['clusters'] = {} for i in range(len(embeddings_processed)): embedding_proc = embeddings_processed[i] # Add the required layers for layer in layers: embedding_proc = layer(embedding_proc) input_clusters_output = clusters_output['input{}'.format(i)] = {} for k in cluster_counts: # Create now the required softmax distributions output_classifier = cluster_softmax[k](embedding_proc) input_clusters_output['cluster{}'.format( k)] = output_classifier network_output.append(output_classifier) # Calculate the real cluster count assert self.__cluster_count_lstm_layers >= 1 cluster_count = embeddings_merged for i in range(self.__cluster_count_lstm_layers - 1): cluster_count = self._s_layer( 'cluster_count_LSTM{}'.format(i), lambda name: Bidirectional(LSTM( self.__cluster_count_lstm_units, return_sequences=True), name=name)(cluster_count)) cluster_count = self._s_layer( 'cluster_count_LSTM{}_batch'.format(i), lambda name: BatchNormalization(name=name))(cluster_count) cluster_count = self._s_layer( 'cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units), name=name)(cluster_count)) cluster_count = self._s_layer( 'cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count) for i in range(self.__cluster_count_dense_layers): cluster_count = self._s_layer( 'cluster_count_dense{}'.format(i), lambda name: Dense(self.__cluster_count_dense_units, name=name ))(cluster_count) cluster_count = self._s_layer( 'cluster_count_batch{}'.format(i), lambda name: BatchNormalization(name=name))(cluster_count) cluster_count = LeakyReLU()(cluster_count) # cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation(LeakyReLU(), name=name))(cluster_count) # The next layer is an output-layer, therefore the name must not be formatted cluster_count = Dropout(0.5)(cluster_count) cluster_count = self._s_layer( 'cluster_count_output', lambda name: Dense( len(cluster_counts), activation='softmax', name=name), format_name=False)(cluster_count) additional_network_outputs['cluster_count_output'] = cluster_count network_output.append(cluster_count) return True