def _build_network(self, network_input, network_output, additional_network_outputs): cluster_counts = list(self.data_provider.get_cluster_counts()) # The simple loss cluster NN requires a specific output: a list of softmax distributions # First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each # object etc. At the end, there is the cluster count output. # First we get an embedding for the network inputs embeddings = self._get_embedding(network_input) # Reshape all embeddings to 1d vectors # embedding_shape = self._embedding_nn.model.layers[-1].output_shape # embedding_size = np.prod(embedding_shape[1:]) embedding_shape = embeddings[0].shape embedding_size = int(str(np.prod(embedding_shape[1:]))) embedding_reshaper = self._s_layer( 'embedding_reshape', lambda name: Reshape( (1, embedding_size), name=name)) embeddings_reshaped = [ embedding_reshaper(embedding) for embedding in embeddings ] # The KL-divergence only makes sense if an embedding is used (otherwise no embedding can be optimized) if self._uses_embedding_layer() and self.__kl_divergence_factor > 0.: # Implement the KL-divergence on this layer. First, like lukic et al., do another fully connedted layer kl_embeddings = embeddings_reshaped kl_dense0 = self._s_layer( 'kl_dense0', lambda name: LeakyReLU(self.__kl_embedding_size, name=name)) # kl_dense0 = self._s_layer('kl_dense0', lambda name: Activation(self.__kl_embedding_size, name=name, activation='relu')) kl_embeddings = [ kl_dense0(kl_embedding) for kl_embedding in kl_embeddings ] kl_softmax = self._s_layer( 'kl_softmax', lambda name: Dense( self.__kl_embedding_size, name=name, activation='softmax')) kl_embeddings = [ kl_softmax(kl_embedding) for kl_embedding in kl_embeddings ] self._register_additional_embedding_comparison_regularisation( 'KL_divergence', lukic_kl_divergence, kl_embeddings, weight=self.__kl_divergence_factor) # We need now the internal representation of the embeddings. This means we have to resize them. internal_embedding_size = self.__internal_embedding_size // 2 * 2 embedding_internal_resizer = self._s_layer( 'internal_embedding_resize', lambda name: Dense(internal_embedding_size, name=name)) embeddings_reshaped = [ embedding_internal_resizer(embedding) for embedding in embeddings_reshaped ] embedding_internal_resizer_act = LeakyReLU() embeddings_reshaped = [ embedding_internal_resizer_act(embedding) for embedding in embeddings_reshaped ] # Merge all embeddings to one tensor embeddings_merged = self._s_layer( 'embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped) # Use now some lstm-layers processed = embeddings_merged second_last_processed = None for i in range(self.__lstm_layers): second_last_processed = processed tmp = self._s_layer( 'LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(internal_embedding_size // 2, return_sequences=True), name=name))(processed) processed = Add()([processed, tmp]) # Split the tensor to seperate layers embeddings_processed = [ self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input)) ] # Implement the simplified center loss center_loss_vectors = embeddings_processed def simple_center_loss(y_true, y_pred): # y_true has no fixed shape, but we require that the second dimension size is already known. Reshape it. n = self.input_count if self.include_self_comparison: y_len = n * (n + 1) // 2 else: y_len = n * (n - 1) // 2 y_true = Lambda(lambda x: y_true)(center_loss_vectors[0]) y_true = Reshape((y_len, ))(y_true) # # Dummy values # y_true = Lambda(lambda y_true: 0. * y_true)(y_true) # Calculate the centers for each vector and then the loss (just MSE) centers = reweight_values(center_loss_vectors, y_true) return mean_squared_error(concat(centers, axis=1), concat(center_loss_vectors, axis=1)) if self.__simplified_center_loss_factor > 0.: self._register_additional_grouping_similarity_loss( 'simple_center_loss', simple_center_loss, False, weight=self.__simplified_center_loss_factor) # Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution. # These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start # with the cluster count / object combinations. # First prepare some generally required layers layers = [] for i in range(self.__output_dense_layers): layers += [ self._s_layer( 'output_dense{}'.format(i), lambda name: Dense(self.__output_dense_units, name=name)), self._s_layer('output_batch'.format(i), lambda name: BatchNormalization(name=name)), LeakyReLU(), self._s_layer('output_relu'.format(i), lambda name: Activation(LeakyReLU(), name=name)), Dropout(0.5) ] # Add forward pass dropout if self._try_get_additional_build_config_value( 'forward_pass_dropout', default_value=False): layers.append( ExtendedDropout(0.5, train_phase_active=False, test_phase_active=True)) cluster_softmax = { k: self._s_layer( 'softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts } # Create now the outputs clusters_output = additional_network_outputs['clusters'] = {} for i in range(len(embeddings_processed)): embedding_proc = embeddings_processed[i] # Add the required layers for layer in layers: embedding_proc = layer(embedding_proc) input_clusters_output = clusters_output['input{}'.format(i)] = {} for k in cluster_counts: # Create now the required softmax distributions output_classifier = cluster_softmax[k](embedding_proc) input_clusters_output['cluster{}'.format( k)] = output_classifier network_output.append(output_classifier) # Calculate the real cluster count assert self.__cluster_count_lstm_layers >= 1 cluster_count = second_last_processed for i in range(self.__cluster_count_lstm_layers - 1): cluster_count = self._s_layer( 'cluster_count_LSTM{}'.format(i), lambda name: Bidirectional(LSTM( self.__cluster_count_lstm_units, return_sequences=True), name=name)(cluster_count)) cluster_count = self._s_layer( 'cluster_count_LSTM{}_batch'.format(i), lambda name: BatchNormalization(name=name))(cluster_count) cluster_count = self._s_layer( 'cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units), name=name)(cluster_count)) cluster_count = self._s_layer( 'cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count) for i in range(self.__cluster_count_dense_layers): cluster_count = self._s_layer( 'cluster_count_dense{}'.format(i), lambda name: Dense(self.__cluster_count_dense_units, name=name ))(cluster_count) cluster_count = LeakyReLU()(cluster_count) cluster_count = self._s_layer( 'cluster_count_batch{}'.format(i), lambda name: BatchNormalization(name=name))(cluster_count) cluster_count = Dropout(0.5)(cluster_count) # Add forward pass dropout if self._try_get_additional_build_config_value( 'forward_pass_dropout', default_value=False): cluster_count = ExtendedDropout( 0.5, train_phase_active=False, test_phase_active=True)(cluster_count) # cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation(LeakyReLU(), name=name))(cluster_count) # The next layer is an output-layer, therefore the name must not be formatted cluster_count = self._s_layer( 'cluster_count_output', lambda name: Dense( len(cluster_counts), activation='softmax', name=name), format_name=False)(cluster_count) additional_network_outputs['cluster_count_output'] = cluster_count network_output.append(cluster_count) return True
def _build_network(self, network_input, network_output, additional_network_outputs): cluster_counts = list(self.data_provider.get_cluster_counts()) # The simple loss cluster NN requires a specific output: a list of softmax distributions # First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each # object etc. At the end, there is the cluster count output. # First we get an embedding for the network inputs embeddings = self._get_embedding(network_input) # Reshape all embeddings to 1d vectors # embedding_shape = self._embedding_nn.model.layers[-1].output_shape # embedding_size = np.prod(embedding_shape[1:]) embedding_shape = embeddings[0].shape embedding_size = int(str(np.prod(embedding_shape[1:]))) embedding_reshaper = self._s_layer( 'embedding_reshape', lambda name: Reshape( (1, embedding_size), name=name)) embeddings_reshaped = [ embedding_reshaper(embedding) for embedding in embeddings ] # Merge all embeddings to one tensor embeddings_merged = self._s_layer( 'embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped) self._add_additional_prediction_output(embeddings_merged, 'Embeddings') # Use now some LSTM-layer to process all embeddings processed = embeddings_merged for i in range(self.__lstm_layers): processed = self._s_layer( 'LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(self.__lstm_units, return_sequences=True), name=name))(processed) processed = self._s_layer( 'LSTM_proc_{}_batch'.format(i), lambda name: BatchNormalization(name=name))(processed) # Split the tensor to seperate layers embeddings_processed = [ self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input)) ] # Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution. # These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start # with the cluster count / object combinations. # First prepare some generally required layers layers = [] for i in range(self.__output_dense_layers): layers += [ self._s_layer( 'output_dense{}'.format(i), lambda name: Dense(self.__output_dense_units, name=name)), self._s_layer('output_batch'.format(i), lambda name: BatchNormalization(name=name)), LeakyReLU() # self._s_layer('output_relu'.format(i), lambda name: Activation(LeakyReLU(), name=name)) ] cluster_softmax = { k: self._s_layer( 'softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts } # Create now the outputs clusters_output = additional_network_outputs['clusters'] = {} for i in range(len(embeddings_processed)): embedding_proc = embeddings_processed[i] # Add the required layers for layer in layers: embedding_proc = layer(embedding_proc) input_clusters_output = clusters_output['input{}'.format(i)] = {} for k in cluster_counts: # Create now the required softmax distributions output_classifier = cluster_softmax[k](embedding_proc) input_clusters_output['cluster{}'.format( k)] = output_classifier network_output.append(output_classifier) # Calculate the real cluster count assert self.__cluster_count_lstm_layers >= 1 for i in range(self.__cluster_count_lstm_layers - 1): cluster_count = self._s_layer( 'cluster_count_LSTM{}'.format(i), lambda name: Bidirectional(LSTM( self.__cluster_count_lstm_units, return_sequences=True), name=name)(embeddings_merged)) cluster_count = self._s_layer( 'cluster_count_LSTM{}_batch'.format(i), lambda name: BatchNormalization(name=name))(cluster_count) cluster_count = self._s_layer( 'cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units), name=name)(cluster_count)) cluster_count = self._s_layer( 'cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count) for i in range(self.__cluster_count_dense_layers): cluster_count = self._s_layer( 'cluster_count_dense{}'.format(i), lambda name: Dense(self.__cluster_count_dense_units, name=name ))(cluster_count) cluster_count = self._s_layer( 'cluster_count_batch{}'.format(i), lambda name: BatchNormalization(name=name))(cluster_count) cluster_count = LeakyReLU()(cluster_count) # cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation(LeakyReLU(), name=name))(cluster_count) # The next layer is an output-layer, therefore the name must not be formatted cluster_count = self._s_layer( 'cluster_count_output', lambda name: Dense( len(cluster_counts), activation='softmax', name=name), format_name=False)(cluster_count) additional_network_outputs['cluster_count_output'] = cluster_count network_output.append(cluster_count) return True
def _build_network(self, network_input, network_output, additional_network_outputs): cluster_counts = list(self.data_provider.get_cluster_counts()) # The simple loss cluster NN requires a specific output: a list of softmax distributions # First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each # object etc. At the end, there is the cluster count output. # First we get an embedding for the network inputs embeddings = self._get_embedding(network_input) # Reshape all embeddings to 1d vectors # embedding_shape = self._embedding_nn.model.layers[-1].output_shape # embedding_size = np.prod(embedding_shape[1:]) embedding_shape = embeddings[0].shape embedding_size = int(str(np.prod(embedding_shape[1:]))) embedding_reshaper = self._s_layer('embedding_reshape', lambda name: Reshape((1, embedding_size), name=name)) embeddings_reshaped = [embedding_reshaper(embedding) for embedding in embeddings] # Merge all embeddings to one tensor embeddings_merged = self._s_layer('embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped) self._add_additional_prediction_output(embeddings_merged, 'Embeddings') # The value range of the embeddings must be [-1, 1] to allow an efficient execution of the cluster evaluations, # so be sure the embeddings are processed by tanh or some similar activation # Use now some LSTM-layer to process all embeddings processed = embeddings_merged for i in range(self.__lstm_layers): processed = self._s_layer( 'LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(self.__lstm_units, return_sequences=True), name=name) )(processed) processed = self._s_layer( 'LSTM_proc_{}_batch'.format(i), lambda name: BatchNormalization(name=name) )(processed) # Split the tensor to seperate layers embeddings_processed = [self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input))] # Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution. # These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start # with the cluster count / object combinations. # First prepare some generally required layers layers = [] for i in range(self.__output_dense_layers): layers += [ self._s_layer('output_dense{}'.format(i), lambda name: Dense(self.__output_dense_units, name=name)), self._s_layer('output_batch'.format(i), lambda name: BatchNormalization(name=name)), self._s_layer('output_relu'.format(i), lambda name: Activation('relu', name=name)) ] cluster_softmax = { k: self._s_layer('softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts } # Create now the outputs clusters_output = additional_network_outputs['clusters'] = {} cluster_classifiers = {k: [] for k in cluster_counts} for i in range(len(embeddings_processed)): embedding_proc = embeddings_processed[i] # Add the required layers for layer in layers: embedding_proc = layer(embedding_proc) input_clusters_output = clusters_output['input{}'.format(i)] = {} for k in cluster_counts: # Create now the required softmax distributions output_classifier = cluster_softmax[k](embedding_proc) input_clusters_output['cluster{}'.format(k)] = output_classifier network_output.append(output_classifier) cluster_classifiers[k].append(output_classifier) cluster_quality_loss = 0 alpha = 0.5 beta = 0.25 self._add_debug_output(Concatenate(axis=1)(embeddings_reshaped), 'eval_embeddings') for k in cluster_counts: # Create evaluation metrics self._add_debug_output(Concatenate(axis=1)(cluster_classifiers[k]), 'eval_classifications_k{}'.format(k)) cluster_centers = get_cluster_centers(embeddings_reshaped, cluster_classifiers[k]) self._add_debug_output(Concatenate(axis=1)(cluster_centers), 'eval_cluster_centers_k{}'.format(k)) cohesion = get_cluster_cohesion(cluster_centers, embeddings_reshaped, cluster_classifiers[k]) self._add_debug_output(cohesion, 'eval_cohesion_k{}'.format(k)) separation = get_cluster_separation(cluster_centers, cluster_classifiers[k]) self._add_debug_output(separation, 'eval_separation_k{}'.format(k)) cluster_quality_loss = Lambda(lambda cohesion: # Update the loss cluster_quality_loss + (alpha * cohesion - beta * separation) )(cohesion) cluster_quality_loss = Lambda(lambda x: x / len(cluster_counts))(cluster_quality_loss) additional_network_outputs['additional_loss'] = Activation('linear', name='additional_loss')(cluster_quality_loss) # Calculate the real cluster count cluster_count = self._s_layer('cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__lstm_units), name=name)(embeddings_merged)) cluster_count = self._s_layer('cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count) for i in range(self.__cluster_count_dense_layers): cluster_count = self._s_layer('cluster_count_dense{}'.format(i), lambda name: Dense(self.__cluster_count_dense_units, name=name))(cluster_count) cluster_count = self._s_layer('cluster_count_batch{}'.format(i), lambda name: BatchNormalization(name=name))(cluster_count) cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation('relu', name=name))(cluster_count) # The next layer is an output-layer, therefore the name must not be formatted cluster_count = self._s_layer( 'cluster_count_output', lambda name: Dense(len(cluster_counts), activation='softmax', name=name), format_name=False )(cluster_count) additional_network_outputs['cluster_count_output'] = cluster_count network_output.append(cluster_count) return True
def _build_network(self, network_input, network_output, additional_network_outputs): cluster_counts = list(self.data_provider.get_cluster_counts()) # The simple loss cluster NN requires a specific output: a list of softmax distributions # First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each # object etc. At the end, there is the cluster count output. # First we get an embedding for the network inputs embeddings = self._get_embedding(network_input) # Reshape all embeddings to 1d vectors # embedding_shape = self._embedding_nn.model.layers[-1].output_shape # embedding_size = np.prod(embedding_shape[1:]) embedding_shape = embeddings[0].shape embedding_size = int(str(np.prod(embedding_shape[1:]))) embedding_reshaper = self._s_layer('embedding_reshape', lambda name: Reshape((1, embedding_size), name=name)) embeddings_reshaped = [embedding_reshaper(embedding) for embedding in embeddings] cohesion_d_layer = self._s_layer('cohesion_d_layer', lambda name: Dense(2, activation='tanh', name=name)) cohesion_embeddings = [cohesion_d_layer(embedding) for embedding in embeddings_reshaped] self._add_additional_prediction_output( concat(cohesion_embeddings, axis=1), 'cohesion_embeddings' ) self._register_additional_embedding_comparison_regularisation( 'cluster_cohesion', # If the values are no longer used (as this is the case here), we need a dissimilarity weight, because # otherwise there is the trivial solution to map all inputs to 0 lambda x, y, similar: meier_cluster_cohesion(x, y, similar, dissimilar_distance_weight=-1), cohesion_embeddings ) # We need now the internal representation of the embeddings. This means we have to resize them. internal_embedding_size = self.__internal_embedding_size // 2 * 2 embedding_internal_resizer = self._s_layer('internal_embedding_resize', lambda name: Dense(internal_embedding_size, name=name)) embeddings_reshaped = [embedding_internal_resizer(embedding) for embedding in embeddings_reshaped] embedding_internal_resizer_act = LeakyReLU() embeddings_reshaped = [embedding_internal_resizer_act(embedding) for embedding in embeddings_reshaped] # Merge all embeddings to one tensor embeddings_merged = self._s_layer('embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped) # Use now some lstm-layers processed = embeddings_merged for i in range(self.__lstm_layers): tmp = self._s_layer( 'LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(internal_embedding_size // 2, return_sequences=True), name=name) )(processed) processed = Add()([processed, tmp]) # Split the tensor to seperate layers embeddings_processed = [self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input))] # Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution. # These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start # with the cluster count / object combinations. # First prepare some generally required layers layers = [] for i in range(self.__output_dense_layers): layers += [ self._s_layer('output_dense{}'.format(i), lambda name: Dense(self.__output_dense_units, name=name)), self._s_layer('output_batch'.format(i), lambda name: BatchNormalization(name=name)), LeakyReLU() # self._s_layer('output_relu'.format(i), lambda name: Activation(LeakyReLU(), name=name)) ] cluster_softmax = { k: self._s_layer('softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts } # Create now the outputs clusters_output = additional_network_outputs['clusters'] = {} for i in range(len(embeddings_processed)): embedding_proc = embeddings_processed[i] # Add the required layers for layer in layers: embedding_proc = layer(embedding_proc) input_clusters_output = clusters_output['input{}'.format(i)] = {} for k in cluster_counts: # Create now the required softmax distributions output_classifier = cluster_softmax[k](embedding_proc) input_clusters_output['cluster{}'.format(k)] = output_classifier network_output.append(output_classifier) # Calculate the real cluster count assert self.__cluster_count_lstm_layers >= 1 cluster_count = embeddings_merged for i in range(self.__cluster_count_lstm_layers - 1): cluster_count = self._s_layer('cluster_count_LSTM{}'.format(i), lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units, return_sequences=True), name=name)(cluster_count)) cluster_count = self._s_layer('cluster_count_LSTM{}_batch'.format(i), lambda name: BatchNormalization(name=name))(cluster_count) cluster_count = self._s_layer('cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units), name=name)(cluster_count)) cluster_count = self._s_layer('cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count) for i in range(self.__cluster_count_dense_layers): cluster_count = self._s_layer('cluster_count_dense{}'.format(i), lambda name: Dense(self.__cluster_count_dense_units, name=name))(cluster_count) cluster_count = self._s_layer('cluster_count_batch{}'.format(i), lambda name: BatchNormalization(name=name))(cluster_count) cluster_count = LeakyReLU()(cluster_count) # cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation(LeakyReLU(), name=name))(cluster_count) # The next layer is an output-layer, therefore the name must not be formatted cluster_count = self._s_layer( 'cluster_count_output', lambda name: Dense(len(cluster_counts), activation='softmax', name=name), format_name=False )(cluster_count) additional_network_outputs['cluster_count_output'] = cluster_count network_output.append(cluster_count) return True
def _build_network(self, network_input, network_output, additional_network_outputs): cluster_counts = list(self.data_provider.get_cluster_counts()) # f_update_global_state def f_update_global_state(x, s): xs = self._s_layer('f_update_global_state_CONCAT', lambda name: Concatenate(axis=2, name=name))( [x, s]) res = self._s_layer( 'f_update_global_state_BDLSTM', lambda name: Bidirectional( LSTM(self.__f_update_global_state_lstm_units), name=name))( xs) res = self._s_layer( 'f_update_global_state_BDLSTM_BNORM', lambda name: BatchNormalization(name=name))(res) for i in range(self.__f_update_global_state_dense_layer_count): res = self._s_layer( 'f_update_global_state_DENSE{}'.format(i), lambda name: Dense(self.__global_state_size, name=name))( res) res = self._s_layer( 'f_update_global_state_DENSE_BNORM{}'.format(i), lambda name: BatchNormalization(name=name))(res) res = self._s_layer('f_update_global_state_RELU{}'.format(i), lambda name: Activation('relu'))(res) return res # f_update_local_state def f_update_local_state(x, g): g = self._s_layer( 'f_update_local_state_REPEAT', lambda name: RepeatVector(self.input_count, name=name))(g) xg = self._s_layer('f_update_local_state_CONCAT', lambda name: Concatenate(axis=2, name=name))( [x, g]) for i in range(self.__f_update_local_state_dense_layer_count): xg = self._s_layer( 'f_update_local_state_DENSE{}'.format(i), lambda name: TimeDistributed( Dense(self.__f_update_local_state_units), name=name))( xg) xg = self._s_layer( 'f_update_local_state_BNORM{}'.format(i), lambda name: BatchNormalization(name=name))(xg) xg = self._s_layer('f_update_local_state_RELU{}'.format(i), lambda name: Activation('relu'))(xg) xg = self._s_layer( 'f_update_local_state_OUT_DENSE', lambda name: TimeDistributed(Dense(self.__local_state_size), name=name))(xg) xg = self._s_layer('f_update_local_state_OUT_BNORM', lambda name: BatchNormalization(name=name))(xg) xg = self._s_layer('f_update_local_state_OUT_RELU', lambda name: Activation('relu'))(xg) return xg # f_cluster_count def f_cluster_count(g): for i in range(self.__f_cluster_count_dense_layer_count): g = self._s_layer( 'f_cluster_count_DENSE{}'.format(i), lambda name: Dense( self.__f_cluster_count_units, name=name))(g) g = self._s_layer( 'f_cluster_count_BNORM{}'.format(i), lambda name: BatchNormalization(name=name))(g) g = self._s_layer('f_cluster_count_RELU{}'.format(i), lambda name: Activation('relu'))(g) return g # f_cluster_assignment def f_cluster_assignment(x, g): g = self._s_layer( 'f_cluster_assignment_gRESHAPE', lambda name: Reshape( (1, self.__global_state_size), name=name))(g) xg = self._s_layer('f_cluster_assignment_CONCAT', lambda name: Concatenate(axis=2, name=name))( [x, g]) for i in range(self.__f_cluster_assignment_dense_layer_count): xg = self._s_layer( 'f_cluster_assignment_DENSE{}'.format(i), lambda name: Dense(self.__f_cluster_assignment_units, name=name))(xg) xg = self._s_layer( 'f_cluster_assignment_BNORM{}'.format(i), lambda name: BatchNormalization(name=name))(xg) xg = self._s_layer('f_cluster_assignment_RELU{}'.format(i), lambda name: Activation('relu'))(xg) return xg # First we get an embedding for the network inputs embeddings = self._get_embedding(network_input) # Reshape all embeddings to 1d vectors embedding_shape = self._embedding_nn.model.layers[-1].output_shape embedding_size = np.prod(embedding_shape[1:]) embedding_reshaper = self._s_layer( 'embedding_reshape', lambda name: Reshape( (1, embedding_size), name=name)) embeddings_reshaped = [ embedding_reshaper(embedding) for embedding in embeddings ] # Merge all embeddings to one tensor embeddings = self._s_layer( 'embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped) # Create empty initial local states s = self._s_layer( 'local_states_init', lambda name: TimeDistributed(Dense(self.__local_state_size, kernel_initializer='zeros', bias_initializer='zeros', trainable=False), name=name))(embeddings) # Do the iterations for i in range(self.__iterations): # Update the gloabl state g = f_update_global_state(embeddings, s) # Update the local states s = f_update_local_state(embeddings, g) # Get the processed embeddings (=states) as a list embeddings_processed = [ self._s_layer('slice_{}'.format(i), lambda name: slice_layer(s, i, name)) for i in range(len(network_input)) ] # Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution. # These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start # with the cluster count / object combinations. # First prepare some generally required layers cluster_softmax = { k: self._s_layer( 'softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts } # Create now the outputs clusters_output = additional_network_outputs['clusters'] = {} for i in range(len(embeddings_processed)): embedding_proc = embeddings_processed[i] embedding_proc = f_cluster_assignment(embedding_proc, g) input_clusters_output = clusters_output['input{}'.format(i)] = {} for k in cluster_counts: # Create now the required softmax distributions output_classifier = cluster_softmax[k](embedding_proc) input_clusters_output['cluster{}'.format( k)] = output_classifier network_output.append(output_classifier) # Calculate the real cluster count # cluster_count = self._s_layer('cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__lstm_units), name=name)(embeddings_merged)) # cluster_count = self._s_layer('cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count) cluster_count = f_cluster_count(g) # The next layer is an output-layer, therefore the name must not be formatted cluster_count = self._s_layer( 'cluster_count_output', lambda name: Dense( len(cluster_counts), activation='softmax', name=name), format_name=False)(cluster_count) additional_network_outputs['cluster_count_output'] = cluster_count network_output.append(cluster_count) return True
def _build_network(self, network_input, network_output, additional_network_outputs): cluster_counts = list(self.data_provider.get_cluster_counts()) # The simple loss cluster NN requires a specific outputlayer: a list of softmax distributions # First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each # object etc. At the end, there is the cluster count output. def point_preprocessor(p): return Lambda(lambda x: K.tanh(x))(p) # Obsolete def add_dbg_output(name, layer): self._add_debug_output(layer, name, create_wrapper_layer=True) # debug_output.append(Activation('linear', name=name)(layer)) # First we get an embedding for the network inputs embeddings = self._get_embedding(network_input) # Reshape all embeddings to 1d vectors # embedding_shape = self._embedding_nn.model.layers[-1].output_shape # embedding_size = np.prod(embedding_shape[1:]) embedding_shape = embeddings[0].shape embedding_size = int(str(np.prod(embedding_shape[1:]))) embedding_reshaper = self._s_layer('embedding_reshape', lambda name: Reshape((1, embedding_size), name=name)) embeddings_reshaped = [embedding_reshaper(embedding) for embedding in embeddings] # Merge all embeddings to one tensor embeddings_merged = self._s_layer('embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped) # Use now some BDLSTM-layer to process all embeddings. processed = embeddings_merged for i in range(self.__lstm_layers): processed = self._s_layer( 'LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(self.__lstm_units, return_sequences=True), name=name) )(processed) processed = self._s_layer( 'LSTM_proc_{}_batch'.format(i), lambda name: BatchNormalization(name=name) )(processed) # Batch normalize everything if not already done if self.__lstm_layers == 0: add_dbg_output('EMBEDDINGS', processed) processed = BatchNormalization()(processed) add_dbg_output('EMBEDDINGS_NORMALIZED', processed) # Split the tensor to seperate layers embeddings_processed = [self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input))] # Prepare the embeddings for the kmeans input if self.__kmeans_input_dimension is not None: # We just update the input dimensions for the kmeans, therefore no activation function is used layers = [ self._s_layer('kmeans_dimension_changer_dense', lambda name: Dense(self.__kmeans_input_dimension, name=name)), self._s_layer('kmeans_dimension_changer_batch', lambda name: BatchNormalization(name=name)) ] for layer in layers: embeddings_processed = [layer(e) for e in embeddings_processed] # Apply the preprocessing for all embeddings (we force the network to have all values in the range [-1, 1]); # this makes kmeans easier embeddings_processed = [point_preprocessor(e) for e in embeddings_processed] add_dbg_output('EMBEDDINGS_PREPROCESSED', Concatenate(axis=1)(embeddings_processed)) self._add_additional_prediction_output(Concatenate(axis=1)(embeddings_processed), 'Processed_Embeddings') # Define a distance-function d = lambda x, y: K.sqrt(K.sum(K.square(x - y))) # euclidean distance def euclideanDistance(inputs, squared=False): if (len(inputs) != 2): raise 'oops' # For better gradient flow: remove the sqrt output = K.sum(K.square(inputs[0] - inputs[1]), axis=-1) if not squared: output = K.sqrt(output) output = K.expand_dims(output, 1) return output # Apply k-means for each possible k assert self.__kmeans_itrs > 0 cluster_vector_size = self.__kmeans_input_dimension #(self.__lstm_units * 2) if self.__lstm_layers > 0 else embedding_size cluster_assignements = {} def guess_initial_cluster(name): def get_name(layer_name): return "{}_{}".format(name, layer_name) def res(input_layer): nw = input_layer nw = self._s_layer( get_name('guess_initial_cluster_bdlstm0'), lambda name: Bidirectional(LSTM(cluster_vector_size * cluster_counts[-1], name=name)) )(nw) nw = self._s_layer( get_name('guess_initial_cluster_dense0'), lambda name: Dense(cluster_vector_size) )(nw) nw = self._s_layer( get_name('guess_initial_cluster_reshape0'), lambda name: Reshape((1, cluster_vector_size)) )(nw) return nw return res for k in cluster_counts: # Create initial cluster centers clusters = [self._s_layer( 'k_{}_init_{}'.format(k, i), lambda name: gaussian_random_layer((1, cluster_vector_size), name=name, only_execute_for_training=False) )(embeddings[0]) for i in range(k)] # # Create initial cluster guesses con_proc_embd = Concatenate(axis=1)(embeddings_processed) clusters = [guess_initial_cluster("k{}_{}".format(k, i))(con_proc_embd) for i in range(k)] # Apply the preprocessing clusters = [point_preprocessor(c) for c in clusters] self._add_additional_prediction_output( Concatenate(axis=1)(clusters), 'Initial cluster guesses (k={})'.format(k) ) # # # Use the first n input points # clusters = embeddings_processed[:k] # # Create initial cluster centers: # # 1) The first cluster center is the mean of all points # def l_mean(inputs): # inputs_len = len(inputs) # l = add(inputs) # l = Lambda(lambda x: x / inputs_len)(l) # return l # clusters = [l_mean(embeddings_processed)] # # 2) Create now all other cluster centers # if k > 1: # for i in range(1, k): # # Sum over all points # c_s = 0 # s_s = 0 # # for e in embeddings_processed: # # Get the distances to all points from the current embedding # dists = [] # for c in clusters: # dists.append(Lambda(lambda x: euclideanDistance(x, False), output_shape=lambda x: (x[0][0], 1))([c, e])) # # if len(dists) == 1: # dists = dists[0] # else: # dists = Concatenate()(dists) # # # Calculate a weight for this data point # w = Lambda(lambda x: K.exp(10 * K.min(x, axis=0)))(dists) # # # Update c_s and s_s # c_s = Lambda(lambda x: c_s + w * x)(e) # s_s = Lambda(lambda x: s_s + w)(e) # # # Calculate the new cluster center # c_s = Lambda(lambda x: c_s / s_s)(c_s) # # # Append it to the clusters list # clusters.append(c_s) # # Choose k random points and use them as initial clusters # c_i_embeddings = Concatenate(axis=1)(embeddings_processed) # c_i_embeddings = Lambda(lambda x: tf.random_shuffle(x))(c_i_embeddings) # non-differentiable; # clusters = [slice_layer(c_i_embeddings, i) for i in range(k)] for i in range(len(clusters)): add_dbg_output('INIT_CLUSTER_{}'.format(i), clusters[i]) # Cluster-assignements current_cluster_assignements = None # Idee zum neuen Mittelwert finden: # - Nicht mit Softmax rechnen sondern nur noch mit der Distanz # - Clusterzentrum mit Punkten Gewichten, falls die Distanz zum Cluster vom Punkt minimal ist (also kein anderer Cluster näher ist) # Do all iterations for i in range(self.__kmeans_itrs): def get_val_at(input, i_i): def at(val, indices): for index in indices: val = val[index] return val return Lambda(lambda x: at(x, i_i), output_shape=(1,))(input) # Recalculate the cluster centers (if required) if i > 0: clusters_old = clusters clusters = [] for c_i in range(k): c = 0 s = 1e-8 # Avoid zero-divison for e_i in range(len(embeddings_processed)): t = get_val_at(current_cluster_assignements[e_i], [1, c_i]) c = Lambda(lambda x: c + x * t, output_shape=(1, cluster_vector_size))(embeddings_processed[e_i]) # c = Lambda(lambda x: c + x * 10. * K.relu(t + 0.1 - K.max(current_cluster_assignements[e_i])), output_shape=(1, cluster_vector_size))(embeddings_processed[e_i]) s = Lambda(lambda x: x + s, output_shape=(1,))(t) # t = current_cluster_assignements[e_i][c_i] # c += t * embeddings_processed[e_i] # s += t c = Lambda(lambda x: x / s, output_shape=(1, cluster_vector_size))(c) # c = c / s c = Lambda(lambda x: 0 + x, output_shape=(1, cluster_vector_size))(c) add_dbg_output("k{}_ITR{}_ci{}".format(k, i, c_i), c) clusters.append(c) # Recalculate the assigned cluster centers for each embedding cluster_assignements[k] = [] current_cluster_assignements = cluster_assignements[k] e_i = 0 for e in embeddings_processed: # Calculate all distances to all cluster centers k_distances = [ Lambda(lambda x: euclideanDistance(x, True), output_shape=lambda x: (x[0][0], 1))([e, cluster]) # merge([e, cluster], mode=euclideanDistance, output_shape=lambda x: (x[0][0], 1)) for cluster in clusters # self._s_layer('kmeans_d', lambda name: Merge([])) ] # Merge the distances and calculate a softmax k_distances = Concatenate(axis=1)(k_distances) k_distances = Reshape((k,))(k_distances) add_dbg_output("k{}_ITR{}_e_i{}_DISTANCES_PLAIN".format(k, i, e_i), k_distances) # k_distances = Lambda(lambda x: 1 / (0.01 + x))(k_distances) # k_distances = Lambda(lambda x: -(1+x)**2)(k_distances) def kmax(c, x): return K.relu(x - c) + c def kmin(c, x): return -kmax(-c, -x) k_distances = Lambda(lambda x: -(1 + x)**2)(k_distances) # k_distances = Lambda(lambda x: -(1 + 3 * K.sqrt(x)) ** 3)(k_distances) # cluster_assignements[p_i][c_i] = -min(500, (1 + 3 * np.sqrt(d)) ** 3) # + 3/(1.+d) k_distances = Activation('softmax')(k_distances) # # Dirty tune the softmax a bit # k_distances = Lambda(lambda x: (x / K.max(x))**4)(k_distances) # Save the new distances current_cluster_assignements.append(k_distances) add_dbg_output("k{}_ITR{}_e_i{}_DISTANCES".format(k, i, e_i), k_distances) e_i += 1 self._add_additional_prediction_output( Concatenate(axis=1)(clusters), 'Final cluster guesses (k={})'.format(k) ) # Reshape all softmax layers for k in cluster_counts: cluster_assignements[k] = [ Reshape((1, k))(s) for s in cluster_assignements[k] ] # Append all softmax layers to the network output network_output += chain.from_iterable(map( lambda k: cluster_assignements[k], cluster_counts )) # Create the additional cluster output clusters_output = additional_network_outputs['clusters'] = {} for i in range(self.input_count): input_clusters_output = clusters_output['input{}'.format(i)] = {} for k in cluster_counts: input_clusters_output['cluster{}'.format(k)] = cluster_assignements[k][i] add_dbg_output('RESULT_INPUT{}_K{}'.format(i, k), cluster_assignements[k][i]) # Calculate the real cluster count cluster_count = self._s_layer('cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__lstm_units), name=name)(embeddings_merged)) cluster_count = self._s_layer('cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count) for i in range(self.__cluster_count_dense_layers): cluster_count = self._s_layer('cluster_count_dense{}'.format(i), lambda name: Dense(self.__cluster_count_dense_units, name=name))(cluster_count) cluster_count = self._s_layer('cluster_count_batch{}'.format(i), lambda name: BatchNormalization(name=name))(cluster_count) cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation('relu', name=name))(cluster_count) # The next layer is an output-layer, therefore the name must not be formatted cluster_count = self._s_layer( 'cluster_count_output', lambda name: Dense(len(cluster_counts), activation='softmax', name=name), format_name=False )(cluster_count) additional_network_outputs['cluster_count_output'] = cluster_count network_output.append(cluster_count) return True
def _build_network(self, network_input, network_output, additional_network_outputs): cluster_counts = self._get_cluster_counts() # The simple loss cluster NN requires a specific output: a list of softmax distributions # First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each # object etc. At the end, there is the cluster count output. # First we get an embedding for the network inputs embeddings = self._get_embedding(network_input) # Reshape all embeddings to 1d vectors # embedding_shape = self._embedding_nn.model.layers[-1].output_shape # embedding_size = np.prod(embedding_shape[1:]) embedding_shape = embeddings[0].shape embedding_size = int(str(np.prod(embedding_shape[1:]))) embedding_reshaper = self._s_layer( 'embedding_reshape', lambda name: Reshape( (1, embedding_size), name=name)) embeddings_reshaped = [ embedding_reshaper(embedding) for embedding in embeddings ] # Merge all embeddings to one tensor embeddings_merged = self._s_layer( 'embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped) # self._add_additional_prediction_output(embeddings_merged, 'Embeddings') # Use now some LSTM-layer to process all embeddings processed = embeddings_merged # Split the tensor to seperate layers embeddings_processed = [ self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input)) ] # Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution. # These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start # with the cluster count / object combinations. # First prepare some generally required layers layers = [] cluster_softmax = { k: self._s_layer( 'softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts } # Create now the outputs clusters_output = additional_network_outputs['clusters'] = {} for i in range(len(embeddings_processed)): embedding_proc = embeddings_processed[i] # Add the required layers for layer in layers: embedding_proc = layer(embedding_proc) input_clusters_output = clusters_output['input{}'.format(i)] = {} for k in cluster_counts: # Create now the required softmax distributions output_classifier = cluster_softmax[k](embedding_proc) input_clusters_output['cluster{}'.format( k)] = output_classifier network_output.append(output_classifier) # Calculate the real cluster count cluster_count = embeddings_merged cluster_count = self._s_layer( 'cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units), name=name)(cluster_count)) cluster_count = self._s_layer( 'cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count) # The next layer is an output-layer, therefore the name must not be formatted cluster_count = self._s_layer( 'cluster_count_output', lambda name: Dense( len(cluster_counts), activation='softmax', name=name), format_name=False)(cluster_count) additional_network_outputs['cluster_count_output'] = cluster_count # Add a regularizer that is based on the "Deep Divergence-Based CLustering" paper # We require these inputs: # 1) A list of embeddings # 2) A list of softmaxs # We have for each cluster count possibility a list of softmaxs and we calculate the regularization for all # possibilities, then they are weighted by their cluster probability. x_inputs = embeddings # ddbc_losses = [] d_a_losses = [] d_m_losses = [] triuAAt_losses = [] for k in cluster_counts: s_inputs = list( map( lambda i: Reshape((k, )) (additional_network_outputs['clusters']['input{}'.format( i)]['cluster{}'.format(k)]), range(len(embeddings)))) # ddbc_losses.append(get_ddbc_loss_function(x_inputs, s_inputs)[0]) _, d_a, triuAAt, d_m = get_ddbc_loss_function(x_inputs, s_inputs) d_a_losses.append(d_a) d_m_losses.append(d_m) triuAAt_losses.append(triuAAt) # ddbc_loss = concat_layer(input_count=len(ddbc_losses), axis=1)(ddbc_losses) # Concatenate(axis=1)(ddbc_losses) # self._add_debug_output(ddbc_loss, "ddbc_loss") # self._add_debug_output(cluster_count, "cluster_count") # ddbc_loss = Dot(axes=1)([ddbc_loss, cluster_count]) # self._add_debug_output(ddbc_loss, "ddbc_loss2") # self._register_additional_regularisation(ddbc_loss, "Ddbc_Loss") d_a_losses = concat(inputs=d_a_losses, axis=1) d_m_losses = concat(inputs=d_m_losses, axis=1) triuAAt_losses = concat(inputs=triuAAt_losses, axis=1) self._add_debug_output(d_a_losses, "d_a_losses") self._add_debug_output(d_m_losses, "d_m_losses") self._add_debug_output(triuAAt_losses, "triuAAt_losses") self._add_debug_output(cluster_count, "cluster_count") d_a_losses = Dot(axes=1)([d_a_losses, cluster_count]) d_m_losses = Dot(axes=1)([d_m_losses, cluster_count]) triuAAt_losses = Dot(axes=1)([triuAAt_losses, cluster_count]) self._add_debug_output(d_a_losses, "d_a_losses2") self._add_debug_output(d_m_losses, "d_m_losses2") self._add_debug_output(triuAAt_losses, "triuAAt_losses2") self._register_additional_regularisation(d_a_losses, "d_a_loss") self._register_additional_regularisation(d_m_losses, "d_m_loss") self._register_additional_regularisation(triuAAt_losses, "triuAAt_losses") network_output.append(cluster_count) return True
def _build_network(self, network_input, network_output, additional_network_outputs): cluster_counts = list(self.data_provider.get_cluster_counts()) # The simple loss cluster NN requires a specific output: a list of softmax distributions # First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each # object etc. At the end, there is the cluster count output. # First we get an embedding for the network inputs embeddings = self._get_embedding(network_input) # Reshape all embeddings to 1d vectors # embedding_shape = self._embedding_nn.model.layers[-1].output_shape # embedding_size = np.prod(embedding_shape[1:]) embedding_shape = embeddings[0].shape embedding_size = int(str(np.prod(embedding_shape[1:]))) embedding_reshaper = self._s_layer( 'embedding_reshape', lambda name: Reshape( (1, embedding_size), name=name)) embeddings_reshaped = [ embedding_reshaper(embedding) for embedding in embeddings ] # Merge all embeddings to one tensor embeddings_merged = self._s_layer( 'embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped) # Use now one LSTM-layer to process all embeddings lstm_units = embedding_size * 4 processed = self._s_layer( 'LSTM_proc_0', lambda name: Bidirectional(LSTM(lstm_units, return_sequences=True), name=name))(embeddings_merged) # Split the tensor to seperate layers embeddings_processed = [ self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input)) ] # Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution. # These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start # with the cluster count / object combinations. # First prepare some generally required layers output_dense_size = embedding_size * 10 layers = [ self._s_layer('output_dense0', lambda name: Dense(output_dense_size, name=name)), self._s_layer('output_batch0', lambda name: BatchNormalization(name=name)), self._s_layer('output_relu0', lambda name: Activation('relu', name=name)) ] cluster_softmax = { k: self._s_layer( 'softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts } # Create now the outputs clusters_output = additional_network_outputs['clusters'] = {} for i in range(len(embeddings_processed)): embedding_proc = embeddings_processed[i] # Add the required layers for layer in layers: embedding_proc = layer(embedding_proc) input_clusters_output = clusters_output['input{}'.format(i)] = {} for k in cluster_counts: # Create now the required softmax distributions output_classifier = cluster_softmax[k](embedding_proc) input_clusters_output['cluster{}'.format( k)] = output_classifier network_output.append(output_classifier) # Calculate the real cluster count cluster_count_dense_units = embedding_size * 8 cluster_count = self._s_layer( 'cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(lstm_units), name=name) (embeddings_merged)) cluster_count = self._s_layer( 'cluster_count_dense0', lambda name: Dense(cluster_count_dense_units, name=name))( cluster_count) cluster_count = self._s_layer( 'cluster_count_batch0', lambda name: BatchNormalization(name=name))(cluster_count) cluster_count = self._s_layer( 'cluster_count_relu0', lambda name: Activation('relu', name=name))(cluster_count) # Use the forward pass dropout self._add_debug_output(cluster_count, '00_cluster_count_before_forward_pass_dropout') if self._try_get_additional_build_config_value('forward_pass_dropout', default_value=False): cluster_count = ExtendedDropout( 0.5, train_phase_active=False, test_phase_active=True)(cluster_count) self._add_debug_output(cluster_count, '01_cluster_count_after_forward_pass_dropout') # The next layer is an output-layer, therefore the name must not be formatted cluster_count = self._s_layer( 'cluster_count_output', lambda name: Dense( len(cluster_counts), activation='softmax', name=name), format_name=False)(cluster_count) self._add_debug_output(cluster_count, '02_cluster_count_result') additional_network_outputs['cluster_count_output'] = cluster_count network_output.append(cluster_count) return True
def _build_network(self, network_input, network_output, additional_network_outputs): cluster_counts = list(self.data_provider.get_cluster_counts()) # The simple loss cluster NN requires a specific output: a list of softmax distributions # First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each # object etc. At the end, there is the cluster count output. # Obsolete def add_dbg_output(name, layer): self._add_debug_output(layer, name) # debug_output.append(Activation('linear', name=name)(layer)) # First we get an embedding for the network inputs embeddings = self._get_embedding(network_input) # Reshape all embeddings to 1d vectors # embedding_shape = self._embedding_nn.model.layers[-1].output_shape # embedding_size = np.prod(embedding_shape[1:]) embedding_shape = embeddings[0].shape embedding_size = int(str(np.prod(embedding_shape[1:]))) embedding_reshaper = self._s_layer( 'embedding_reshape', lambda name: Reshape( (1, embedding_size), name=name)) embeddings_reshaped = [ embedding_reshaper(embedding) for embedding in embeddings ] # Merge all embeddings to one tensor embeddings_merged = self._s_layer( 'embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped) # Use now some BDLSTM-layer to process all embeddings. processed = embeddings_merged for i in range(self.__lstm_layers): processed = self._s_layer( 'LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(self.__lstm_units, return_sequences=True), name=name))(processed) processed = self._s_layer( 'LSTM_proc_{}_batch'.format(i), lambda name: BatchNormalization(name=name))(processed) # Batch normalize everything if not already done add_dbg_output('EMBEDDINGS', processed) if self.__lstm_layers == 0: processed = BatchNormalization()(processed) add_dbg_output('EMBEDDINGS_NORMALIZED', processed) # Split the tensor to seperate layers embeddings_processed = [ self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input)) ] # Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution. # These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start # with the cluster count / object combinations. # # First prepare some generally required layers # layers = [] # for i in range(self.__output_dense_layers): # layers += [ # self._s_layer('output_dense{}'.format(i), lambda name: Dense(self.__output_dense_units, name=name)), # self._s_layer('output_batch'.format(i), lambda name: BatchNormalization(name=name)), # self._s_layer('output_relu'.format(i), lambda name: Activation('relu', name=name)) # ] # cluster_softmax = { # k: self._s_layer('softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts # } # # # Create now the outputs # clusters_output = additional_network_outputs['clusters'] = {} # for i in range(len(embeddings_processed)): # embedding_proc = embeddings_processed[i] # # # Add the required layers # for layer in layers: # embedding_proc = layer(embedding_proc) # # input_clusters_output = clusters_output['input{}'.format(i)] = {} # for k in cluster_counts: # # # Create now the required softmax distributions # output_classifier = cluster_softmax[k](embedding_proc) # input_clusters_output['cluster{}'.format(k)] = output_classifier # network_output.append(output_classifier) # Define a distance-function d = lambda x, y: K.sqrt(K.sum(K.square(x - y))) # euclidean distance def euclideanDistance(inputs, squared=False): if (len(inputs) != 2): raise 'oops' # For better gradient flow: remove the sqrt output = K.sum(K.square(inputs[0] - inputs[1]), axis=-1) if not squared: output = K.sqrt(output) output = K.expand_dims(output, 1) return output # Apply k-means for each possible k assert self.__kmeans_itrs > 0 cluster_vector_size = (self.__lstm_units * 2) if self.__lstm_layers > 0 else embedding_size cluster_assignements = {} for k in cluster_counts: # Create initial cluster centers # clusters = [self._s_layer( # 'k_{}_init_{}'.format(k, i), lambda name: gaussian_random_layer((1, cluster_vector_size), name=name, only_execute_for_training=False) # )(embeddings[0]) for i in range(k)] # # Use the first n input points clusters = embeddings_processed[:k] for i in range(len(clusters)): add_dbg_output('INIT_CLUSTER_{}'.format(i), clusters[i]) # Cluster-assignements current_cluster_assignements = None # Idee zum neuen Mittelwert finden: # - Nicht mit Softmax rechnen sondern nur noch mit der Distanz # - Clusterzentrum mit Punkten Gewichten, falls die Distanz zum Cluster vom Punkt minimal ist (also kein anderer Cluster näher ist) # Do all iterations for i in range(self.__kmeans_itrs): def get_val_at(input, i_i): def at(val, indices): for index in indices: val = val[index] return val return Lambda(lambda x: at(x, i_i), output_shape=(1, ))(input) # Recalculate the cluster centers (if required) if i > 0: clusters_old = clusters clusters = [] for c_i in range(k): c = 0 s = 0 for e_i in range(len(embeddings_processed)): t = get_val_at(current_cluster_assignements[e_i], [1, c_i]) c = Lambda(lambda x: c + x * t, output_shape=(1, cluster_vector_size))( embeddings_processed[e_i]) # c = Lambda(lambda x: c + x * 10. * K.relu(t + 0.1 - K.max(current_cluster_assignements[e_i])), output_shape=(1, cluster_vector_size))(embeddings_processed[e_i]) s = Lambda(lambda x: x + s, output_shape=(1, ))(t) # t = current_cluster_assignements[e_i][c_i] # c += t * embeddings_processed[e_i] # s += t c = Lambda(lambda x: x / s, output_shape=(1, cluster_vector_size))(c) # c = c / s c = Lambda(lambda x: 0 + x, output_shape=(1, cluster_vector_size))(c) add_dbg_output("k{}_ITR{}_ci{}".format(k, i, c_i), c) clusters.append(c) # Recalculate the assigned cluster centers for each embedding cluster_assignements[k] = [] current_cluster_assignements = cluster_assignements[k] e_i = 0 for e in embeddings_processed: # Calculate all distances to all cluster centers k_distances = [ Lambda(lambda x: euclideanDistance(x, True), output_shape=lambda x: (x[0][0], 1))([e, cluster]) # merge([e, cluster], mode=euclideanDistance, output_shape=lambda x: (x[0][0], 1)) for cluster in clusters # self._s_layer('kmeans_d', lambda name: Merge([])) ] # Merge the distances and calculate a softmax k_distances = Concatenate()(k_distances) k_distances = Reshape((k, ))(k_distances) add_dbg_output( "k{}_ITR{}_e_i{}_DISTANCES_PLAIN".format(k, i, e_i), k_distances) # k_distances = Lambda(lambda x: 1 / (0.01 + x))(k_distances) k_distances = Lambda(lambda x: -(1 + x)**2)(k_distances) k_distances = Activation('softmax')(k_distances) # # Dirty tune the softmax a bit # k_distances = Lambda(lambda x: (x / K.max(x))**4)(k_distances) # Save the new distances current_cluster_assignements.append(k_distances) add_dbg_output( "k{}_ITR{}_e_i{}_DISTANCES".format(k, i, e_i), k_distances) e_i += 1 # Reshape all softmax layers for k in cluster_counts: cluster_assignements[k] = [ Reshape((1, k))(s) for s in cluster_assignements[k] ] # Append all softmax layers to the network output network_output += chain.from_iterable( map(lambda k: cluster_assignements[k], cluster_counts)) # Create the additional cluster output clusters_output = additional_network_outputs['clusters'] = {} for i in range(self.input_count): input_clusters_output = clusters_output['input{}'.format(i)] = {} for k in cluster_counts: input_clusters_output['cluster{}'.format( k)] = cluster_assignements[k][i] add_dbg_output('RESULT_INPUT{}_K{}'.format(i, k), cluster_assignements[k][i]) # Calculate the real cluster count cluster_count = self._s_layer( 'cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__lstm_units), name=name) (embeddings_merged)) cluster_count = self._s_layer( 'cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count) for i in range(self.__cluster_count_dense_layers): cluster_count = self._s_layer( 'cluster_count_dense{}'.format(i), lambda name: Dense(self.__cluster_count_dense_units, name=name ))(cluster_count) cluster_count = self._s_layer( 'cluster_count_batch{}'.format(i), lambda name: BatchNormalization(name=name))(cluster_count) cluster_count = self._s_layer( 'cluster_count_relu{}'.format(i), lambda name: Activation('relu', name=name))(cluster_count) # The next layer is an output-layer, therefore the name must not be formatted cluster_count = self._s_layer( 'cluster_count_output', lambda name: Dense( len(cluster_counts), activation='softmax', name=name), format_name=False)(cluster_count) additional_network_outputs['cluster_count_output'] = cluster_count network_output.append(cluster_count) return True
def _build_network(self, network_input, network_output, additional_network_outputs): cluster_counts = list(self.data_provider.get_cluster_counts()) # The simple loss cluster NN requires a specific output: a list of softmax distributions # First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each # object etc. At the end, there is the cluster count output. # First we get an embedding for the network inputs embeddings = self._get_embedding(network_input) # Reshape all embeddings to 1d vectors # embedding_shape = self._embedding_nn.model.layers[-1].output_shape # embedding_size = np.prod(embedding_shape[1:]) embedding_shape = embeddings[0].shape embedding_size = int(str(np.prod(embedding_shape[1:]))) embedding_reshaper = self._s_layer('embedding_reshape', lambda name: Reshape((1, embedding_size), name=name)) embeddings_reshaped = [embedding_reshaper(embedding) for embedding in embeddings] # Merge all embeddings to one tensor embeddings_merged = self._s_layer('embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped) self._add_additional_prediction_output(embeddings_merged, '0_Embeddings') # The value range of the embeddings must be [-1, 1] to allow an efficient execution of the cluster evaluations, # so be sure the embeddings are processed by tanh or some similar activation # Use some LSTMs for some kind of preprocessing. After these layers a regularisation is applied processed = embeddings_merged for i in range(self.__pre_lstm_layers): processed = self._s_layer( 'PRE_LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(self.__lstm_units, return_sequences=True), name=name) )(processed) processed = self._s_layer( 'PRE_LSTM_proc_{}_batch'.format(i), lambda name: BatchNormalization(name=name) )(processed) # Reshape the data now to a embeddings sized representation representation processed = TimeDistributed(Dense(embedding_size, activation='tanh'))(processed) # # Store these processed embeddings # preprocessed_embeddings = [slice_layer(processed, i) for i in range(len(network_input))] self._add_additional_prediction_output(processed, "1_LSTM_Preprocessed_Embeddings") # Use now some LSTM-layer to process all embeddings # processed = embeddings_merged for i in range(self.__lstm_layers): processed = self._s_layer( 'LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(self.__lstm_units, return_sequences=True), name=name) )(processed) processed = self._s_layer( 'LSTM_proc_{}_batch'.format(i), lambda name: BatchNormalization(name=name) )(processed) # Split the tensor to seperate layers embeddings_processed = [self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input))] # Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution. # These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start # with the cluster count / object combinations. # First prepare some generally required layers layers = [] for i in range(self.__output_dense_layers): layers += [ self._s_layer('output_dense{}'.format(i), lambda name: Dense(self.__output_dense_units, name=name)), self._s_layer('output_batch'.format(i), lambda name: BatchNormalization(name=name)), # self._s_layer('output_relu'.format(i), lambda name: Activation('relu', name=name)) LeakyReLU() ] cluster_softmax = { k: self._s_layer('softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts } # Create now the outputs clusters_output = additional_network_outputs['clusters'] = {} cluster_classifiers = {k: [] for k in cluster_counts} for i in range(len(embeddings_processed)): embedding_proc = embeddings_processed[i] # Add the required layers for layer in layers: embedding_proc = layer(embedding_proc) input_clusters_output = clusters_output['input{}'.format(i)] = {} for k in cluster_counts: # Create now the required softmax distributions output_classifier = cluster_softmax[k](embedding_proc) input_clusters_output['cluster{}'.format(k)] = output_classifier network_output.append(output_classifier) cluster_classifiers[k].append(output_classifier) # clustering_quality = 0 # sum_cohesion = 0 # sum_separation = 0 # alpha = 0.5 # beta = 0.25 # self._add_debug_output(Concatenate(axis=1)(preprocessed_embeddings), 'eval_embeddings') # # Squared euclidean distance # distance_f = lambda x, y: K.sum(K.square(x - y), axis=2) # # Euclidean distance # distance_f = lambda x, y: K.sqrt(K.sum(K.square(x - y), axis=2)) # for k in cluster_counts: # # # Create evaluation metrics # self._add_debug_output(Concatenate(axis=1)(cluster_classifiers[k]), 'eval_classifications_k{}'.format(k)) # cluster_centers = get_cluster_centers(preprocessed_embeddings, cluster_classifiers[k]) # self._add_debug_output(Concatenate(axis=1)(cluster_centers), 'eval_cluster_centers_k{}'.format(k)) # cohesion = get_cluster_cohesion(cluster_centers, preprocessed_embeddings, cluster_classifiers[k]) # self._add_debug_output(cohesion, 'eval_cohesion_k{}'.format(k)) # separation = get_cluster_separation(cluster_centers, cluster_classifiers[k]) # self._add_debug_output(separation, 'eval_separation_k{}'.format(k)) # # self._add_additional_prediction_output( # Concatenate(axis=1, name='2_cluster_centers_k{}'.format(k))(cluster_centers), # 'cluster_centers_k{}'.format(k) # ) # # sum_cohesion = Lambda(lambda cohesion: sum_cohesion + cohesion)(cohesion) # sum_separation = Lambda(lambda separation: sum_separation + separation)(separation) # # clustering_quality = Lambda(lambda cohesion: # # # # # Update the loss # # clustering_quality + (alpha * cohesion - beta * separation) # # )(cohesion) # # Add alpha and beta to the cohesion and the separation # sum_cohesion = Lambda(lambda sum_cohesion: alpha * sum_cohesion)(sum_cohesion) # sum_separation = Lambda(lambda sum_separation: beta * sum_separation)(sum_separation) # # # Normalize the cohesion and the separation by the cluster_counts # sum_cohesion = Lambda(lambda sum_cohesion: sum_cohesion / len(cluster_counts))(sum_cohesion) # sum_separation = Lambda(lambda sum_separation: sum_separation / len(cluster_counts))(sum_separation) # # # Add the losses for the cohesion and the separation. Use for both the same loss function # cluster_quality_loss = lambda x: lambda similiarty_loss, x=x: K.exp(- similiarty_loss * similiarty_loss * 4) * x # self._register_additional_grouping_similarity_loss( # 'cluster_cohesion', # cluster_quality_loss(sum_cohesion) # ) # self._register_additional_grouping_similarity_loss( # 'cluster_separation', # cluster_quality_loss(- K.log(1 + 2 * sum_separation)) # ) # # Normalize the cluster quality # clustering_quality = Lambda(lambda x: x / len(cluster_counts))(clustering_quality) # # # What to do with the cluster quality? We use it for an additional loss, this loss should optimize # # the cluster quality as soon as the clustering works relatively well. # self._register_additional_grouping_similarity_loss( # 'cluster_quality', # lambda similiarty_loss: K.exp(- similiarty_loss * similiarty_loss) * clustering_quality # ) # Calculate the real cluster count cluster_count = self._s_layer('cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__lstm_units), name=name)(embeddings_merged)) cluster_count = self._s_layer('cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count) for i in range(self.__cluster_count_dense_layers): cluster_count = self._s_layer('cluster_count_dense{}'.format(i), lambda name: Dense(self.__cluster_count_dense_units, name=name))(cluster_count) cluster_count = self._s_layer('cluster_count_batch{}'.format(i), lambda name: BatchNormalization(name=name))(cluster_count) # cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation('relu', name=name))(cluster_count) cluster_count = LeakyReLU()(cluster_count) # The next layer is an output-layer, therefore the name must not be formatted cluster_count = self._s_layer( 'cluster_count_output', lambda name: Dense(len(cluster_counts), activation='softmax', name=name), format_name=False )(cluster_count) additional_network_outputs['cluster_count_output'] = cluster_count network_output.append(cluster_count) return True
def _build_network(self, network_input, network_output, additional_network_outputs): cluster_counts = list(self.data_provider.get_cluster_counts()) # The simple loss cluster NN requires a specific output: a list of softmax distributions # First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each # object etc. At the end, there is the cluster count output. # First we get an embedding for the network inputs embeddings = self._get_embedding(network_input) # Do some dropout d_out = self._s_layer('embedding_dout', lambda name: Dropout(0.5, name=name)) embeddings = [d_out(embedding) for embedding in embeddings] # Reshape all embeddings to 1d vectors # embedding_shape = self._embedding_nn.model.layers[-1].output_shape # embedding_size = np.prod(embedding_shape[1:]) embedding_shape = embeddings[0].shape embedding_size = int(str(np.prod(embedding_shape[1:]))) embedding_reshaper = self._s_layer( 'embedding_reshape', lambda name: Reshape( (1, embedding_size), name=name)) embeddings_reshaped = [ embedding_reshaper(embedding) for embedding in embeddings ] self._add_debug_output(concat(embeddings_reshaped, axis=1), 'embeddings_reshaped') embeddings_reshaped = reweight_values(embeddings_reshaped, self._get_clustering_hint()) self._add_debug_output(concat(embeddings_reshaped, axis=1), 'embeddings_processed_new') # # Implement the KL-divergence on this layer. First, like lukic et al., do another fully connedted layer # kl_embeddings = embeddings_reshaped # kl_dense0 = self._s_layer('kl_dense0', lambda name: Dense(self.__kl_embedding_size, name=name, activation='relu')) # kl_embeddings = [kl_dense0(kl_embedding) for kl_embedding in kl_embeddings] # kl_softmax = self._s_layer('kl_softmax', lambda name: Dense(self.__kl_embedding_size, name=name, activation='softmax')) # kl_embeddings = [kl_softmax(kl_embedding) for kl_embedding in kl_embeddings] # self._register_additional_embedding_comparison_regularisation( # 'KL_divergence', # lukic_kl_divergence, # kl_embeddings, # weight=self.__kl_divergence_factor # ) # We need now the internal representation of the embeddings. This means we have to resize them. internal_embedding_size = self.__internal_embedding_size // 2 * 2 embedding_internal_resizer = self._s_layer( 'internal_embedding_resize', lambda name: Dense(internal_embedding_size, name=name)) embeddings_reshaped = [ embedding_internal_resizer(embedding) for embedding in embeddings_reshaped ] embedding_internal_resizer_act = LeakyReLU() embeddings_reshaped = [ embedding_internal_resizer_act(embedding) for embedding in embeddings_reshaped ] # Merge all embeddings to one tensor embeddings_merged = self._s_layer( 'embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped) # Use now some lstm-layers processed = embeddings_merged for i in range(self.__lstm_layers): tmp = self._s_layer( 'LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(internal_embedding_size // 2, return_sequences=True), name=name))(processed) processed = Add()([processed, tmp]) # Split the tensor to seperate layers embeddings_processed = [ self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input)) ] # Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution. # These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start # with the cluster count / object combinations. # First prepare some generally required layers layers = [] for i in range(self.__output_dense_layers): layers += [ self._s_layer( 'output_dense{}'.format(i), lambda name: Dense(self.__output_dense_units, name=name)), self._s_layer('output_batch'.format(i), lambda name: BatchNormalization(name=name)), LeakyReLU(), Dropout(0.5) # self._s_layer('output_relu'.format(i), lambda name: Activation(LeakyReLU(), name=name)) ] cluster_softmax = { k: self._s_layer( 'softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts } # Create now the outputs clusters_output = additional_network_outputs['clusters'] = {} for i in range(len(embeddings_processed)): embedding_proc = embeddings_processed[i] # Add the required layers for layer in layers: embedding_proc = layer(embedding_proc) input_clusters_output = clusters_output['input{}'.format(i)] = {} for k in cluster_counts: # Create now the required softmax distributions output_classifier = cluster_softmax[k](embedding_proc) input_clusters_output['cluster{}'.format( k)] = output_classifier network_output.append(output_classifier) # Calculate the real cluster count assert self.__cluster_count_lstm_layers >= 1 cluster_count = embeddings_merged for i in range(self.__cluster_count_lstm_layers - 1): cluster_count = self._s_layer( 'cluster_count_LSTM{}'.format(i), lambda name: Bidirectional(LSTM( self.__cluster_count_lstm_units, return_sequences=True), name=name)(cluster_count)) cluster_count = self._s_layer( 'cluster_count_LSTM{}_batch'.format(i), lambda name: BatchNormalization(name=name))(cluster_count) cluster_count = self._s_layer( 'cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__cluster_count_lstm_units), name=name)(cluster_count)) cluster_count = self._s_layer( 'cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count) for i in range(self.__cluster_count_dense_layers): cluster_count = self._s_layer( 'cluster_count_dense{}'.format(i), lambda name: Dense(self.__cluster_count_dense_units, name=name ))(cluster_count) cluster_count = self._s_layer( 'cluster_count_batch{}'.format(i), lambda name: BatchNormalization(name=name))(cluster_count) cluster_count = LeakyReLU()(cluster_count) # cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation(LeakyReLU(), name=name))(cluster_count) # The next layer is an output-layer, therefore the name must not be formatted cluster_count = Dropout(0.5)(cluster_count) cluster_count = self._s_layer( 'cluster_count_output', lambda name: Dense( len(cluster_counts), activation='softmax', name=name), format_name=False)(cluster_count) additional_network_outputs['cluster_count_output'] = cluster_count network_output.append(cluster_count) return True