def _conv_maxout_layer(last_layer, n_filts, name_prefix, dropout=True): conv_a = Conv2D( n_filts, (self.config.filt_time_width, self.config.filt_freq_width), strides=( self.config.filt_time_stride, self.config.filt_freq_stride, ), padding='same', name=name_prefix + '_a', **_layer_kwargs() )(last_layer) conv_b = Conv2D( n_filts, (self.config.filt_time_width, self.config.filt_freq_width), strides=( self.config.filt_time_stride, self.config.filt_freq_stride, ), padding='same', name=name_prefix + '_b', **_layer_kwargs() )(last_layer) last = Maximum(name=name_prefix + '_m')([conv_a, conv_b]) # pre-weights (i.e. post max), as per # http://jmlr.org/proceedings/papers/v28/goodfellow13.pdf if dropout: last = Dropout( train_config.dropout, name=name_prefix + '_d', seed=self._cur_weight_seed)(last) self._cur_weight_seed += 1 return last
def build_generator(self): example = Input(shape=(self.feature_dims, )) noise = Input(shape=(self.z_dims, )) x = Concatenate(axis=1)([example, noise]) for dim in self.generator_layers[1:]: x = Dense(dim)(x) x = Activation(activation='sigmoid')(x) x = Maximum()([example, x]) generator = Model([example, noise], x, name='generator') generator.summary() return generator
def build_generator(self, n_hidden=256, h_activation='relu', regularizers={}, batchnorm=False, out_activation='sigmoid', drop_rate=0, summary=False): """Builds a generator using the passed hyperparameters""" # Input: xz, z, or x x = Input(shape=(self.x_dim,), name='g_x_input') z = Input(shape=(self.z_dim,), name='g_z_input') if self.g_input == 'z': g_input = z elif self.g_input == 'x': g_input = x else: g_input = Concatenate(axis=1, name='g_xz_input')([x, z]) # Hidden hidden = Dense(n_hidden, activation=h_activation, name='g_hidden_relu')(g_input) if batchnorm: hidden = BatchNormalization(name='g_hidden_bn')(hidden) perturb = Dense(self.x_dim, activation=out_activation, **regularizers, name='g_perturb_sigmoid')(hidden) # Dropout perturb = Dropout(drop_rate, name='perturb_dropout')(perturb) perturb = K.minimum(perturb, 1) # NB: dropout scales up the kept inputs, # so clip to stay <=1 (for max later).. # use K.clip Or K.minimum(perturb, 1) # Output x_adv = Maximum(name='g_adv_max')([perturb, x]) self.generator = Model([x, z], x_adv, name='Generator') if summary: self.generator.summary(); print() # G parameters for logging self.reg = get_reg_factors(regularizers) self.g_log = {'in': self.g_input, 'h': f'({n_hidden},{h_activation})', 'bn': batchnorm, 'reg': self.reg, 'drop': drop_rate} return self.generator
def _conv_maxout_layer(last_layer, n_filts, name_prefix, dropout=True): conv_a = Conv2D(n_filts, self._filt_size, padding='same', name=name_prefix + '_a', **self._layer_kwargs)(last_layer) conv_b = Conv2D(n_filts, self._filt_size, padding='same', name=name_prefix + '_b', **self._layer_kwargs)(last_layer) last = Maximum(name=name_prefix + '_m')([conv_a, conv_b]) # pre-weights (i.e. post max), as per # http://jmlr.org/proceedings/papers/v28/goodfellow13.pdf if dropout: last = Dropout(self._dropout_p, name=name_prefix + '_d')(last) return last
def build_network(self, X=None, y=None): shape = (self.img_size[0], self.img_size[1], 3) input_layer = Input(shape=shape, name='input') last_ = input_layer # Convolutional section last_conv_per_level = [] sizes = [] for conv_level in range(self.conv_num + 1): nfilters = self.conv_filter_num * 2**conv_level sizes.append(nfilters) # Convolutional layers for c in range(self.conv_consecutive): #cfs = max(2, self.conv_filter_size[0] - conv_level) #cfs = (cfs, cfs) cfs = self.conv_filter_size last_ = Conv2D(nfilters, cfs, activation=self.conv_activation, padding=self.conv_padding, kernel_initializer=glorot_uniform(seed=42), name='conv%d-%d' % (conv_level, c))(last_) last_conv_per_level.append(last_) # Pooling layer if conv_level != self.conv_num: last_ = MaxPooling2D(pool_size=self.pool_size, strides=self.pool_strides, name='pool%d' % conv_level)(last_) last_per_label = [None] if self.multiple_out_streams: last_before = last_ for d in range(1, self.num_labels + 1): last_ = last_before # Deconvolutional section for conv_level in range(self.conv_num)[::-1]: # FIXME #cc = Concatenate(axis=3) cc = Add() last_ = cc([ Conv2DTranspose( sizes[conv_level], self.pool_size, strides=(self.pool_size, self.pool_size), )(last_), last_conv_per_level[conv_level] ]) for c in range(self.conv_consecutive): glorot = glorot_uniform(seed=42) last_ = Conv2D(sizes[conv_level], self.conv_filter_size, activation=self.conv_activation, kernel_initializer=glorot, padding=self.conv_padding)(last_) last_per_label.append(last_) else: # Deconvolutional section for conv_level in range(self.conv_num)[::-1]: # FIXME #cc = Concatenate(axis=3) cc = Add() last_ = cc([ Conv2DTranspose( sizes[conv_level], self.pool_size, strides=(self.pool_size, self.pool_size), )(last_), last_conv_per_level[conv_level] ]) for c in range(self.conv_consecutive): #cfs = max(2, self.conv_filter_size[0] - conv_level) #cfs = (cfs, cfs) cfs = self.conv_filter_size last_ = Conv2D(sizes[conv_level], cfs, activation=self.conv_activation, kernel_initializer=glorot_uniform(seed=42), padding=self.conv_padding)(last_) for d in range(1, self.num_labels + 1): last_per_label.append(last_) outputs = [] scoring_outputs = [] prob_outputs = [] reverse_layer = Lambda(lambda x: -x) minimum_layer = \ lambda x, y: reverse_layer(Maximum()([reverse_layer(x), reverse_layer(y)])) scoring = Conv2D(1, (1, 1), activation='linear')(last_per_label[-1]) for d in range(1, self.num_labels + 1): if self.pointwise_ordinal: if d > 1: if self.parallel_hyperplanes: next_output = Conv2D(1, (1, 1), activation='linear')(scoring) else: next_output = Conv2D(1, (1, 1), activation='linear')( last_per_label[d]) next_output = Activation('sigmoid')(next_output) scoring_outputs.append(next_output) prob_outputs.append(next_output) if self.pointwise_ordinal == 'min': next_output = minimum_layer(outputs[-1], next_output) elif self.pointwise_ordinal == 'mul': next_output = Multiply()([outputs[-1], next_output]) else: if self.parallel_hyperplanes: next_output = Conv2D(1, (1, 1), activation='linear')(scoring) else: next_output = Conv2D(1, (1, 1), activation='linear')( last_per_label[d]) next_output = Activation('sigmoid', name='sigmoid%d' % d)(next_output) scoring_outputs.append(next_output) prob_outputs.append(next_output) else: next_output = Conv2D(1, (1, 1), activation='linear', name='out%d' % d)(last_per_label[d]) next_output = Activation('sigmoid', name='sigmoid%d' % d)(next_output) scoring_outputs.append(next_output) prob_outputs.append(next_output) outputs.append(next_output) out = Concatenate(axis=-1, name='output')(outputs) subtract = lambda x, y: Add()([x, reverse_layer(y)]) elem_hinge = lambda x: Lambda(lambda k: K.maximum(k, 0.))(x) sign = lambda x: Lambda(lambda k: K.sign(k))(x) hor_monoticity_output = None ver_monoticity_output = None if self.include_distances: dshape = (self.img_size[0], self.img_size[1], 1) distance_input = Input(shape=dshape, name='dists') w = 5 left_crop = Cropping2D(((0, 0), (w, 0))) right_crop = Cropping2D(((0, 0), (0, w))) top_crop = Cropping2D(((w, 0), (0, 0))) bottom_crop = Cropping2D(((0, w), (0, 0))) left_crop_dists = left_crop(distance_input) right_crop_dists = right_crop(distance_input) top_crop_dists = top_crop(distance_input) bottom_crop_dists = bottom_crop(distance_input) horizontal_dist_diff = sign( subtract(left_crop_dists, right_crop_dists)) vertical_dist_diff = sign( subtract(top_crop_dists, bottom_crop_dists)) hor_monoticity_outputs = [] ver_monoticity_outputs = [] for p in scoring_outputs: left_crop_probs = left_crop(p) right_crop_probs = right_crop(p) top_crop_probs = top_crop(p) bottom_crop_probs = bottom_crop(p) horizontal_prob_diff = subtract(left_crop_probs, right_crop_probs) horizontal_monoticity = Multiply()( [horizontal_dist_diff, horizontal_prob_diff]) vertical_prob_diff = subtract(top_crop_probs, bottom_crop_probs) vertical_monoticity = Multiply()( [vertical_dist_diff, vertical_prob_diff]) horizontal_monoticity = elem_hinge(horizontal_monoticity) vertical_monoticity = elem_hinge(vertical_monoticity) hor_monoticity_outputs.append(horizontal_monoticity) ver_monoticity_outputs.append(vertical_monoticity) hor_monoticity_output = \ Add(name='hor-monoticity')(hor_monoticity_outputs) ver_monoticity_output = \ Add(name='ver-monoticity')(ver_monoticity_outputs) print len(prob_outputs) """ if self.num_labels > 2: ord_regs = [] for d in range(self.num_labels - 1): ordinal_diff = Add()([prob_outputs[d + 1], Lambda(lambda x: -x)(prob_outputs[d])]) ordinal_reg = Lambda(lambda x: K.maximum(x, 0.))(ordinal_diff) ord_regs.append(ordinal_reg) ord_regs = Add(name='ord-reg')(ord_regs) else: ordinal_diff = Add()([prob_outputs[1], Lambda(lambda x: -x)(prob_outputs[0])]) ord_regs = Lambda(lambda x: K.maximum(x, 0.), name='ord-reg')(ordinal_diff) """ model_inputs = [input_layer] model_outputs = [out] loss = {'output': self.loss} loss_weights = {'output': 1.} if self.include_distances: smoothness_weight = 0.5 * self.smoothness_weight model_inputs.append(distance_input) model_outputs.append(hor_monoticity_output) model_outputs.append(ver_monoticity_output) loss['hor-monoticity'] = mean_value_loss loss['ver-monoticity'] = mean_value_loss loss_weights['hor-monoticity'] = smoothness_weight loss_weights['ver-monoticity'] = smoothness_weight model = Model(inputs=[input_layer], outputs=[out]) opt_model = Model(inputs=model_inputs, outputs=model_outputs) opt_model.compile( 'adadelta', loss=loss, loss_weights=loss_weights, metrics={ 'output': [worst_dice_coef, best_dice_coef], }, ) self.final_shape = out._keras_shape return opt_model, model
# for each member of support set for tt in combinations(range(numsupportset), 2): aa = siam([convolutionlayers[tt[0]], convolutionlayers[tt[1]]]) pairwiseinteractions[tt[0]].append(aa) pairwiseinteractions[tt[1]].append(aa) # Get Siamese interactions for query image targetinteractions = [] for i in range(numsupportset): aa = siam([targetembedding, convolutionlayers[i]]) targetinteractions.append(aa) pairwiseinteractions[i].append( aa) # add this interaction to the set of interaction for this member # Take 4 layer MLP transform on Max pooling of interactions to serve as Full Context Embedding (FCE) maxi = Maximum() modelinputs = [] for i in range(numsupportset): modelinputs.append(relationalembedding(maxi(pairwiseinteractions[i]))) modelinputs.append(relationalembedding(maxi(targetinteractions))) supportlabels = Input((numsupportset, classes_per_set)) modelinputs.append(supportlabels) knnsimilarity = MatchEuclidean(nway=classes_per_set)(modelinputs) model = Model(inputs=[input1, supportlabels], outputs=knnsimilarity) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
def combined_model(self, batch_size, max_n_sentences, max_sentence_length, max_word_length): sentence_remainder, word_remainder = self.remainder_input_tensor( max_n_sentences, max_sentence_length) exists = Input(shape=(max_n_sentences, max_sentence_length), dtype='float32', name='exists') known = Input(shape=(max_n_sentences, max_sentence_length), dtype='float32', name='known') # embed word word_input = Input(shape=(max_n_sentences, max_sentence_length), dtype='int32', name='doc-wd_in') embedded_word = self.embed_word_document(word_input) # masked # embed char char_input = Input(shape=(max_n_sentences, max_sentence_length, max_word_length), dtype='int32', name='doc_ch_in') embedded_char = self.embed_char_document(char_input) # combine (masked during merging) if False: # concat_word = Concatenate()([embedded_char, embedded_word]) # concat_word = self.conv_dense(concat_word) embedded_word = self.max_relu(embedded_word) embedded_char = self.max_relu(embedded_char) concat_word = Maximum()([embedded_word, embedded_char]) # concat_word = self.max_tanh(concat_word) elif self.mode == 'max': # embedded_word = self.word_linear(embedded_word) # embedded_char = self.char_linear(embedded_char) concat_word = Maximum()([embedded_word, embedded_char]) # concat_word = self.max_tanh(concat_word) elif self.mode == 'switch': if True: concat_word = Switch(self.word_embedding_dim, 4)( [exists, known, embedded_word, embedded_char]) else: concat_word = embedded_char def expand(x): y = K.expand_dims(exists) return K.repeat_elements(y, self.word_embedding_dim, axis=3) def calc_output_shape(in_shape): return in_shape + (self.word_embedding_dim, ) expanded_exists = Lambda(expand, output_shape=calc_output_shape)(exists) concat_word = Multiply()([concat_word, expanded_exists]) # to document output = self.embedded_word_to_document(max_sentence_length, self.word_embedding_dim, concat_word, sentence_remainder, word_remainder) if not self.REMAINDER: model = Model([exists, known, word_input, char_input], output) else: model = Model([ exists, known, word_input, char_input, sentence_remainder, word_remainder ], output) self.compile_model(model) return model
counter += 1 if counter % 50 == 0: print("i=" + str(counter)) if counter == int((1 - splits) * num_batches): counter = 0 if shuffle: np.random.shuffle(smp_idx) nb_hidden = 1000 input1 = Input(shape=(1000, )) input2 = Input(shape=(1000, )) input3 = Input(shape=(1000, )) avgL = Average()([input1, input2, input3]) maxL = Maximum()([input1, input2, input3]) minL = Minimum()([input1, input2, input3]) concatL = Concatenate()([avgL, maxL]) layer1 = Dense(nb_hidden, kernel_regularizer=regularizers.l1(0.00001), bias_initializer='zeros', activation='relu')(concatL) #layer1=Dropout(0.5)(layer1) out = Dense(nb_classes, kernel_regularizer=regularizers.l1(0.00001), bias_initializer='zeros', kernel_initializer='identity', activation='softmax')(layer1) model = Model([input1, input2, input3], out) #model=load_model('genFit_ens450.h5');
def _construct_acoustic_model(self, train_config=TrainConfig()): # construct an acoustic model from scratch self._cur_weight_seed = self.config.weight_seed def _layer_kwargs(): ret = { 'activation': 'linear', 'kernel_initializer': RandomUniform( minval=-self.config.weight_init_mag, maxval=self.config.weight_init_mag, seed=self._cur_weight_seed, ), } self._cur_weight_seed = self._cur_weight_seed + 1 if train_config.training_stage == 'sgd': ret['kernel_regularizer'] = l2(train_config.sgd_reg) return ret # convolutional layer pattern def _conv_maxout_layer(last_layer, n_filts, name_prefix, dropout=True): conv_a = Conv2D( n_filts, (self.config.filt_time_width, self.config.filt_freq_width), strides=( self.config.filt_time_stride, self.config.filt_freq_stride, ), padding='same', name=name_prefix + '_a', **_layer_kwargs() )(last_layer) conv_b = Conv2D( n_filts, (self.config.filt_time_width, self.config.filt_freq_width), strides=( self.config.filt_time_stride, self.config.filt_freq_stride, ), padding='same', name=name_prefix + '_b', **_layer_kwargs() )(last_layer) last = Maximum(name=name_prefix + '_m')([conv_a, conv_b]) # pre-weights (i.e. post max), as per # http://jmlr.org/proceedings/papers/v28/goodfellow13.pdf if dropout: last = Dropout( train_config.dropout, name=name_prefix + '_d', seed=self._cur_weight_seed)(last) self._cur_weight_seed += 1 return last # inputs feat_input = Input( shape=( None, self.config.num_feats * (1 + self.config.delta_order), 1, ), name='feat_in', ) feat_size_input = Input( shape=(1,), dtype='int32', name='feat_size_in') label_input = Input( shape=(None,), dtype='int32', name='label_in') label_size_input = Input( shape=(1,), dtype='int32', name='label_size_in') last_layer = feat_input # convolutional layers n_filts = self.config.init_num_filt_channels last_layer = _conv_maxout_layer( last_layer, n_filts, 'conv_1', dropout=False) last_layer = MaxPooling2D( pool_size=( self.config.pool_time_width, self.config.pool_freq_width), name='conv_1_p')(last_layer) last_layer = Dropout( train_config.dropout, name='conv_1_d', seed=self._cur_weight_seed)(last_layer) self._cur_weight_seed += 1 for layer_no in range(2, 11): if layer_no == 5: n_filts *= 2 last_layer = _conv_maxout_layer( last_layer, n_filts, 'conv_{}'.format(layer_no)) last_layer = Lambda( lambda layer: K.max(layer, axis=2), output_shape=(None, n_filts), name='max_freq_into_channel', )(last_layer) # dense layers for layer_no in range(1, 4): name_prefix = 'dense_{}'.format(layer_no) dense_a = Dense( self.config.num_dense_hidden, name=name_prefix + '_a', **_layer_kwargs() ) dense_b = Dense( self.config.num_dense_hidden, name=name_prefix + '_b', **_layer_kwargs() ) td_a = TimeDistributed( dense_a, name=name_prefix + '_td_a')(last_layer) td_b = TimeDistributed( dense_b, name=name_prefix + '_td_b')(last_layer) last_layer = Maximum(name=name_prefix + '_m')([td_a, td_b]) last_layer = Dropout( train_config.dropout, name=name_prefix + '_d', seed=self._cur_weight_seed, )(last_layer) self._cur_weight_seed += 1 activation_dense = Dense( self.config.num_labels, name='dense_activation', **_layer_kwargs() ) activation_layer = TimeDistributed( activation_dense, name='dense_activation_td')(last_layer) # we take a page from the image_ocr example and treat the ctc as a # lambda layer. loss_layer = Lambda( lambda args: _ctc_loss(*args), output_shape=(1,), name='ctc_loss' )([ label_input, activation_layer, feat_size_input, label_size_input ]) self.model = Model( inputs=[ feat_input, label_input, feat_size_input, label_size_input, ], outputs=[loss_layer], )
def _construct_acoustic_model(self): # construct acoustic model # convolutional layer pattern def _conv_maxout_layer(last_layer, n_filts, name_prefix, dropout=True): conv_a = Conv2D(n_filts, self._filt_size, padding='same', name=name_prefix + '_a', **self._layer_kwargs)(last_layer) conv_b = Conv2D(n_filts, self._filt_size, padding='same', name=name_prefix + '_b', **self._layer_kwargs)(last_layer) last = Maximum(name=name_prefix + '_m')([conv_a, conv_b]) # pre-weights (i.e. post max), as per # http://jmlr.org/proceedings/papers/v28/goodfellow13.pdf if dropout: last = Dropout(self._dropout_p, name=name_prefix + '_d')(last) return last n_filts = self._initial_filts # inputs audio_input_shape = [self._input_shape[0], self._input_shape[1], 1] if self._deltas: if self._deltas.concatenate: audio_input_shape[1] *= self._deltas.num_deltas + 1 else: audio_input_shape[2] *= self._deltas.num_deltas + 1 self._audio_input = Input(shape=audio_input_shape, name='audio_in') self._audio_size_input = Input(shape=(1, ), name='audio_size_in') self._label_input = Input(shape=(None, ), name='label_in') self._label_size_input = Input(shape=(1, ), name='label_size_in') last_layer = self._audio_input # convolutional layers last_layer = _conv_maxout_layer(last_layer, n_filts, 'conv_1', dropout=False) last_layer = MaxPooling2D(pool_size=self._pool_size, name='conv_1_p')(last_layer) last_layer = Dropout(self._dropout_p, name='conv_1_d')(last_layer) for layer_no in range(2, 11): if layer_no == 5: n_filts *= 2 last_layer = _conv_maxout_layer(last_layer, n_filts, 'conv_{}'.format(layer_no)) last_layer = Lambda( lambda layer: K.max(layer, axis=2), output_shape=( self._input_shape[0], n_filts, ), name='max_freq_into_channel', )(last_layer) # dense layers for layer_no in range(1, 4): name_prefix = 'dense_{}'.format(layer_no) dense_a = Dense(self._dense_size, name=name_prefix + '_a', **self._layer_kwargs) dense_b = Dense(self._dense_size, name=name_prefix + '_b', **self._layer_kwargs) td_a = TimeDistributed(dense_a, name=name_prefix + '_td_a')(last_layer) td_b = TimeDistributed(dense_b, name=name_prefix + '_td_b')(last_layer) last_layer = Maximum(name=name_prefix + '_m')([td_a, td_b]) last_layer = Dropout(self._dropout_p, name=name_prefix + '_d')(last_layer) activation_dense = Dense(self._num_labels, name='dense_activation', **self._layer_kwargs) self._activation_layer = TimeDistributed( activation_dense, name='dense_activation_td')(last_layer) # we take a page from the image_ocr example and treat the ctc as a # lambda layer. self._loss_layer = Lambda(lambda args: _ctc_loss(*args), output_shape=(1, ), name='ctc_loss')([ self._label_input, self._activation_layer, self._audio_size_input, self._label_size_input ]) self._acoustic_model = Model( inputs=[ self._audio_input, self._label_input, self._audio_size_input, self._label_size_input, ], outputs=[self._loss_layer], )