def test_fit(self): model = keras.models.Sequential() model.add( TargetedDropout(layer=keras.layers.Dense(units=3, activation='tanh'), drop_rate=0.8, target_rate=0.2, drop_patterns=['kernel'], mode=TargetedDropout.MODE_WEIGHT, input_shape=(5, ))) model.add( TargetedDropout( layer=keras.layers.Dense(units=2, activation='softmax'), drop_rate=0.8, target_rate=0.2, drop_patterns=['kernel'], mode=TargetedDropout.MODE_UNIT, )) model.compile('adam', 'sparse_categorical_crossentropy') x, y, w = self._get_data() model.fit(x, y, epochs=50) model_path = os.path.join( tempfile.gettempdir(), 'keras_targeted_dropout_%f.h5' % np.random.random()) model.save(model_path) model = keras.models.load_model( model_path, custom_objects={'TargetedDropout': TargetedDropout}) x, y, _ = self._get_data(w) predicted = model.predict(x).argmax(axis=-1) self.assertLess(np.sum(np.not_equal(y, predicted)), x.shape[0] // 10)
def test_unit_drop_shape(self): model = keras.models.Sequential() model.add( TargetedDropout(layer=Weight(), drop_rate=0.0, target_rate=0.35, drop_patterns=['kernel'], mode=TargetedDropout.MODE_UNIT, input_shape=(1, ))) model.compile('sgd', 'mse') model_path = os.path.join( tempfile.gettempdir(), 'keras_targeted_dropout_%f.h5' % np.random.random()) model.save(model_path) model = keras.models.load_model(model_path, custom_objects={ 'Weight': Weight, 'TargetedDropout': TargetedDropout, }) dropped = model.predict(np.ones((1, 1)))[0] col = dropped.shape[-1] count = 0 for c in range(col): if np.sum(dropped[:, c]) == 0.0: count += 1 self.assertEqual(35, count) rate = np.mean(np.equal(dropped, 0.0)) self.assertGreater(rate, 0.34) self.assertLess(rate, 0.36)
def capsulenet_model(embeddings, max_sequence_length, num_words, embedding_dim, labels_index, trainable=False): K.clear_session() inp = Input(shape=(max_sequence_length,)) x = Embedding(num_words, embedding_dim, weights=[embeddings], trainable=False)(inp) x = SpatialDropout1D(rate=0.2)(x) x = Bidirectional(GRU(40, return_sequences=True))(x) #, kernel_initializer=glorot_normal(seed=1230), recurrent_initializer=orthogonal(gain=1.0, seed=1000) # x = Bidirectional(LSTM(40, return_sequences=True))(x) x = Capsule(num_capsule=10, dim_capsule=16, routings=4, share_weights=True)(x) x = Flatten()(x) # avg_pool = GlobalAveragePooling1D()(x) # x = GlobalMaxPooling1D()(x) # x = concatenate([flatt, avg_pool]) x = Dense(16, activation="relu", kernel_initializer=glorot_normal(seed=1230))(x) # x = Dropout(0.5)(x) x = TargetedDropout(drop_rate=0.5, target_rate=0.5)(x) x = BatchNormalization()(x) x = Dense(labels_index, activation="sigmoid")(x) model = Model(inputs=inp, outputs=x) model.compile(loss='binary_crossentropy', optimizer=Adam(),metrics=[f1])#Adam() model.summary() return model
def model_lstm_atten(embeddings, max_sequence_length, num_words, embedding_dim, labels_index): inp = Input(shape=(max_sequence_length, )) x = Embedding(num_words, embedding_dim, weights=[embeddings], trainable=False)(inp) x = SpatialDropout1D(0.1)(x) x = Bidirectional(CuDNNLSTM(150, return_sequences=True))(x) y = Bidirectional(CuDNNGRU(150, return_sequences=True))(x) atten_1 = Attention(max_sequence_length)(x) # skip connect atten_2 = Attention(max_sequence_length)(y) avg_pool = GlobalAveragePooling1D()(y) max_pool = GlobalMaxPooling1D()(y) conc = concatenate([atten_1, atten_2, avg_pool, max_pool]) conc = Dense(256, activation="relu")(conc) # conc = Dropout(0.5)(conc) conc = TargetedDropout(drop_rate=0.5, target_rate=0.5)(conc) outp = Dense(labels_index, activation="softmax")(conc) model = Model(inputs=inp, outputs=outp) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) model.summary() return model
def get_targeted_dropout_model(): model = keras.models.Sequential() model.add( TargetedDropout(input_shape=(28, 28, 1), drop_rate=0.5, target_rate=0.5, name='Input-Dropout')) model.add( keras.layers.Conv2D(filters=64, kernel_size=3, activation='relu', padding='same', name='Conv-1')) model.add(keras.layers.MaxPool2D(pool_size=2, name='Pool-1')) model.add(TargetedDropout(drop_rate=0.5, target_rate=0.5, name='Dropout-1')) model.add( keras.layers.Conv2D(filters=32, kernel_size=3, activation='relu', padding='same', name='Conv-2')) model.add(keras.layers.MaxPool2D(pool_size=2, name='Pool-2')) model.add(TargetedDropout(drop_rate=0.5, target_rate=0.5, name='Dropout-2')) model.add(keras.layers.Flatten(name='Flatten')) model.add(keras.layers.Dense(units=256, activation='relu', name='Dense')) model.add( TargetedDropout(drop_rate=0.5, target_rate=0.5, name='Dense-Dropout')) model.add( keras.layers.Dense(units=10, activation='softmax', name='Softmax')) model.compile( optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'], ) return model
def test_target_rate(self): model = keras.models.Sequential() model.add( TargetedDropout(input_shape=(None, None), drop_rate=0.0, target_rate=0.4)) model.compile(optimizer='adam', loss='mse') model_path = os.path.join( tempfile.gettempdir(), 'keras_targeted_dropout_%f.h5' % random.random()) model.save(model_path) model = keras.models.load_model( model_path, custom_objects={'TargetedDropout': TargetedDropout}, ) inputs = np.reshape(np.arange(20), (-1, 1, 4)) outputs = model.predict(inputs) expected = np.array([ [[0., 0., 0., 0.]], [[0., 0., 0., 0.]], [[8., 9., 10., 11.]], [[12., 13., 14., 15.]], [[16., 17., 18., 19.]], ]) self.assertTrue(np.allclose(expected, outputs), (expected, outputs)) inputs = np.array([ [[1, 5, 2, 3]], [[4, 8, 1, 5]], [[5, 2, 5, 1]], [[2, 9, 4, 3]], [[4, 7, 5, 6]], ]) outputs = model.predict(inputs) expected = np.array([ [[0., 0., 0., 0.]], [[4., 8., 0., 5.]], [[5., 0., 5., 0.]], [[0., 9., 4., 0.]], [[4., 7., 5., 6.]], ]) self.assertTrue(np.allclose(expected, outputs), (expected, outputs)) inputs = np.random.random((100, 10, 10)) outputs = model.predict(inputs) zero_num = np.sum((outputs == 0.0).astype(keras.backend.floatx())) actual_rate = zero_num / 10000.0 self.assertTrue(0.39 < actual_rate < 0.41)
def test_masked(self): model = keras.models.Sequential() model.add( keras.layers.Masking( batch_size=None, input_shape=(None, None), )) model.add(TargetedDropout( drop_rate=0.4, target_rate=0.6, )) model.compile(optimizer='adam', loss='mse') model_path = os.path.join( tempfile.gettempdir(), 'keras_targeted_dropout_%f.h5' % random.random()) model.save(model_path) model = keras.models.load_model( model_path, custom_objects={'TargetedDropout': TargetedDropout}, ) model.summary() inputs = np.array([ [[1, 5, 2, 3]], [[0, 0, 0, 0]], [[5, 2, 5, 1]], [[2, 9, 4, 3]], [[4, 7, 5, 6]], ]) outputs = model.predict(inputs) expected = np.array([ [[0., 0., 0., 0.]], [[0., 0., 0., 0.]], [[5., 0., 5., 0.]], [[0., 9., 0., 0.]], [[4., 7., 5., 6.]], ]) self.assertTrue(np.allclose(expected, outputs), (expected, outputs))
def model_HAN(embedding_matrix): inp = Input(shape=(maxlen,)) x = Embedding(nb_words, embed_size, weights=[embedding_matrix], trainable=False)(inp) x = SpatialDropout1D(0.2)(x) x = Bidirectional(LSTM(40, return_sequences=True))(x) y = Bidirectional(GRU(40, return_sequences=True))(x) atten_1 = HAN_AttLayer()(x) # skip connect atten_2 = HAN_AttLayer()(y) avg_pool = GlobalAveragePooling1D()(y) max_pool = GlobalMaxPooling1D()(y) conc = concatenate([atten_1, atten_2, avg_pool, max_pool]) conc = Dense(16, activation="relu")(conc) # conc = Dropout(0.1)(conc) conc = TargetedDropout(drop_rate=0.5, target_rate=0.5)(conc) outp = Dense(1, activation="sigmoid")(conc) model = Model(inputs=inp, outputs=outp) model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=[f1]) return model
def test_drop_rate(self): model = keras.models.Sequential() model.add( keras.layers.Lambda( function=lambda x: TargetedDropout( drop_rate=0.4, target_rate=0.4, )(x, training=True), input_shape=(None, None, None), )) model.compile(optimizer='adam', loss='mse') model_path = os.path.join( tempfile.gettempdir(), 'keras_targeted_dropout_%f.h5' % random.random()) model.save(model_path) model = keras.models.load_model( model_path, custom_objects={'TargetedDropout': TargetedDropout}, ) inputs = np.random.random((100, 10, 10, 10)) outputs = model.predict(inputs) zero_num = np.sum((outputs == 0.0).astype(keras.backend.floatx())) actual_rate = zero_num / 100000.0 self.assertTrue(0.15 < actual_rate < 0.17)