def _build_clockwork_block(self, units, period, input_shape): output_shape = input_shape[:-1] + (units,) pool = MaxPooling1D(1, period) rnn = self.rnn_dtype(units=units, **self.rnn_kwargs) unpool = UpSampling1D(period) crop = Lambda(lambda x: self._crop(x[0], x[1]), output_shape=output_shape[1:]) delay = Lambda(lambda x: self._delay(x), output_shape=output_shape[1:]) concat = Concatenate() block = (pool, rnn, unpool, crop, delay, concat) pool.build(input_shape) pool_output_shape = pool.compute_output_shape(input_shape) rnn.build(pool_output_shape) self._trainable_weights.extend(rnn.trainable_weights) rnn_output_shape = rnn.compute_output_shape(pool_output_shape) unpool.build(rnn_output_shape) crop.build([unpool.compute_output_shape(rnn_output_shape), ()]) delay.build(output_shape) concat.build([input_shape, output_shape]) return block, output_shape, \ concat.compute_output_shape([input_shape, output_shape])
alpha = K.softmax(e) # (batch_size, input_length) # eqn 5 c = K.batch_dot(h, alpha, axes=1) # (batch_size, encoding_dim) recurrence_result = K.expand_dims( K.concatenate([c, y_i], axis=1), dim=1) # (batch_size, 1, 2 * encoding_dim) expanded_h = Input(shape=(1, 2 * encoding_dim), name='expanded_h') gru = Sequential([ GRU(output_dim, return_sequences=False, input_shape=(1, 2 * encoding_dim)) ]) model = Model(input=[expanded_h], output=[gru(expanded_h)]) # (batch_size, 1, output_dim) return model(recurrence_result) output, _ = theano.scan(recurrence, sequences=K.permute_dimensions(y, [1, 0, 2]), non_sequences=h) layer = Lambda(lambda encoded_state: output, output_shape=(batch_size, output_dim)) layer.build((input_length, encoding_dim)) print(K.eval(layer(h)))
# eqn 6 alpha = K.softmax(e) # (batch_size, input_length) # eqn 5 c = K.batch_dot(h, alpha, axes=1) # (batch_size, encoding_dim) recurrence_result = K.expand_dims(K.concatenate( [c, y_i], axis=1), dim=1) # (batch_size, 1, 2 * encoding_dim) expanded_h = Input(shape=(1, 2 * encoding_dim), name='expanded_h') gru = Sequential([ GRU(output_dim, return_sequences=False, input_shape=(1, 2 * encoding_dim)) ]) model = Model(input=[expanded_h], output=[gru(expanded_h)]) # (batch_size, 1, output_dim) return model(recurrence_result) output, _ = theano.scan(recurrence, sequences=K.permute_dimensions(y, [1, 0, 2]), non_sequences=h) layer = Lambda(lambda encoded_state: output, output_shape=(batch_size, output_dim)) layer.build((input_length, encoding_dim)) print(K.eval(layer(h)))
def window(w, n): w = w + list(reversed(w)) w = numpy.asarray(w) layer = Lambda(lambda x: x * w) layer.build((n, )) return layer
class ResidualBlock(Layer): def __init__(self, dilation_rate, nb_filters, kernel_size, padding, activation='relu', dropout_rate=0, kernel_initializer='he_normal', use_batch_norm=False, last_block=True, **kwargs): # type: (int, int, int, str, str, float, str, bool, dict) -> None """Defines the residual block for the WaveNet TCN Args: x: The previous layer in the model training: boolean indicating whether the layer should behave in training mode or in inference mode dilation_rate: The dilation power of 2 we are using for this residual block nb_filters: The number of convolutional filters to use in this block kernel_size: The size of the convolutional kernel padding: The padding used in the convolutional layers, 'same' or 'causal'. activation: The final activation used in o = Activation(x + F(x)) dropout_rate: Float between 0 and 1. Fraction of the input units to drop. kernel_initializer: Initializer for the kernel weights matrix (Conv1D). use_batch_norm: Whether to use batch normalization in the residual layers or not. kwargs: Any initializers for Layer class. """ self.dilation_rate = dilation_rate self.nb_filters = nb_filters self.kernel_size = kernel_size self.padding = padding self.activation = activation self.dropout_rate = dropout_rate self.use_batch_norm = use_batch_norm self.kernel_initializer = kernel_initializer self.last_block = last_block super(ResidualBlock, self).__init__(**kwargs) def _add_and_activate_layer(self, layer): """Helper function for building layer Args: layer: Appends layer to internal layer list and builds it based on the current output shape of ResidualBlock. Updates current output shape. """ self.residual_layers.append(layer) self.residual_layers[-1].build(self.res_output_shape) self.res_output_shape = self.residual_layers[-1].compute_output_shape( self.res_output_shape) def build(self, input_shape): with K.name_scope( self.name ): # name scope used to make sure weights get unique names self.residual_layers = list() self.res_output_shape = input_shape for k in range(2): name = 'conv1D_{}'.format(k) with K.name_scope( name ): # name scope used to make sure weights get unique names self._add_and_activate_layer( Conv1D(filters=self.nb_filters, kernel_size=self.kernel_size, dilation_rate=self.dilation_rate, padding=self.padding, name=name, kernel_initializer=self.kernel_initializer)) if self.use_batch_norm: # TODO should be WeightNorm here, but using batchNorm instead self._add_and_activate_layer(BatchNormalization()) self._add_and_activate_layer(Activation('relu')) self._add_and_activate_layer( SpatialDropout1D(rate=self.dropout_rate)) if not self.last_block: # 1x1 conv to match the shapes (channel dimension). name = 'conv1D_{}'.format(k + 1) with K.name_scope(name): # make and build this layer separately because it directly uses input_shape self.shape_match_conv = Conv1D( filters=self.nb_filters, kernel_size=1, padding='same', name=name, kernel_initializer=self.kernel_initializer) else: self.shape_match_conv = Lambda(lambda x: x, name='identity') self.shape_match_conv.build(input_shape) self.res_output_shape = self.shape_match_conv.compute_output_shape( input_shape) self.final_activation = Activation(self.activation) self.final_activation.build( self.res_output_shape) # probably isn't necessary # this is done to force keras to add the layers in the list to self._layers for layer in self.residual_layers: self.__setattr__(layer.name, layer) super(ResidualBlock, self).build( input_shape) # done to make sure self.built is set True def call(self, inputs, training=None): """ Returns: A tuple where the first element is the residual model tensor, and the second is the skip connection tensor. """ x = inputs for layer in self.residual_layers: if isinstance(layer, SpatialDropout1D): x = layer(x, training=training) else: x = layer(x) x2 = self.shape_match_conv(inputs) res_x = add([x2, x]) return [self.final_activation(res_x), x] def compute_output_shape(self, input_shape): return [self.res_output_shape, self.res_output_shape]
dim=1) # (batch_size, 1, 2 * encoding_dim) return model(input_per_timesteps) output, _ = theano.scan(recurrence, sequences=K.permute_dimensions(y, [1, 0, 2]), non_sequences=h) return K.permute_dimensions(output, [1, 0, 2]) x = Input(shape=(input_length,), dtype='int32', name='x') y = Input(shape=(output_length,), dtype='int32', name='y') embed = Embedding(voc_size, hidden_dim, input_length=input_length) decode = Lambda(decode_func, output_shape=(batch_size, output_dim), arguments={'y': embed(y)}) decode.build((input_length, hidden_dim)) gru = GRU(hidden_dim, input_shape=(input_length, hidden_dim), unroll=True, return_sequences=True, consume_less='gpu') encoder = Sequential([embed]) for _ in range(depth): encoder.add(gru) hidden_state = encoder(x) def model(train): def decoder_unit(): return GRU(hidden_dim, return_sequences=True,
def mse(y_true, y_pred): cost = cost + K.mean(K.square(y_pred - y_true), axis=-1) # Autoencoder (or) Denoising autoencoder with the comb denoising function # Decoder begins here with the upsampling -- connecting to the last convolutional layer output of the noisy encoder # NOTE: There is no scaling or shifting in the decoder implementation according to the paper. There is only batch normalization # Getting the output from the final dense layer (dense_3) of the noisy encoder and passing it to the decoder module lam = Lambda(lambda x: x, output_shape=(ns_enc.get_layer('dense_3').output_shape[1], w, h, 1)) lam.build((ns_enc.get_layer('dense_3').output_shape[1], w, h, 1)) x = K.reshape( ns_enc.get_layer('dense_3').output, (ns_enc.get_layer('dense_3').output_shape[1], w, h, 1)) out = lam(x) cde1 = Conv2DTranspose(32, 3, 3, name="conv_de_1")(out) btde1 = BatchNormalization(center=False, scale=False, name="btde1")(cde1) a1 = LeakyReLU(name="actde1")(btde1) #denoising function comb call if denoising autoencoder cde2 = Conv2DTranspose(64, 3, 3, name="conv_de_2")(a1) btde2 = BatchNormalization(center=False, scale=False, name="btde2")(cde2) a2 = LeakyReLU(name="actde2")(btde2) #denoising function comb call if denoising autoencoder cde3 = Conv2DTranspose(64, 3, 3, name="conv_de_3")(a2) btde3 = BatchNormalization(center=False, scale=False, name="btde3")(cde3)
class ClockworkRNN(Layer): """Clockwork RNN ([Koutnik et al., 2014](https://arxiv.org/abs/1402.3511)). Constructs a CW-RNN from RNNs of a given type. # Arguments periods: List of positive integers. The periods of each internal RNN. units_per_period: Positive integer or list of positive integers. Number of units for each internal RNN. If list, it must have the same length as `periods`. input_shape: Shape of the input data. output_units: Positive integer. Dimensionality of the output space. output_activation: String or callable. Activation function to use. If you don't specify anything, no activation is applied (i.e., "linear" activation: `a(x) = x`). return_sequences: Boolean (default False). Whether to return the last output in the output sequence, or the full sequence. sort_ascending: Boolean (default False). Whether to sort the periods in ascending or descending order (default, as in the original paper). include_top: Whether to include the fully-connected layer at the top of the network. dense_kwargs: Dictionary. Optional arguments for the trailing Dense unit (`activation` and `units` keys will be ignored). rnn_dtype: The type of RNN to use as clockwork layer. Can be a string ("SimpleRNN", "GRU", "LSTM", "CuDNNGRU", "CuDNNLSTM") or any RNN subclass. rnn_kwargs: Dictionary. Optional arguments for the internal RNNs (`return_sequences` and `return_state` will be ignored). """ def __init__(self, periods, units_per_period, output_units, output_activtion='linear', return_sequences=False, sort_ascending=False, include_top=True, dense_kwargs=None, rnn_dtype="SimpleRNN", rnn_kwargs=None, **kwargs): if type(rnn_dtype) is str: self.rnn_dtype = getattr(layers, rnn_dtype) else: self.rnn_dtype = rnn_dtype ClockworkRNN.__name__ = "Clockwork" + self.rnn_dtype.__name__ super(ClockworkRNN, self).__init__(**kwargs) if type(units_per_period) is list: self.units_per_period = units_per_period else: self.units_per_period = [units_per_period] * len(periods) self.periods = periods self.rnn_kwargs = rnn_kwargs or {} self.rnn_kwargs['return_sequences'] = True self.rnn_kwargs['return_state'] = False self.rnn_kwargs.pop("units", True) self.dense_kwargs = dense_kwargs or {} self.dense_kwargs['activation'] = output_activtion self.dense_kwargs['units'] = output_units self.include_top = include_top self.return_sequences = return_sequences self.sort_ascending = sort_ascending self.blocks = [] def build(self, input_shape): last_shape = input_shape output_shapes = [] for period, units in sorted(zip(self.periods, self.units_per_period), reverse=not self.sort_ascending): block, output_shape, last_shape = self._build_clockwork_block( units, period, last_shape) output_shapes.append(output_shape) self.blocks.append(block) self.concat_all = Concatenate() self.concat_all.build(output_shapes) last_shape = self.concat_all.compute_output_shape(output_shapes) if not self.return_sequences: self.lambda_last = Lambda(lambda x: x[:, -1]) self.lambda_last.build(last_shape) last_shape = self.lambda_last.compute_output_shape(last_shape) if self.include_top: if self.return_sequences: self.dense = TimeDistributed(Dense(**self.dense_kwargs)) else: self.dense = Dense(**self.dense_kwargs) self.dense.build(last_shape) self._trainable_weights.extend(self.dense.trainable_weights) last_shape = self.dense.compute_output_shape(last_shape) super(ClockworkRNN, self).build(input_shape) def call(self, x): rnns = [] to_next_block = x for block in self.blocks: to_dense, to_next_block = self._call_clockwork_block( to_next_block, *block) rnns.append(to_dense) out = self.concat_all(rnns) if not self.return_sequences: out = self.lambda_last(out) if self.include_top: out = self.dense(out) return out def compute_output_shape(self, input_shape): if self.include_top: out_dim = self.dense_kwargs['units'] else: out_dim = np.sum(self.units_per_period) if self.return_sequences: return input_shape[:-1] + (out_dim,) else: return input_shape[:-2] + (out_dim,) def _delay(self, x): return K.temporal_padding(x, (1, 0))[:, :-1] def _crop(self, x, timesteps): return x[:, :K.cast(timesteps, "int32")] def _build_clockwork_block(self, units, period, input_shape): output_shape = input_shape[:-1] + (units,) pool = MaxPooling1D(1, period) rnn = self.rnn_dtype(units=units, **self.rnn_kwargs) unpool = UpSampling1D(period) crop = Lambda(lambda x: self._crop(x[0], x[1]), output_shape=output_shape[1:]) delay = Lambda(lambda x: self._delay(x), output_shape=output_shape[1:]) concat = Concatenate() block = (pool, rnn, unpool, crop, delay, concat) pool.build(input_shape) pool_output_shape = pool.compute_output_shape(input_shape) rnn.build(pool_output_shape) self._trainable_weights.extend(rnn.trainable_weights) rnn_output_shape = rnn.compute_output_shape(pool_output_shape) unpool.build(rnn_output_shape) crop.build([unpool.compute_output_shape(rnn_output_shape), ()]) delay.build(output_shape) concat.build([input_shape, output_shape]) return block, output_shape, \ concat.compute_output_shape([input_shape, output_shape]) def _call_clockwork_block(self, x, pool, rnn, unpool, crop, delay, concat): pooled = pool(x) rnn_out = rnn(pooled) unpooled = unpool(rnn_out) to_dense = crop([unpooled, K.shape(x)[1]]) delayed = delay(to_dense) to_next_block = concat([x, delayed]) return to_dense, to_next_block def get_config(self): config = super(ClockworkRNN, self).get_config() config['units_per_period'] = self.units_per_period config['periods'] = self.periods config['rnn_dtype'] = self.rnn_dtype.__name__ config['rnn_kwargs'] = self.rnn_kwargs config['dense_kwargs'] = self.dense_kwargs config['include_top'] = self.include_top config['return_sequences'] = self.return_sequences config['sort_ascending'] = self.sort_ascending return config
def get_vnet_gen_model(): inputs = Input((1, num_slices, img_width, img_height)) # output 128 x 128 x 16 x 16 conv1 = Conv3D(16, 5, 5, 5, activation='relu', border_mode='same')(inputs) # output 128 x 128 x 16 x 16 m0 = merge([inputs, inputs], mode='concat', concat_axis=1) for i in range(14): m0 = merge([inputs, m0], mode='concat', concat_axis=1) m1 = merge([m0, conv1], output_shape=(None, 16, 16, 128, 128), mode='sum') lam = Lambda(lambda x: x, output_shape=(16, 16, 128, 128)) lam.build((16, 16, 128, 128)) out = lam(m1) # output 64 x 64 x 8 x 16 pool1 = MaxPooling3D(pool_size=(2, 2, 2))(out) # output 64 x 64 x 8 x 32 conv2 = Conv3D(32, 5, 5, 5, activation='relu', border_mode='same')(pool1) # output 64 x 64 x 8 x 32 conv2 = Conv3D(32, 5, 5, 5, activation='relu', border_mode='same')(conv2) # output 64 x 64 x 8 x 32 m0 = merge([pool1, pool1], mode='concat', concat_axis=1) #print m0.shape #for i in range(14): # m0 = merge([pool1, m0], mode='concat', concat_axis=1) # print m0.shape # output 64 x 64 x 8 x 32 m2 = merge([m0, conv2], mode='sum') #lam = Lambda(lambda x: x, output_shape=(16, 16, 128, 128)) #lam.build((16, 16, 128, 128)) #out = lam(m2) # output 64 x 64 x 8 x 64 #m2 = merge([pool1, conv2], mode='sum') # output 32 x 32 x 4 x 32 pool2 = MaxPooling3D(pool_size=(2, 2, 2))(m2) # output 32 x 32 x 4 x 64 conv3 = Conv3D(64, 5, 5, 5, activation='relu', border_mode='same')(pool2) # output 32 x 32 x 4 x 64 conv3 = Conv3D(64, 5, 5, 5, activation='relu', border_mode='same')(conv3) # output 32 x 32 x 4 x 64 conv3 = Conv3D(64, 5, 5, 5, activation='relu', border_mode='same')(conv3) # output 32 x 32 x 4 x 64 m0 = merge([pool2, pool2], mode='concat', concat_axis=1) # output 32 x 32 x 4 x 64 m3 = merge([m0, conv3], mode='sum') # output 16 x 16 x 2 x 64 pool3 = MaxPooling3D(pool_size=(2, 2, 2))(m3) conv4 = Conv3D(128, 5, 5, 5, activation='relu', border_mode='same')(pool3) conv4 = Conv3D(128, 5, 5, 5, activation='relu', border_mode='same')(conv4) # output 16 x 16 x 2 x 128 conv4 = Conv3D(128, 5, 5, 5, activation='relu', border_mode='same')(conv4) # output 16 x 16 x 2 x 128 m0 = merge([pool3, pool3], mode='concat', concat_axis=1) # output 16 x 16 x 2 x 128 m4 = merge([m0, conv4], mode='sum') # output 8 x 8 x 1 x 128 pool4 = MaxPooling3D(pool_size=(2, 2, 2))(m4) conv5 = Conv3D(256, 5, 5, 5, activation='relu', border_mode='same')(pool4) conv5 = Conv3D(256, 5, 5, 5, activation='relu', border_mode='same')(conv5) # output 8 x 8 x 1 x 256 conv5 = Conv3D(256, 5, 5, 5, activation='relu', border_mode='same')(conv5) # output 8 x 8 x 1 x 256 m0 = merge([pool4, pool4], mode='concat', concat_axis=1) # output 8 x 8 x 1 x 256 m5 = merge([m0, conv5], mode='sum') # output 16 x 16 x 2 x 256 up5 = UpSampling3D(size=(2, 2, 2))(m5) # output 16 x 16 x 2 x 384 m0 = merge([m4, up5], mode='concat', concat_axis=1) conv6 = Conv3D(256, 5, 5, 5, activation='relu', border_mode='same')(m0) conv6 = Conv3D(256, 5, 5, 5, activation='relu', border_mode='same')(conv6) # output 16 x 16 x 2 x 256 conv6 = Conv3D(256, 5, 5, 5, activation='relu', border_mode='same')(conv6) # output 16 x 16 x 2 x 256 m6 = merge([up5, conv6], mode='sum') # output 32 x 32 x 4 x 256 up6 = UpSampling3D(size=(2, 2, 2))(m6) conv7 = Conv3D(128, 5, 5, 5, activation='relu', border_mode='same')(up6) # output 32 x 32 x 4 x 192 m0 = merge([m3, conv7], mode='concat', concat_axis=1) conv7 = Conv3D(128, 5, 5, 5, activation='relu', border_mode='same')(m0) # output 32 x 32 x 4 x 128 conv7 = Conv3D(128, 5, 5, 5, activation='relu', border_mode='same')(conv7) # output 32 x 32 x 4 x 256 m0 = merge([conv7, conv7], mode='concat', concat_axis=1) # output 32 x 32 x 4 x 256 m7 = merge([up6, m0], mode='sum') # output 64 x 64 x 8 x 256 up7 = UpSampling3D(size=(2, 2, 2))(m7) conv8 = Conv3D(64, 5, 5, 5, activation='relu', border_mode='same')(up7) # output 64 x 64 x 8x 96 m0 = merge([m2, conv8], mode='concat', concat_axis=1) # output 64 x 64 x 8 x 64 conv8 = Conv3D(64, 5, 5, 5, activation='relu', border_mode='same')(m0) # output 64 x 64 x 8 x 128 m0 = merge([conv8, conv8], mode='concat', concat_axis=1) # output 64 x 64 x 8 x 256 m0 = merge([m0, m0], mode='concat', concat_axis=1) # output 64 x 64 x 8 x 256 m8 = merge([up7, m0], mode='sum') # output 128 x 128 x 16 x 256 up8 = UpSampling3D(size=(2, 2, 2))(m8) # output 128 x 128 x 16 x 32 conv9 = Conv3D(32, 5, 5, 5, activation='relu', border_mode='same')(up8) # output 128 x 128 x 16 x 48 m0 = merge([m1, conv9], mode='concat', concat_axis=1) # output 128 x 128 x 16 x 96 m0 = merge([m0, m0], mode='concat', concat_axis=1) # output 128 x 128 x 16 x 192 m0 = merge([m0, m0], mode='concat', concat_axis=1) # output 128 x 128 x 16 x 64 m1 = merge([conv9, conv9], mode='concat', concat_axis=1) m2 = merge([m0, m1], mode='concat', concat_axis=1) # output 128 x 128 x 16 x 256 m9 = merge([up8, m2], mode='sum') conv10 = Conv3D(1, 1, 1, 1, activation='sigmoid', border_mode='same')(m9) model = Model(input=inputs, output=conv10) model.compile(optimizer=Adam(lr=1.0e-5), loss=dice_coef_loss, metrics=[dice_coef]) model.summary() return model