Пример #1
0
    def __init__(self, n_neurons=128, batch_size=4, user=6):
        """

        Actor Encoder class

        Args:
            n_neurons: int
                Hidden layer of LSTM

            user: int
                Num of user

        Outputs:
            enc_outputs: 3D tensor [batch, user, n_neurons]
                Whole sequence outputs


            enc_state: 1D list [tensor, tensor]

                enc_state[0]: 2D tensor [batch, n_neurons]
                    Final memory state

                enc_state[1]: 2D tensor [batch, n_neurons]
                    Final carry state
        """

        self.n_neurons = n_neurons
        self.batch_size = batch_size
        self.user = user

        # Define Recursive cell
        self.enc_rec_cell = LSTMCell(self.n_neurons)
Пример #2
0
    def __init__(self, n_neurons=128, batch_size=4, seq_length=10):
        # パラメタ設定
        self.n_neurons = n_neurons
        self.batch_size = batch_size
        self.seq_length = seq_length

        # 再帰セル定義
        self.enc_rec_cell = LSTMCell(self.n_neurons)
Пример #3
0
 def __init__(self):
     super(CharRNN, self).__init__()
     self.dense_layer = Dense(65)
     self.cells = [
         LSTMCell(units=512),
         LSTMCell(units=512),
         LSTMCell(units=512)
     ]
     self.lstm = StackedRNNCells(self.cells)
Пример #4
0
    def __init__(self):
        super(MyModel, self).__init__()

        # define all the layers and input

        self.conv2d = Conv2D(input_shape=(MyModel.img_width,
                                          MyModel.img_height, 1),
                             padding='same',
                             strides=(1, 1),
                             filters=32,
                             kernel_size=(5, 5),
                             activation='relu')
        self.max_pool = MaxPool2D(pool_size=(2, 2), strides=(2, 2))
        self.batch_norm = BatchNormalization()

        self.conv2d_1 = Conv2D(padding='same',
                               strides=(1, 1),
                               filters=64,
                               kernel_size=(5, 5),
                               activation='relu')
        self.max_pool_1 = MaxPool2D(pool_size=(2, 2), strides=(2, 2))
        self.batch_norm_1 = BatchNormalization()

        self.conv2d_2 = Conv2D(padding='same',
                               strides=(1, 1),
                               filters=128,
                               kernel_size=(3, 3),
                               activation='relu')
        self.max_pool_2 = MaxPool2D(pool_size=(1, 5), strides=(1, 1))
        self.batch_norm_2 = BatchNormalization()

        self.conv2d_3 = Conv2D(padding='same',
                               strides=(1, 1),
                               filters=128,
                               kernel_size=(3, 3),
                               activation='relu')
        self.max_pool_3 = MaxPool2D(pool_size=(1, 3), strides=(1, 1))
        self.batch_norm_3 = BatchNormalization()

        self.conv2d_4 = Conv2D(padding='same',
                               strides=(1, 1),
                               filters=256,
                               kernel_size=(3, 3),
                               activation='relu')
        self.max_pool_4 = MaxPool2D(pool_size=(1, 2), strides=(1, 1))
        self.batch_norm_4 = BatchNormalization()

        self.rnn = Bidirectional(
            RNN([
                LSTMCell(256),
                LSTMCell(256),
            ], return_sequences=True))

        self.atrous_conv2d = Conv2D(padding='same',
                                    kernel_size=(1, 1),
                                    filters=MyModel.num_chars)
Пример #5
0
    def __init__(self,
                 input_seq_len=60,
                 output_seq_len=14,
                 layers=[128],
                 n_in_features=1,
                 n_out_features=1,
                 bidirectional=False,
                 loss='mean_absolute_error'):
        self.input_seq_len = input_seq_len
        self.output_seq_len = output_seq_len
        self.n_in_features = n_in_features
        self.n_out_features = n_out_features
        n_layers = len(layers)

        ## Encoder
        encoder_inputs = Input(shape=(None, n_in_features))
        lstm_cells = [LSTMCell(hidden_dim) for hidden_dim in layers]
        # lstm_cells = LSTMCell(layers)
        if bidirectional:
            encoder = Bidirectional(RNN(lstm_cells, return_state=True))
            encoder_outputs_and_states = encoder(encoder_inputs)
            bi_encoder_states = encoder_outputs_and_states[1:]
            encoder_states = []
            for i in range(int(len(bi_encoder_states) / 2)):
                temp = concatenate([
                    bi_encoder_states[i], bi_encoder_states[2 * n_layers + i]
                ],
                                   axis=-1)
                encoder_states.append(temp)
        else:
            encoder = RNN(lstm_cells, return_state=True)
            encoder_outputs_and_states = encoder(encoder_inputs)
            encoder_states = encoder_outputs_and_states[1:]

        ## Decoder
        decoder_inputs = Input(shape=(None, n_out_features))
        if bidirectional:
            decoder_cells = [LSTMCell(hidden_dim * 2) for hidden_dim in layers]
        else:
            decoder_cells = [LSTMCell(hidden_dim) for hidden_dim in layers]

        decoder_lstm = RNN(decoder_cells,
                           return_sequences=True,
                           return_state=True)

        decoder_outputs_and_states = decoder_lstm(decoder_inputs,
                                                  initial_state=encoder_states)
        decoder_outputs = decoder_outputs_and_states[0]

        decoder_dense = Dense(n_out_features)
        decoder_outputs = decoder_dense(decoder_outputs)

        self.model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
        self.model.compile(Adam(), loss=loss)
Пример #6
0
def create_model(layers, bidirectional=False):

    n_layers = len(layers)

    ## Encoder
    encoder_inputs = Input(shape=(None, n_in_features))
    lstm_cells = [LSTMCell(hidden_dim) for hidden_dim in layers]

    if bidirectional:

        encoder = Bidirectional(RNN(lstm_cells, return_state=True))
        encoder_outputs_and_states = encoder(encoder_inputs)
        bi_encoder_states = encoder_outputs_and_states[1:]
        encoder_states = []

        for i in range(int(len(bi_encoder_states) / 2)):

            temp = []
            for j in range(2):

                temp.append(
                    concatenate([
                        bi_encoder_states[i][j],
                        bi_encoder_states[n_layers + i][j]
                    ],
                                axis=-1))

            encoder_states.append(temp)
    else:

        encoder = RNN(lstm_cells, return_state=True)
        encoder_outputs_and_states = encoder(encoder_inputs)
        encoder_states = encoder_outputs_and_states[1:]

    ## Decoder
    decoder_inputs = Input(shape=(None, n_out_features))

    if bidirectional:

        decoder_cells = [LSTMCell(hidden_dim * 2) for hidden_dim in layers]
    else:

        decoder_cells = [LSTMCell(hidden_dim) for hidden_dim in layers]

    decoder_lstm = RNN(decoder_cells, return_sequences=True, return_state=True)
    decoder_outputs_and_states = decoder_lstm(decoder_inputs,
                                              initial_state=encoder_states)
    decoder_outputs = decoder_outputs_and_states[0]
    decoder_dense = Dense(n_out_features)
    decoder_outputs = decoder_dense(decoder_outputs)

    model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
    return model
    def __init__(self,
                 encoder=None,
                 vocab_size=1,
                 embedding_size=32,
                 hidden_size=64):
        """Initialization method.

        Args:
            encoder (IntegerEncoder): An index to vocabulary encoder.
            vocab_size (int): The size of the vocabulary.
            embedding_size (int): The size of the embedding layer.
            hidden_size (int): The amount of hidden neurons.

        """

        logger.info('Overriding class: Generator -> BiLSTMGenerator.')

        # Overrides its parent class with any custom arguments if needed
        super(BiLSTMGenerator, self).__init__(name='G_bi_lstm')

        # Creates a property for holding the used encoder
        self.encoder = encoder

        # Creates an embedding layer
        self.embedding = Embedding(vocab_size,
                                   embedding_size,
                                   name='embedding')

        # Creates a forward LSTM cell
        cell_f = LSTMCell(hidden_size, name='lstm_cell_f')

        # And a orward RNN layer
        self.forward = RNN(cell_f,
                           name='forward_rnn',
                           return_sequences=True,
                           stateful=True)

        # Creates a backward LSTM cell
        cell_b = LSTMCell(hidden_size, name='lstm_cell_b')

        # And a backward RNN layer
        self.backward = RNN(cell_b,
                            name='backward_rnn',
                            return_sequences=True,
                            stateful=True,
                            go_backwards=True)

        # Creates the linear (Dense) layer
        self.linear = Dense(vocab_size, name='out')

        logger.info('Class overrided.')
    def __init__(self, units_1, units_2, dropout_1, dropout_2, output_steps,
                 output_size):
        super().__init__()
        self.output_steps = output_steps
        self.units_1 = units_1
        self.units_2 = units_2
        self.lstm_cells = [
            LSTMCell(units_1, dropout=dropout_1),
            LSTMCell(units_2, dropout=dropout_2),
        ]
        self.lstm_stack = StackedRNNCells(self.lstm_cells)

        self.lstm_rnn = RNN(self.lstm_stack, return_state=True)
        self.dense = Dense(output_size, activation="softmax")
    def __init__(self, units_1, units_2, dropout_1, dropout_2, output_steps, output_size):
        super().__init__()
        self.output_steps = output_steps
        self.units_1 = units_1
        self.units_2 = units_2
        self.dropout_1 = dropout_1
        self.dropout_2 = dropout_2

        self.lstm_cell_1 = LSTMCell(units_1, dropout=dropout_1)
        self.lstm_cell_2 = LSTMCell(units_2, dropout=dropout_2)

        self.lstm_rnn_1 = RNN(self.lstm_cell_1, return_state=True, return_sequences=True)
        self.lstm_rnn_2 = RNN(self.lstm_cell_2, return_state=True)
        self.dense = Dense(output_size, activation="softmax")
Пример #10
0
    def __init__(self, opts):

        super(LSTM, self).__init__()
        self.cell = LSTMCell(opts.lstm_units)

        # Output layer.
        self.__out_layer = Dense(opts.output_size,
                                 activation=None,
                                 use_bias=True)

        # Interface layer.
        self.__interface = Dense(opts.interface_size,
                                 activation=None,
                                 use_bias=True)
Пример #11
0
    def init_layers(self,
                    layers=(20, 20),
                    hardness=0.0,
                    learning_rate=0.01,
                    epsilon=1e-10,
                    pool=[],
                    use_meta_features=False,
                    time_scale=1000.,
                    lr_multiplier_scale=0.0,
                    warmup_lstm_update=False,
                    **kwargs):
        """Initialize layers."""
        self.choices = [
            getattr(analytical, p["class_name"] + "Optimizer")(**p["config"])
            for p in pool
        ]

        self.hardness = hardness
        self.epsilon = epsilon

        self.use_meta_features = use_meta_features
        self.time_scale = time_scale
        self.lr_multiplier_scale = lr_multiplier_scale
        self.warmup_lstm_update = warmup_lstm_update
        self.learning_rate = learning_rate

        self.recurrent = [LSTMCell(hsize, **kwargs) for hsize in layers]
        self.choice = Dense(len(pool), input_shape=(layers[-1], ))

        if self.lr_multiplier_scale > 0.0:
            self.lr_multiplier = Dense(len(pool),
                                       input_shape=(layers[-1], ),
                                       kernel_initializer='zeros',
                                       bias_initializer='zeros')
Пример #12
0
 def __init__(self, lr=1e-4, dropout_rate=0.2, units=300, beam_width=12, vocab_size=9000):
     super().__init__()
     self.embedding = tf.Variable(np.load('../data/embedding.npy'),
                                  dtype=tf.float32,
                                  name='pretrained_embedding',
                                  trainable=False)
     self.encoder = Encoder(units=units, dropout_rate=dropout_rate)
     self.attention_mechanism = BahdanauAttention(units=units)
     self.decoder_cell = AttentionWrapper(
         LSTMCell(units),
         self.attention_mechanism,
         attention_layer_size=units)
     self.projected_layer = ProjectedLayer(self.embed.embedding)
     self.sampler = tfa.seq2seq.sampler.TrainingSampler()
     self.decoder = BasicDecoder(
         self.decoder_cell,
         self.sampler,
         output_layer=self.projected_layer)
     self.beam_search = BeamSearchDecoder(
         self.decoder_cell,
         beam_width=beam_width,
         embedding_fn=lambda x: tf.nn.embedding_lookup(self.embedding, x),
         output_layer=self.projected_layer)
     self.vocab_size = vocab_size
     self.optimizer = Adam(lr)
     self.accuracy = tf.keras.metrics.Accuracy()
     self.mean = tf.keras.metrics.Mean()
     self.decay_lr = tf.optimizers.schedules.ExponentialDecay(lr, 1000, 0.95)
     self.logger = logging.getLogger('tensorflow')
     self.logger.setLevel(logging.INFO)
Пример #13
0
    def setUp(self):
        seq = Input((None, ), dtype="int32")
        x = Embedding(8000, 128)(seq)
        self.embedder = Model(seq, x)

        x = RNN(LSTMCell(128))(x)
        self.lstmer = Model(seq, x)
Пример #14
0
    def __init__(self, n_neurons=128, batch_size=4, seq_length=10):
        # パラメタ設定
        self.n_neurons = n_neurons
        self.batch_size = batch_size
        self.seq_length = seq_length

        # 地点マスク用罰則
        self.infty = 1.0E+08
        # 地点マスクbit(テンソル)
        self.mask = 0
        # サンプリングのシード
        self.seed = None

        # 初期入力値のパラメータ変数(Encoderセルの出力次元、
        # [batch_size, n_neuron])
        first_input = tf.get_variable('GO', [1, self.n_neurons])
        self.dec_first_input = tf.tile(first_input, [self.batch_size, 1])

        # Pointing機構のパラメータ変数
        self.W_ref = tf.get_variable('W_ref',
                                     [1, self.n_neurons, self.n_neurons])
        self.W_out = tf.get_variable('W_out', [self.n_neurons, self.n_neurons])
        self.v = tf.get_variable('v', [self.n_neurons])

        # 再帰セル定義
        self.dec_rec_cell = LSTMCell(self.n_neurons)
Пример #15
0
    def __init__(self, Shape, Episode_Length, Input_Dim, Output_Dim, Alpha = 0, Activation = 'tanh'):
    
        Activation_Method = self._Activation(Activation)
        Regularizer       = tf.keras.regularizers.l2(Alpha) if Alpha != 0 else None
        self.Episode_Length = Episode_Length
        
        self.State     = tf.placeholder(shape = [None, self.Episode_Length, Input_Dim], dtype = tf.float32, name = 'A_States')
        self.Action    = tf.placeholder(shape = [None, Output_Dim], dtype = tf.float32, name = 'Action_True')
        self.Advantage = tf.placeholder(shape = [None, 1], dtype = tf.float32, name = 'Advantage')
        self.Sigma     = tf.placeholder(shape = [None, Output_Dim], dtype = tf.float32, name = 'Sigma')

        Input = self.State
        cells  = []                                                    
        for Neuron_units in Shape:
            cells.append(LSTMCell(Neuron_units,activation = Activation_Method, kernel_regularizer = Regularizer))
            
        LSTM_Layers = RNN(cells, return_sequences = False)            
        Last_Layer = LSTM_Layers(Input)          
                
        self.Action_Pred = tf.layers.dense(Last_Layer, Output_Dim, activation = None, activity_regularizer = Regularizer, name = 'Action_Pred')

        self.Learning_Rate = tf.placeholder(tf.float32, shape = (), name = 'A_Learning_Rate')

        self.Optimizer = tf.train.AdamOptimizer(learning_rate = self.Learning_Rate)

        Loglik_Loss = tf.log(tf.sqrt(2 * np.pi * (self.Sigma ** 2))) + ((self.Action - self.Action_Pred) ** 2) / (2 * (self.Sigma ** 2))
        Loglik_Loss = tf.reduce_sum(Loglik_Loss, axis = 1, keepdims = True) * self.Advantage

        self.Fit = self.Optimizer.minimize(tf.reduce_mean(Loglik_Loss))
Пример #16
0
    def __init__(self, Shape, Episode_Length, Input_Dim, Output_Dim, Alpha = 0, Activation = "Sigmoid"):

        Activation_Method = self._Activation(Activation)
        Regularizer       = tf.keras.regularizers.l2(Alpha) if Alpha != 0 else None

        self.Episode_Length = Episode_Length
        self.State     = tf.placeholder(shape = [None, self.Episode_Length, Input_Dim], dtype = tf.float32, name = 'A_States')
        self.Value = tf.placeholder(shape = [None, Output_Dim], dtype = tf.float32, name = 'Value_True')
        
        Input = self.State
        cells  = []                                                  # To create a multiple layers stacked RNN   
        for Neuron_units in Shape:
            cells.append(LSTMCell(Neuron_units,activation = Activation_Method, kernel_regularizer = Regularizer))
            
        LSTM_Layers = RNN(cells, return_sequences = False)            # We abandon all the intermediate processings and only train with final state
        
        Last_Layer = LSTM_Layers(Input)
        
        self.Value_Pred = tf.layers.dense(Last_Layer, Output_Dim, activation = None, activity_regularizer = Regularizer, name = 'Value_Pred')

        self.Learning_Rate = tf.placeholder(tf.float32, shape = (), name = 'C_Learning_Rate')

        self.Optimizer = tf.train.AdamOptimizer(learning_rate = self.Learning_Rate)

        self.Fit = self.Optimizer.minimize(tf.losses.mean_squared_error(self.Value, self.Value_Pred))
Пример #17
0
class LSTM(Model):
    def __init__(self, opts):

        super(LSTM, self).__init__()
        self.cell = LSTMCell(opts.lstm_units)

        # Output layer.
        self.__out_layer = Dense(opts.output_size,
                                 activation=None,
                                 use_bias=True)

        # Interface layer.
        self.__interface = Dense(opts.interface_size,
                                 activation=None,
                                 use_bias=True)

    def __call__(self, inputs, state):

        hidden, state = self.cell(inputs, state)

        ctrl_output = {
            'output': self.__out_layer(hidden),
            'interface': self.__interface(hidden)
        }

        return ctrl_output, state

    def initialize(self, batch_size):

        return self.cell.get_initial_state(batch_size=batch_size,
                                           dtype=tf.float32)
Пример #18
0
    def __init__(self, embedding_size=32, hidden_size=64):
        """Initialization method.

        Args:
            embedding_size (int): The size of the embedding layer.
            hidden_size (int): The amount of hidden neurons.

        """

        logger.info('Overriding class: Discriminator -> LSTMDiscriminator.')

        # Overrides its parent class with any custom arguments if needed
        super(LSTMDiscriminator, self).__init__(name='D_lstm')

        # Creates an embedding layer
        self.embedding = Dense(embedding_size, name='embedding')

        # Creates a LSTM cell
        self.cell = LSTMCell(hidden_size, name='lstm_cell')

        # Creates the RNN loop itself
        self.rnn = RNN(self.cell,
                       name='rnn_layer',
                       return_sequences=True,
                       stateful=True)

        # And finally, defining the output layer
        self.out = Dense(1, name='out')

        logger.info('Class overrided.')
Пример #19
0
    def init_layers(self,
                    layers=(20, 20),
                    beta_1=0.9,
                    beta_2=0.999,
                    learning_rate=None,
                    epsilon=1e-10,
                    time_scale=2000.,
                    warmup_lstm_update=False,
                    **kwargs):
        """Initialize layers."""
        self.beta_1 = beta_1
        self.beta_2 = beta_2
        self.epsilon = epsilon
        self.time_scale = time_scale
        self.warmup_lstm_update = warmup_lstm_update

        if learning_rate is None:
            learning_rate = {
                "sgd": 0.2,
                "momentum": 0.5,
                "rmsprop": 0.005,
                "adam": 0.005,
                "powersign": 0.1,
                "addsign": 0.1
            }
        self.learning_rate = learning_rate

        self.recurrent = [LSTMCell(hsize, **kwargs) for hsize in layers]
        self.choice = Dense(6, input_shape=(layers[-1], ))
    def _build_model(self):
        # Neural Net for Deep-Q learning Model
        observation_input = Input(shape=(self.state_size, ))

        UNITS = 32
        cell_0_units = UNITS
        cell_1_units = UNITS
        cell_2_units = UNITS

        cell_0_state_in = [Input(shape=(cell_0_units, )) for _ in range(2)]
        cell_1_state_in = [Input(shape=(cell_1_units, )) for _ in range(2)]
        cell_2_state_in = [Input(shape=(cell_2_units, )) for _ in range(2)]

        output, cell_0_output, cell_0_cell_state = LSTMCell(
            cell_0_units,
            input_shape=(self.state_size, ))(inputs=observation_input,
                                             states=cell_0_state_in)
        output = Dropout(rate=0.3)(output)
        output, cell_1_output, cell_1_cell_state = LSTMCell(
            cell_1_units,
            input_shape=(self.state_size, ))(inputs=observation_input,
                                             states=cell_1_state_in)
        output = Dropout(rate=0.3)(output)
        #output, cell_2_state = LSTMCell(cell_2_units, input_shape=(self.state_size,))([observation_input, cell_2_state_in])
        #output = Dropout(rate=0.3)(output)

        value_estimator = Dense(1)(output)
        value_estimator = LeakyReLU()(value_estimator)

        advantage_estimator = Dense(self.action_size)(output)
        advantage_estimator = LeakyReLU()(advantage_estimator)

        quality_estimator = Lambda(lambda tensors: tensors[0] + tensors[
            1] - keras_backend.mean(tensors[1], axis=1, keepdims=True),
                                   output_shape=(self.action_size, ))(
                                       [value_estimator, advantage_estimator])

        model = Model(inputs=[observation_input] + cell_0_state_in +
                      cell_1_state_in + cell_2_state_in,
                      outputs=[
                          quality_estimator, cell_0_output, cell_0_cell_state,
                          cell_1_output, cell_1_cell_state
                      ])

        model.compile(loss=mean_squared_error,
                      optimizer=Adam(lr=self.learning_rate))
        return model
Пример #21
0
    def __init__(self,
                 model,
                 lstm_size=64,
                 lstm_num_layers=1,
                 tanh_constant=1.5,
                 cell_exit_extra_step=False,
                 skip_target=0.4,
                 temperature=None,
                 branch_bias=0.25,
                 entropy_reduction='sum'):
        super().__init__(model)
        self.tanh_constant = tanh_constant
        self.temperature = temperature
        self.cell_exit_extra_step = cell_exit_extra_step

        cells = [
            LSTMCell(units=lstm_size, use_bias=False)
            for _ in range(lstm_num_layers)
        ]
        self.lstm = RNN(cells, stateful=True)
        self.g_emb = tf.random.normal((1, 1, lstm_size)) * 0.1
        self.skip_targets = tf.constant([1.0 - skip_target, skip_target])

        self.max_layer_choice = 0
        self.bias_dict = {}
        for mutable in self.mutables:
            if isinstance(mutable, LayerChoice):
                if self.max_layer_choice == 0:
                    self.max_layer_choice = len(mutable)
                assert self.max_layer_choice == len(mutable), \
                        "ENAS mutator requires all layer choice have the same number of candidates."
                if 'reduce' in mutable.key:
                    bias = []
                    for choice in mutable.choices:
                        if 'conv' in str(type(choice)).lower():
                            bias.append(branch_bias)
                        else:
                            bias.append(-branch_bias)
                    self.bias_dict[mutable.key] = tf.constant(bias)

        # exposed for trainer
        self.sample_log_prob = 0
        self.sample_entropy = 0
        self.sample_skip_penalty = 0

        # internal nn layers
        self.embedding = Embedding(self.max_layer_choice + 1, lstm_size)
        self.soft = Dense(self.max_layer_choice, use_bias=False)
        self.attn_anchor = Dense(lstm_size, use_bias=False)
        self.attn_query = Dense(lstm_size, use_bias=False)
        self.v_attn = Dense(1, use_bias=False)
        assert entropy_reduction in [
            'sum', 'mean'
        ], 'Entropy reduction must be one of sum and mean.'
        self.entropy_reduction = tf.reduce_sum if entropy_reduction == 'sum' else tf.reduce_mean
        self.cross_entropy_loss = SparseCategoricalCrossentropy(
            from_logits=True, reduction=Reduction.NONE)

        self._first_sample = True
    def __init__(self, units, dropout, output_steps, output_size):
        super().__init__()
        self.output_steps = output_steps
        self.units = units
        self.lstm_cell = LSTMCell(units, dropout=dropout)

        self.lstm_rnn = RNN(self.lstm_cell, return_state=True)
        self.dense = Dense(output_size, activation="softmax")
Пример #23
0
    def __init__(self):

        self.layers = hps.DecoderRNN_layers
        self.size = hps.DecoderRNN_size

        self.lstm_list = [LSTMCell(self.size) for _ in range(self.layers)]

        self.lstm_cell = RNN(self.lstm_list,
                             return_state=True,
                             return_sequences=True)
 def __init__(self, num_class, name='rnn-classifier', **kwargs):
     super(BiLSTMClassifier, self).__init__(name=name, **kwargs)
     self.hidden_size = 128
     self.cells = [
         LSTMCell(self.hidden_size, kernel_regularizer=l2(),
                  recurrent_regularizer=l2(), dropout=0.2,
                  recurrent_dropout=0.2),
         LSTMCell(self.hidden_size, kernel_regularizer=l2(),
                  recurrent_regularizer=l2(), dropout=0.2,
                  recurrent_dropout=0.2)
     ]
     self.rnn1 = Bidirectional(
         RNN(self.cells, return_sequences=True)
     )
     self.rnn2 = Bidirectional(
         LSTM(num_class, return_sequences=True, activation='softmax', kernel_regularizer=l2(),
              recurrent_regularizer=l2(), dropout=0.2,
              recurrent_dropout=0.2), merge_mode='sum')
     self.crf = CRF(num_class)
Пример #25
0
 def __init__(self, n_units, n_acts, conv_fn="impala_cnn", **conv_kwarg):
     super(RLconv, self).__init__()
     self.n_units = n_units
     self.n_acts = n_acts
     self.cnn_model = cnn_dict[conv_fn](**conv_kwarg)
     self.lstmcell = LSTMCell(self.n_units)
     self.act_dense = Dense(self.n_acts,
                            kernel_initializer=normalized_col_init(0.01))
     self.critic_dense = Dense(1,
                               kernel_initializer=normalized_col_init(1.0))
Пример #26
0
def create_model(data, config, is_training=True):

    with tf.varaiable_scope("tacotron2", reuse=tf.AUTO_REUSE):

        inputs,input_sequences_length,target,target_sequences_length, \
            target_inputs = data['inputs'],data['input_sequences_length'],data['target'], \
                            data['target_sequences_length'],data['target_inputs']

        batch_size = tf.shape(inputs)[0]

        # Start of Encoder layers
        embedding_table = tf.Variable(
            name='embedding_table',
            shape=[VOCABULARY_SIZE, 128],
            dtype=tf.float32,
            initializer=tf.truncated_normal_initializer(stddev=0.5))

        embedding_layer = tf.nn.embedding_lookup(embedding_table, inputs)
        conv_outputs = n_layer_1d_convolution(embedding_layer,
                                              n=3,
                                              filter_width=5,
                                              channels=512,
                                              name="convolution_encoder")
        cell_fw = ZoneoutWrapper(LSTMCell(256, name="encoder_lstm_forward"),
                                 0.1,
                                 is_training=is_training)
        cell_bw = ZoneoutWrapper(LSTMCell(256, name="encoder_lstm_backward"),
                                 0.1,
                                 is_training=is_training)

        outputs = tf.keras.layers.Bidirectional(cell_fw,
                                                backward_layer=cell_bw)

        # Stacking rnn starts

        model = tf.keras.Sequential([
            Bidirectional(layer=cell_fw,
                          merge_mode="concat",
                          backward_layer=cell_bw),
        ])
        inputs = conv_outputs
Пример #27
0
 def __init__(self):
     tf.reset_default_graph()
     self.encoder_sege_file = "./tf_data_new/enc.segement"
     self.decoder_sege_file = "./tf_data_new/dec.segement"
     self.encoder_vocabulary = "./tf_data_new/enc.vocab"
     self.decoder_vocabulary = "./tf_data_new/dec.vocab"
     self.eval_enc = "./tf_data_new/eval_enc"
     self.eval_dec = "./tf_data_new/eval_dec"
     self.vocab_file = "./tf_data_new/en_de_vocabs"
     self.batch_size = 20
     self.max_batches = 15000
     self.show_epoch = 10
     self.model_path = './model_2/'
     self.transform_model = Transformer(
         embedding_size=128,
         num_layers=6,
         keep_prob_rate=0.2,
         learning_rate=0.0001,
         learning_decay_rate=0.99,
         clip_gradient=True,
         is_embedding_scale=True,
         multihead_num=8,
         max_gradient_norm=5,
         vocab_size=40020,
         max_encoder_len=200,
         max_decoder_len=200,
         share_embedding=True,
         pad_index=0,
         learning_decay_steps=500,
         dimension_feedforword=2048,
         dimension_model=512,
     )
     self.LSTMmodel = dynamicSeq2seq(encoder_cell=LSTMCell(500),
                                     decoder_cell=LSTMCell(500),
                                     encoder_vocab_size=70824,
                                     decoder_vocab_size=70833,
                                     embedding_size=128,
                                     attention=False,
                                     bidirectional=False,
                                     debug=False,
                                     time_major=True)
Пример #28
0
    def __init__(self, rnn_cell_sizes, label_dim, keep_prob):
        """
        Construct RNNColorBot
        rnn_cell_sizes: list of integers denoting the size of each LSTM cell in the RNN; rnn_cell_sizes[i] is the size of the i-th layer cell
        label_dimension: the length of the labels
        """
        super()__init__(name='')
        self.rnn_cell_sizes = rnn_cell_sizes
        self.label_dimension = label_dim
        self.keep_prob = keep_prob

        self.cells = [LSTMCell(size) for size in self.rnn_cell_sizes]
        self.relu = Dense(self.label_dimension, activation=tf.nn.relu)
Пример #29
0
def test_lstm_cell():
    n_inputs = 3
    n_units = 4
    batch_size = 1
    inputs = tx.Input(n_units=n_inputs)

    lstm0 = tx.LSTMCell(
        inputs,
        n_units,
        activation=tf.tanh,
        gate_activation=tf.sigmoid,
        forget_bias_init=tf.initializers.ones(),
    )

    lstm1 = LSTMCell(n_units,
                     activation='tanh',
                     recurrent_activation='sigmoid',
                     unit_forget_bias=True,
                     implementation=2)

    state0 = [s() for s in lstm0.previous_state]
    #  get_initial_state from keras returns either a tuple or a single
    #  state see `test_rnn_cell`, but the __call__ API requires an iterable
    state1 = lstm1.get_initial_state(inputs, batch_size=1)

    assert tx.tensor_equal(state1, state0)

    inputs.value = tf.ones([batch_size, n_inputs])
    res1 = lstm1(inputs, state0)
    res1_ = lstm1(inputs, state0)

    for r1, r2 in zip(res1, res1_):
        assert tx.tensor_equal(r1, r2)

    # the only difference is that keras kernels are fused together
    kernel = tf.concat([w.weights.value() for w in lstm0.layer_state.w],
                       axis=-1)
    w_i, _, _, _ = tf.split(kernel, 4, axis=1)
    assert tx.tensor_equal(w_i, lstm0.w[0].weights.value())

    recurrent_kernel = tf.concat([u.weights for u in lstm0.layer_state.u],
                                 axis=-1)
    bias = tf.concat([w.bias for w in lstm0.layer_state.w], axis=-1)

    assert tx.tensor_equal(tf.shape(kernel), tf.shape(lstm1.kernel))
    assert tx.tensor_equal(tf.shape(recurrent_kernel),
                           tf.shape(lstm1.recurrent_kernel))
    assert tx.tensor_equal(tf.shape(bias), tf.shape(lstm1.bias))

    lstm1.kernel = kernel
    lstm1.recurrent_kernel = recurrent_kernel
    lstm1.bias = bias

    res2 = lstm1(inputs, state0)
    for i in range(len(res1)):
        assert not tx.tensor_equal(res1[i], res2[i])
    res0 = lstm0()
    assert tx.tensor_equal(res0, res2[0])
Пример #30
0
    def add_lstm(self, inputs):

        if 'lstm_conf' in self.nn_config:
            lstm_conf = self.nn_config['lstm_conf']

            activation = None if lstm_conf['batch_norm'] else lstm_conf[
                'lstm_activation']

            return_seq = True if '1dCNN' in self.nn_config else False

            cell = LSTMCell(units=lstm_conf['lstm_units'],
                            activation=activation)

            if lstm_conf['method'] == 'dynamic_rnn':
                rnn_outputs1, states = dynamic_rnn(cell,
                                                   inputs,
                                                   dtype=tf.float32)
                lstm_outputs = tf.reshape(rnn_outputs1[:, -1, :],
                                          [-1, lstm_conf['lstm_units']])

            elif lstm_conf['method'] == 'keras_lstm_layer':
                lstm_outputs = LSTM(lstm_conf['lstm_units'],
                                    name="first_lstm",
                                    activation=activation,
                                    input_shape=(self.data_config['lookback'],
                                                 self.ins),
                                    return_sequences=return_seq)(inputs)

            else:
                rnn_layer = RNN(cell, return_sequences=return_seq)
                lstm_outputs = rnn_layer(inputs)  # [batch_size, neurons]
                if self.verbose > 0:
                    print(lstm_outputs.shape, 'before reshaping',
                          K.eval(tf.rank(lstm_outputs)))
                lstm_outputs = tf.reshape(lstm_outputs[:, :],
                                          [-1, lstm_conf['lstm_units']])
                if self.verbose > 0:
                    print(lstm_outputs.shape, 'after reshaping',
                          K.eval(tf.rank(lstm_outputs)))

            if lstm_conf['batch_norm']:
                rnn_outputs3 = BatchNormalization()(lstm_outputs)
                lstm_outputs = Activation('relu')(rnn_outputs3)

            if lstm_conf['dropout'] is not None:
                lstm_outputs = Dropout(lstm_conf['dropout'])(lstm_outputs)
        else:
            lstm_outputs = inputs

        return lstm_outputs