Esempio n. 1
0
    def __init__(self,
                 input_size,
                 output_size,
                 batch_size,
                 activation_function=None,
                 name=None,
                 init_weights=True):
        self.name = NameCreator.name_it(self, name)
        self.input_size = input_size
        self.output_size = output_size
        self.batch_size = batch_size
        self.activation_function = activation_function

        if init_weights:
            with tf.variable_scope(self.name):
                self.W = tf.Variable(tf.random_uniform(
                    [input_size, output_size], -1. / input_size**0.5,
                    1. / input_size**0.5),
                                     name='W')
                self.b = tf.Variable(tf.zeros([1, output_size]), name='b')

            self.eval_model = type(self)(input_size, output_size, 1,
                                         activation_function,
                                         self.name + '_eval', False)
            self.eval_model.W = self.W
            self.eval_model.b = self.b
Esempio n. 2
0
    def __init__(self,
                 size,
                 batch_size,
                 n_hidden_layers,
                 name=None,
                 init_weights=True,
                 trainable=True,
                 connection_mode='head'):
        self.name = NameCreator.name_it(self, name)
        self.n_hidden_layers = n_hidden_layers
        self.size = size
        self.batch_size = batch_size
        self.input_size = size
        self.output_size = size
        self.connection_mode = connection_mode

        with tf.variable_scope(self.name):
            self.layers = []
            if init_weights:
                for i in range(n_hidden_layers):
                    self.layers.append(
                        TridirectionalHighwayLayer(size,
                                                   batch_size,
                                                   trainable=trainable))

                self.eval_model = type(self)(size, 1, n_hidden_layers,
                                             self.name + '_eval', False,
                                             trainable)
                self.eval_model.layers = [
                    layer.eval_model for layer in self.layers
                ]

        self.tail = self.head = None
Esempio n. 3
0
    def __init__(self,
                 input_size,
                 output_size,
                 batch_size,
                 activation_function=tf.nn.sigmoid,
                 name=None,
                 init_weights=True):
        self.name = NameCreator.name_it(self, name)
        self.input_size = input_size
        self.output_size = output_size
        self.batch_size = batch_size
        self.activation_function = activation_function
        self._state_shape = [batch_size, output_size]

        with tf.variable_scope(self.name):
            if init_weights:
                self.iW = tf.Variable(
                    tf.truncated_normal([input_size, output_size], 0, 0.1))
                self.oW = tf.Variable(
                    tf.truncated_normal([output_size, output_size], 0, 0.1))
                self.vb = tf.Variable(tf.zeros([1, output_size]))

            self.saved_output = tf.Variable(tf.zeros(self._state_shape),
                                            trainable=False)

            self.output = self.saved_output

        if init_weights:
            self.eval_model = type(self)(input_size, output_size, 1,
                                         activation_function,
                                         self.name + '_eval', False)
            self.eval_model.iW = self.iW
            self.eval_model.oW = self.oW
            self.eval_model.vb = self.vb
Esempio n. 4
0
    def __init__(self,
                 mem_key_size,
                 mem_content_size,
                 num_cells,
                 batch_size,
                 name=None,
                 init_weights=True):
        self.name = NameCreator.name_it(self, name)
        self.mem_key_size = mem_key_size
        self.mem_content_size = mem_content_size
        self.num_cells = num_cells
        self.batch_size = batch_size
        self._content_size = [batch_size, num_cells, mem_content_size]

        with tf.variable_scope(self.name):
            if init_weights:
                # Additional key for NOP
                self.keys = tf.Variable(tf.truncated_normal(
                    [num_cells + 1, mem_key_size], -0.1, 0.1),
                                        name='keys')

            self.saved_content = tf.Variable(tf.zeros(self._content_size),
                                             dtype=tf.float32,
                                             trainable=False,
                                             name='keys')
            self.content = self.saved_content

        if init_weights:
            self.eval_model = Tape(mem_key_size, mem_content_size, num_cells,
                                   1, self.name + '_eval', False)
            self.eval_model.keys = self.keys
Esempio n. 5
0
    def __init__(self,
                 input_size,
                 output_size,
                 batch_size,
                 controller,
                 mem_key_size,
                 mem_content_size,
                 num_cells,
                 activation_function=None,
                 name=None,
                 init_weights=True):

        self.name = NameCreator.name_it(self, name)
        self.input_size = input_size
        self.output_size = output_size
        self.batch_size = batch_size
        self.mem_key_size = mem_key_size
        self.mem_content_size = mem_content_size
        self.num_cells = num_cells
        self.activation_function = activation_function
        self.extended_input_size = input_size + mem_content_size
        self.extended_output_size = output_size + mem_content_size + 2 * mem_key_size + 3
        self._read_result_shape = (batch_size, mem_content_size)

        if init_weights:
            self.tape = Tape(mem_key_size, mem_content_size, num_cells,
                             batch_size, self.name + '_Tape')
            with tf.variable_scope(self.name):
                self.input_adapter = FeedForward(self.extended_input_size,
                                                 controller.input_size,
                                                 batch_size)
                self.output_adapter = FeedForward(controller.output_size,
                                                  self.extended_output_size,
                                                  batch_size)
                self.controller = controller
                self.extended_controller = ConnectLayers(
                    [self.input_adapter, controller, self.output_adapter])

        with tf.variable_scope(self.name):
            self.saved_read_result = tf.Variable(
                tf.zeros(self._read_result_shape))
            self.read_result = self.saved_read_result

        if batch_size > 1:
            self.eval_model = NTM(input_size,
                                  output_size,
                                  1,
                                  controller,
                                  mem_key_size,
                                  mem_content_size,
                                  num_cells,
                                  activation_function,
                                  self.name + '_eval',
                                  init_weights=False)
            self.eval_model.tape = self.tape.eval_model
            self.eval_model.extended_controller = self.extended_controller.eval_model
Esempio n. 6
0
    def __init__(self,
                 input_size,
                 output_size,
                 batch_size,
                 name=None,
                 init_weights=True):
        self.name = NameCreator.name_it(self, name)
        self.input_size = input_size
        self.output_size = output_size
        self.batch_size = batch_size
        self._state_shape = [batch_size, output_size]

        with tf.variable_scope(self.name):
            if init_weights:
                self.iW_g = tf.Variable(tf.truncated_normal(
                    [input_size, 2 * output_size], 0, 0.1),
                                        name='iW_g')
                self.oW_g = tf.Variable(tf.truncated_normal(
                    [output_size, 2 * output_size], 0, 0.1),
                                        name='oW_g')

                self.iW = tf.Variable(tf.truncated_normal(
                    [input_size, output_size], 0, 0.1),
                                      name='iW')
                self.oW = tf.Variable(tf.truncated_normal(
                    [output_size, output_size], 0, 0.1),
                                      name='oW')

                self.b_g = tf.Variable(2 * tf.ones([1, 2 * output_size]),
                                       name='b_g')
                self.b = tf.Variable(tf.zeros([1, output_size]), name='b')

            self.saved_output = tf.Variable(tf.zeros(self._state_shape),
                                            trainable=False,
                                            name='saved_output')
            self.output = self.saved_output

        if init_weights:
            self.eval_model = type(self)(input_size,
                                         output_size,
                                         1,
                                         name=self.name + '_eval',
                                         init_weights=False)
            self.eval_model.iW_g = self.iW_g
            self.eval_model.oW_g = self.oW_g
            self.eval_model.iW = self.iW
            self.eval_model.oW = self.oW
            self.eval_model.b_g = self.b_g
            self.eval_model.b = self.b
Esempio n. 7
0
    def __init__(self,
                 input_size,
                 output_size,
                 batch_size,
                 name=None,
                 init_weights=True):
        self.name = NameCreator.name_it(self, name)
        self.input_size = input_size
        self.output_size = output_size
        self.batch_size = batch_size
        self._state_shape = [batch_size, output_size]

        with tf.variable_scope(self.name):
            self.saved_output = tf.Variable(tf.zeros(self._state_shape),
                                            trainable=False,
                                            name='saved_output')
            self.saved_state = tf.Variable(tf.zeros(self._state_shape),
                                           trainable=False,
                                           name='saved_input')

        self.output = self.saved_output
        self.state = self.saved_state

        if init_weights:
            with tf.variable_scope(self.name):
                b = np.column_stack((-2 * np.ones(
                    (1, output_size)), 2 * np.ones(
                        (1, 2 * output_size)), np.zeros((1, output_size))))
                b = np.array(b, dtype=np.float32)
                self.iW = tf.Variable(tf.truncated_normal(
                    [input_size, 4 * output_size], 0, 0.1),
                                      name='iW')
                self.oW = tf.Variable(tf.truncated_normal(
                    [output_size, 4 * output_size], 0, 0.1),
                                      name='oW')
                self.b = tf.Variable(b)

            self.eval_model = type(self)(input_size,
                                         output_size,
                                         1,
                                         name=self.name + '_eval',
                                         init_weights=False)
            self.eval_model.iW = self.iW
            self.eval_model.oW = self.oW
            self.eval_model.b = self.b
Esempio n. 8
0
    def __init__(self, layer, name=None, init_weights=True):
        self.name = NameCreator.name_it(self, name)
        self.input_size = layer.input_size
        self.output_size = layer.output_size
        self.batch_size = layer.batch_size
        self.layer = layer
        if init_weights:
            with tf.variable_scope(self.name):
                self.W = tf.Variable(tf.random_uniform(
                    [self.input_size, self.output_size],
                    -1. / self.input_size**0.5, 1. / self.input_size**0.5),
                                     name='W')
                self.b = tf.Variable(3 * tf.ones([1, self.output_size]),
                                     name='b')

            self.eval_model = type(self)(layer.eval_model, self.name + '_eval',
                                         False)
            self.eval_model.W = self.W
            self.eval_model.b = self.b
Esempio n. 9
0
    def __init__(self,
                 size,
                 batch_size,
                 name=None,
                 init_weights=True,
                 trainable=True):
        self.name = NameCreator.name_it(self, name)
        self.size = size
        self.batch_size = batch_size
        self.input_size = size
        self.output_size = size
        self._state_shape = [batch_size, size]

        with tf.variable_scope(self.name):
            if init_weights:
                W_g1 = np.random.uniform(-0.1, 0.1, (2 * size, 3 * size))
                W_g2 = np.random.uniform(-0.1, 0.1, (2 * size, 3 * size))

                W_o1 = np.array((np.ones((size, size)) - np.eye(size)) *
                                np.random.uniform(-0.1, 0.1,
                                                  (size, size)) + np.eye(size),
                                dtype=np.float32)
                W_o2 = np.array((np.ones((size, size)) - np.eye(size)) *
                                np.random.uniform(-0.1, 0.1,
                                                  (size, size)) + np.eye(size),
                                dtype=np.float32)

                W_i1 = np.array(np.random.uniform(-0.1, 0.1, (size, size)),
                                dtype=np.float32)
                W_i2 = np.array(np.random.uniform(-0.1, 0.1, (size, size)),
                                dtype=np.float32)

                self.b_g1 = tf.Variable(2 * tf.ones((1, 3 * size)), trainable)
                self.b_g2 = tf.Variable(10 * tf.ones((1, 3 * size)), trainable)
                self.W_g1 = tf.Variable(W_g1, trainable, dtype=tf.float32)
                self.W_g2 = tf.Variable(W_g2, trainable, dtype=tf.float32)
                self.b_o1 = tf.Variable(tf.zeros((1, size)), trainable)
                self.b_o2 = tf.Variable(tf.zeros((1, size)), trainable)
                self.W_o1 = tf.Variable(W_o1, trainable)
                self.W_o2 = tf.Variable(W_o2, trainable)
                self.b_i1 = tf.Variable(tf.zeros((1, size)), trainable)
                self.b_i2 = tf.Variable(tf.zeros((1, size)), trainable)
                self.W_i1 = tf.Variable(W_i1, trainable)
                self.W_i2 = tf.Variable(W_i2, trainable)

                self.eval_model = type(self)(size, 1, self.name + '_eval',
                                             False, trainable)

                self.eval_model.b_g1 = self.b_g1
                self.eval_model.b_g2 = self.b_g2
                self.eval_model.W_g1 = self.W_g1
                self.eval_model.W_g2 = self.W_g2
                self.eval_model.b_o1 = self.b_o1
                self.eval_model.b_o2 = self.b_o2
                self.eval_model.W_o1 = self.W_o1
                self.eval_model.W_o2 = self.W_o2
                self.eval_model.b_i1 = self.b_i1
                self.eval_model.b_i2 = self.b_i2
                self.eval_model.W_i1 = self.W_i1
                self.eval_model.W_i2 = self.W_i2

        self.saved_output = tf.Variable(
            tf.zeros([batch_size, size], name="default_output"), False)
        self.output = self.saved_output

        self.feed_direction = "f"