def __init__( self, learning_rate, num_layers, size, size_layer, output_size, kernel_size=3, n_attn_heads=16, dropout=0.9, ): self.X = tf.placeholder(tf.float32, (None, None, size)) self.Y = tf.placeholder(tf.float32, (None, output_size)) encoder_embedded = tf.layers.dense(self.X, size_layer) encoder_embedded += position_encoding(encoder_embedded) e = tf.identity(encoder_embedded) for i in range(num_layers): dilation_rate = 2**i pad_sz = (kernel_size - 1) * dilation_rate with tf.variable_scope('block_%d' % i): encoder_embedded += cnn_block(encoder_embedded, dilation_rate, pad_sz, size_layer, kernel_size) encoder_output, output_memory = encoder_embedded, encoder_embedded + e g = tf.identity(encoder_embedded) for i in range(num_layers): dilation_rate = 2**i pad_sz = (kernel_size - 1) * dilation_rate with tf.variable_scope('decode_%d' % i): attn_res = h = cnn_block(encoder_embedded, dilation_rate, pad_sz, size_layer, kernel_size) C = [] for j in range(n_attn_heads): h_ = tf.layers.dense(h, size_layer // n_attn_heads) g_ = tf.layers.dense(g, size_layer // n_attn_heads) zu_ = tf.layers.dense(encoder_output, size_layer // n_attn_heads) ze_ = tf.layers.dense(output_memory, size_layer // n_attn_heads) d = tf.layers.dense(h_, size_layer // n_attn_heads) + g_ dz = tf.matmul(d, tf.transpose(zu_, [0, 2, 1])) a = tf.nn.softmax(dz) c_ = tf.matmul(a, ze_) C.append(c_) c = tf.concat(C, 2) h = tf.layers.dense(attn_res + c, size_layer) h = tf.nn.dropout(h, keep_prob=dropout) encoder_embedded += h encoder_embedded = tf.sigmoid(encoder_embedded[-1]) self.logits = tf.layers.dense(encoder_embedded, output_size) self.cost = tf.reduce_mean(tf.square(self.Y - self.logits)) self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize( self.cost)
def _build(self, inputs, prev_state): """Connects the DNC core into the graph. Args: inputs: Tensor input. prev_state: A `DNCState` tuple containing the fields `access_output`, `access_state` and `controller_state`. `access_state` is a 3-D Tensor of shape `[batch_size, num_reads, word_size]` containing read words. `access_state` is a tuple of the access module's state, and `controller_state` is a tuple of controller module's state. Returns: A tuple `(output, next_state)` where `output` is a tensor and `next_state` is a `DNCState` tuple containing the fields `access_output`, `access_state`, and `controller_state`. """ prev_access_output = prev_state.access_output prev_access_state = prev_state.access_state prev_controller_state = prev_state.controller_state batch_flatten = snt.BatchFlatten() controller_input = tf.concat( [batch_flatten(inputs), batch_flatten(prev_access_output)], 1) controller_output, controller_state = self._controller( controller_input, prev_controller_state) controller_output = self._clip_if_enabled(controller_output) controller_state = snt.nest.map(self._clip_if_enabled, controller_state) access_output, access_state = self._access(controller_output, prev_access_state) output = tf.concat([controller_output, batch_flatten(access_output)], 1) output = snt.Linear(output_size=self._output_size.as_list()[0], name='output_linear')(output) output = self._clip_if_enabled(output) return output, DNCState(access_output=access_output, access_state=access_state, controller_state=controller_state)
def __init__( self, learning_rate, num_layers, size, size_layer, output_size, forget_bias = 0.1, lambda_coeff = 0.5 ): def lstm_cell(size_layer): return tf.nn.rnn_cell.GRUCell(size_layer) rnn_cells = tf.nn.rnn_cell.MultiRNNCell( [lstm_cell(size_layer) for _ in range(num_layers)], state_is_tuple = False, ) self.X = tf.placeholder(tf.float32, (None, None, size)) self.Y = tf.placeholder(tf.float32, (None, output_size)) drop = tf.contrib.rnn.DropoutWrapper( rnn_cells, output_keep_prob = forget_bias ) self.hidden_layer = tf.placeholder( tf.float32, (None, num_layers * size_layer) ) _, last_state = tf.nn.dynamic_rnn( drop, self.X, initial_state = self.hidden_layer, dtype = tf.float32 ) self.z_mean = tf.layers.dense(last_state, size) self.z_log_sigma = tf.layers.dense(last_state, size) epsilon = tf.random_normal(tf.shape(self.z_log_sigma)) self.z_vector = self.z_mean + tf.exp(self.z_log_sigma) with tf.variable_scope('decoder', reuse = False): rnn_cells_dec = tf.nn.rnn_cell.MultiRNNCell( [lstm_cell(size_layer) for _ in range(num_layers)], state_is_tuple = False ) drop_dec = tf.contrib.rnn.DropoutWrapper( rnn_cells_dec, output_keep_prob = forget_bias ) x = tf.concat([tf.expand_dims(self.z_vector, axis=0), self.X], axis = 1) self.outputs, self.last_state = tf.nn.dynamic_rnn( drop_dec, self.X, initial_state = last_state, dtype = tf.float32 ) self.logits = tf.layers.dense(self.outputs[-1], output_size) self.lambda_coeff = lambda_coeff self.kl_loss = -0.5 * tf.reduce_sum(1.0 + 2 * self.z_log_sigma - self.z_mean ** 2 - tf.exp(2 * self.z_log_sigma), 1) self.kl_loss = tf.scalar_mul(self.lambda_coeff, self.kl_loss) self.cost = tf.reduce_mean(tf.square(self.Y - self.logits) + self.kl_loss) self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize( self.cost )
def multihead_attn(queries, keys, q_masks, k_masks, future_binding, num_units, num_heads): T_q = tf.shape(queries)[1] T_k = tf.shape(keys)[1] Q = tf.layers.dense(queries, num_units, name='Q') K_V = tf.layers.dense(keys, 2 * num_units, name='K_V') K, V = tf.split(K_V, 2, -1) Q_ = tf.concat(tf.split(Q, num_heads, axis=2), axis=0) K_ = tf.concat(tf.split(K, num_heads, axis=2), axis=0) V_ = tf.concat(tf.split(V, num_heads, axis=2), axis=0) align = tf.matmul(Q_, tf.transpose(K_, [0, 2, 1])) align = align / np.sqrt(K_.get_shape().as_list()[-1]) paddings = tf.fill(tf.shape(align), float('-inf')) key_masks = k_masks key_masks = tf.tile(key_masks, [num_heads, 1]) key_masks = tf.tile(tf.expand_dims(key_masks, 1), [1, T_q, 1]) align = tf.where(tf.equal(key_masks, 0), paddings, align) if future_binding: lower_tri = tf.ones([T_q, T_k]) lower_tri = tf.linalg.LinearOperatorLowerTriangular( lower_tri).to_dense() masks = tf.tile(tf.expand_dims(lower_tri, 0), [tf.shape(align)[0], 1, 1]) align = tf.where(tf.equal(masks, 0), paddings, align) align = tf.nn.softmax(align) query_masks = tf.to_float(q_masks) query_masks = tf.tile(query_masks, [num_heads, 1]) query_masks = tf.tile(tf.expand_dims(query_masks, -1), [1, 1, T_k]) align *= query_masks outputs = tf.matmul(align, V_) outputs = tf.concat(tf.split(outputs, num_heads, axis=0), axis=2) outputs += queries outputs = layer_norm(outputs) return outputs
def sinusoidal_position_encoding(inputs, mask, repr_dim): T = tf.shape(inputs)[1] pos = tf.reshape(tf.range(0.0, tf.to_float(T), dtype=tf.float32), [-1, 1]) i = np.arange(0, repr_dim, 2, np.float32) denom = np.reshape(np.power(10000.0, i / repr_dim), [1, -1]) enc = tf.expand_dims( tf.concat( [tf.sin(pos / denom), tf.cos(pos / denom)], 1), 0) return tf.tile(enc, [tf.shape(inputs)[0], 1, 1]) * tf.expand_dims( tf.to_float(mask), -1)
def cnn_block(x, dilation_rate, pad_sz, hidden_dim, kernel_size): x = layer_norm(x) pad = tf.zeros([tf.shape(x)[0], pad_sz, hidden_dim]) x = tf.layers.conv1d(inputs=tf.concat([pad, x, pad], 1), filters=hidden_dim, kernel_size=kernel_size, dilation_rate=dilation_rate) x = x[:, :-pad_sz, :] x = tf.nn.relu(x) return x
def position_encoding(inputs): T = tf.shape(inputs)[1] repr_dim = inputs.get_shape()[-1].value pos = tf.reshape(tf.range(0.0, tf.to_float(T), dtype=tf.float32), [-1, 1]) i = np.arange(0, repr_dim, 2, np.float32) denom = np.reshape(np.power(10000.0, i / repr_dim), [1, -1]) enc = tf.expand_dims( tf.concat( [tf.sin(pos / denom), tf.cos(pos / denom)], 1), 0) return tf.tile(enc, [tf.shape(inputs)[0], 1, 1])
def __init__(self, state_size, window_size, trend, skip): self.state_size = state_size self.window_size = window_size self.half_window = window_size // 2 self.trend = trend self.skip = skip tf.reset_default_graph() self.X = tf.placeholder(tf.float32, (None, self.state_size)) self.Y = tf.placeholder(tf.float32, (None, self.state_size)) self.ACTION = tf.placeholder(tf.float32, (None)) self.REWARD = tf.placeholder(tf.float32, (None)) self.batch_size = tf.shape(self.ACTION)[0] with tf.variable_scope('curiosity_model'): action = tf.reshape(self.ACTION, (-1,1)) state_action = tf.concat([self.X, action], axis=1) save_state = tf.identity(self.Y) feed = tf.layers.dense(state_action, 32, activation=tf.nn.relu) self.curiosity_logits = tf.layers.dense(feed, self.state_size) self.curiosity_cost = tf.reduce_sum(tf.square(save_state - self.curiosity_logits), axis=1) self.curiosity_optimizer = tf.train.RMSPropOptimizer(self.LEARNING_RATE) .minimize(tf.reduce_mean(self.curiosity_cost)) total_reward = tf.add(self.curiosity_cost, self.REWARD) with tf.variable_scope("q_model"): with tf.variable_scope("eval_net"): x_action = tf.layers.dense(self.X, 128, tf.nn.relu) self.logits = tf.layers.dense(x_action, self.OUTPUT_SIZE) with tf.variable_scope("target_net"): y_action = tf.layers.dense(self.Y, 128, tf.nn.relu) y_q = tf.layers.dense(y_action, self.OUTPUT_SIZE) q_target = total_reward + self.GAMMA * tf.reduce_max(y_q, axis=1) action = tf.cast(self.ACTION, tf.int32) action_indices = tf.stack([tf.range(self.batch_size, dtype=tf.int32), action], axis=1) q = tf.gather_nd(params=self.logits, indices=action_indices) self.cost = tf.losses.mean_squared_error(labels=q_target, predictions=q) self.optimizer = tf.train.RMSPropOptimizer(self.LEARNING_RATE).minimize( self.cost, var_list=tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "q_model/eval_net")) t_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='q_model/target_net') e_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='q_model/eval_net') self.target_replace_op = [tf.assign(t, e) for t, e in zip(t_params, e_params)] self.sess = tf.InteractiveSession() self.sess.run(tf.global_variables_initializer())
def __init__( self, learning_rate, num_layers, size, size_layer, output_size, forget_bias = 0.1, ): def lstm_cell(size_layer): return tf.nn.rnn_cell.LSTMCell(size_layer, state_is_tuple = False) backward_rnn_cells = tf.nn.rnn_cell.MultiRNNCell( [lstm_cell(size_layer) for _ in range(num_layers)], state_is_tuple = False, ) forward_rnn_cells = tf.nn.rnn_cell.MultiRNNCell( [lstm_cell(size_layer) for _ in range(num_layers)], state_is_tuple = False, ) self.X = tf.placeholder(tf.float32, (None, None, size)) self.Y = tf.placeholder(tf.float32, (None, output_size)) drop_backward = tf.contrib.rnn.DropoutWrapper( backward_rnn_cells, output_keep_prob = forget_bias ) forward_backward = tf.contrib.rnn.DropoutWrapper( forward_rnn_cells, output_keep_prob = forget_bias ) self.backward_hidden_layer = tf.placeholder( tf.float32, shape = (None, num_layers * 2 * size_layer) ) self.forward_hidden_layer = tf.placeholder( tf.float32, shape = (None, num_layers * 2 * size_layer) ) self.outputs, self.last_state = tf.nn.bidirectional_dynamic_rnn( forward_backward, drop_backward, self.X, initial_state_fw = self.forward_hidden_layer, initial_state_bw = self.backward_hidden_layer, dtype = tf.float32, ) self.outputs = tf.concat(self.outputs, 2) self.logits = tf.layers.dense(self.outputs[-1], output_size) self.cost = tf.reduce_mean(tf.square(self.Y - self.logits)) self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize( self.cost )
def __init__( self, learning_rate, num_layers, size, size_layer, output_size, kernel_size=3, n_attn_heads=16, dropout=0.9, ): self.X = tf.placeholder(tf.float32, (None, None, size)) self.Y = tf.placeholder(tf.float32, (None, output_size)) encoder_embedded = tf.layers.dense(self.X, size_layer) e = tf.identity(encoder_embedded) for i in range(num_layers): z = layer( encoder_embedded, encoder_block, kernel_size, size_layer * 2, encoder_embedded, ) z = tf.nn.dropout(z, keep_prob=dropout) encoder_embedded = z encoder_output, output_memory = z, z + e g = tf.identity(encoder_embedded) for i in range(num_layers): attn_res = h = layer( encoder_embedded, decoder_block, kernel_size, size_layer * 2, residual=tf.zeros_like(encoder_embedded), ) C = [] for j in range(n_attn_heads): h_ = tf.layers.dense(h, size_layer // n_attn_heads) g_ = tf.layers.dense(g, size_layer // n_attn_heads) zu_ = tf.layers.dense(encoder_output, size_layer // n_attn_heads) ze_ = tf.layers.dense(output_memory, size_layer // n_attn_heads) d = tf.layers.dense(h_, size_layer // n_attn_heads) + g_ dz = tf.matmul(d, tf.transpose(zu_, [0, 2, 1])) a = tf.nn.softmax(dz) c_ = tf.matmul(a, ze_) C.append(c_) c = tf.concat(C, 2) h = tf.layers.dense(attn_res + c, size_layer) h = tf.nn.dropout(h, keep_prob=dropout) encoder_embedded = h encoder_embedded = tf.sigmoid(encoder_embedded[-1]) self.logits = tf.layers.dense(encoder_embedded, output_size) self.cost = tf.reduce_mean(tf.square(self.Y - self.logits)) self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize( self.cost)
def __init__(self, state_size, window_size, trend, skip): self.state_size = state_size self.window_size = window_size self.half_window = window_size // 2 self.trend = trend self.skip = skip tf.reset_default_graph() self.INITIAL_FEATURES = np.zeros((4, self.state_size)) self.X = tf.placeholder(tf.float32, (None, None, self.state_size)) self.Y = tf.placeholder(tf.float32, (None, None, self.state_size)) self.hidden_layer = tf.placeholder(tf.float32, (None, 2 * self.LAYER_SIZE)) self.ACTION = tf.placeholder(tf.float32, (None)) self.REWARD = tf.placeholder(tf.float32, (None)) self.batch_size = tf.shape(self.ACTION)[0] self.seq_len = tf.shape(self.X)[1] with tf.variable_scope('curiosity_model'): action = tf.reshape(self.ACTION, (-1, 1, 1)) repeat_action = tf.tile(action, [1, self.seq_len, 1]) state_action = tf.concat([self.X, repeat_action], axis=-1) save_state = tf.identity(self.Y) cell = tf.nn.rnn_cell.LSTMCell(self.LAYER_SIZE, state_is_tuple=False) self.rnn, last_state = tf.nn.dynamic_rnn( inputs=state_action, cell=cell, dtype=tf.float32, initial_state=self.hidden_layer) self.curiosity_logits = tf.layers.dense(self.rnn[:, -1], self.state_size) self.curiosity_cost = tf.reduce_sum( tf.square(save_state[:, -1] - self.curiosity_logits), axis=1) self.curiosity_optimizer = tf.train.RMSPropOptimizer( self.LEARNING_RATE).minimize( tf.reduce_mean(self.curiosity_cost)) total_reward = tf.add(self.curiosity_cost, self.REWARD) with tf.variable_scope("q_model"): with tf.variable_scope("eval_net"): cell = tf.nn.rnn_cell.LSTMCell(self.LAYER_SIZE, state_is_tuple=False) rnn, self.last_state = tf.nn.dynamic_rnn( inputs=self.X, cell=cell, dtype=tf.float32, initial_state=self.hidden_layer) self.logits = tf.layers.dense(rnn[:, -1], self.OUTPUT_SIZE) with tf.variable_scope("target_net"): cell = tf.nn.rnn_cell.LSTMCell(self.LAYER_SIZE, state_is_tuple=False) rnn, last_state = tf.nn.dynamic_rnn( inputs=self.Y, cell=cell, dtype=tf.float32, initial_state=self.hidden_layer) y_q = tf.layers.dense(rnn[:, -1], self.OUTPUT_SIZE) q_target = total_reward + self.GAMMA * tf.reduce_max(y_q, axis=1) action = tf.cast(self.ACTION, tf.int32) action_indices = tf.stack( [tf.range(self.batch_size, dtype=tf.int32), action], axis=1) q = tf.gather_nd(params=self.logits, indices=action_indices) self.cost = tf.losses.mean_squared_error(labels=q_target, predictions=q) self.optimizer = tf.train.RMSPropOptimizer( self.LEARNING_RATE).minimize( self.cost, var_list=tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, "q_model/eval_net")) t_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='q_model/target_net') e_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='q_model/eval_net') self.target_replace_op = [ tf.assign(t, e) for t, e in zip(t_params, e_params) ] self.sess = tf.InteractiveSession() self.sess.run(tf.global_variables_initializer())