def __init__(self, args, is_training=True): self.args = args if not is_training: args.batch_size = 1 args.seq_length = 1 if args.model == 'rnn': self.cell = rnn_cell.BasicRNNCell(args.rnn_size) elif args.model == 'gru': self.cell = rnn_cell.GRUCell(args.rnn_size) elif args.model == 'lstm': self.cell = rnn_cell.BasicLSTMCell(args.rnn_size) else: raise Exception('model type not supported: {}'.format(args.model)) self.cell = rnn_cell.MultiRNNCell([self.cell] * args.num_layers) self.input_data = tf.placeholder(tf.int32, [args.batch_size, args.seq_length]) self.targets = tf.placeholder(tf.int32, [args.batch_size, args.seq_length]) # Target replication self.initial_state = self.cell.zero_state(args.batch_size, tf.float32) with tf.variable_scope('rnn'): softmax_w = tf.get_variable('softmax_w', [args.rnn_size, 2]) softmax_b = tf.get_variable('softmax_b', [2]) with tf.device('/cpu:0'): embedding = tf.get_variable('embedding', [args.vocab_size, args.rnn_size]) inputs = tf.split(1, args.seq_length, tf.nn.embedding_lookup(embedding, self.input_data)) inputs = [tf.squeeze(i, [1]) for i in inputs] outputs, last_state = seq2seq.rnn_decoder(inputs, self.initial_state, self.cell, loop_function=None) output_tf = tf.reshape(tf.concat(1, outputs), [-1, args.rnn_size]) self.logits = tf.nn.xw_plus_b(output_tf, softmax_w, softmax_b) self.probs = tf.nn.softmax(self.logits) loss = seq2seq.sequence_loss_by_example( [self.logits], [tf.reshape(self.targets, [-1])], [tf.ones([args.batch_size * args.seq_length])]) self.cost = tf.reduce_sum(loss) / args.batch_size / args.seq_length self.final_state = last_state self.lr = tf.Variable(0.0, trainable = False) tvars = tf.trainable_variables() grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, tvars, aggregation_method=2), args.grad_clip) optimizer = tf.train.AdamOptimizer(self.lr) self.train_op = optimizer.apply_gradients(zip(grads, tvars))
def __init__(self, sess, vocab_size, cell_size, embedding_size, num_layer, memory_size, log_dir, learning_rate=0.001, momentum=0.9, learning_rate_decay_factor=0.85, use_dropout=True, l2_coef=1e-6): with tf.name_scope("io"): self.inputs = tf.placeholder(dtype=tf.int32, shape=(None, None), name="prev_words") self.input_lens = tf.placeholder(dtype=tf.int32, shape=(None, ), name="sent_len") self.labels = tf.placeholder(dtype=tf.int32, shape=(None, None), name="next_words") self.keep_prob = tf.placeholder(dtype=tf.float32, name="keep_prob") self.learning_rate = tf.Variable(float(learning_rate), trainable=False) self.learning_rate_decay_op = self.learning_rate.assign( self.learning_rate * learning_rate_decay_factor) max_sent_len = array_ops.shape(self.labels)[1] with variable_scope.variable_scope("word-embedding"): embedding = tf.get_variable("embedding", [vocab_size, embedding_size], dtype=tf.float32) input_embedding = embedding_ops.embedding_lookup( embedding, tf.squeeze(tf.reshape(self.inputs, [-1, 1]), squeeze_dims=[1])) input_embedding = tf.reshape(input_embedding, [-1, max_sent_len, embedding_size]) with variable_scope.variable_scope("rnn"): # cell = tf_helpers.MemoryGRUCell(cell_size, memory_size, attn_size=100) cell = rnn_cell.BasicLSTMCell(cell_size) if use_dropout: cell = rnn_cell.DropoutWrapper(cell, output_keep_prob=self.keep_prob, input_keep_prob=self.keep_prob) if num_layer > 1: cell = rnn_cell.MultiRNNCell([cell] * num_layer, state_is_tuple=True) # add output projection cell = tf.nn.rnn_cell.OutputProjectionWrapper(cell, vocab_size) # and enc_last_state will be same as the true last state self.logits, last_state = tf.nn.dynamic_rnn( cell, input_embedding, dtype=tf.float32, sequence_length=self.input_lens, ) self.loss = self.sequence_loss() tf.scalar_summary("entropy_loss", self.loss) tf.scalar_summary("perplexity", tf.exp(self.loss)) self.summary_op = tf.merge_all_summaries() # weight decay """ if l2_coef > 0.0: all_weights = [] vars = tf.trainable_variables() for v in vars: if "bias" not in v.name.lower(): all_weights.append(tf.nn.l2_loss(v)) print("adding l2 to %s" %v.name) loss_l2= tf.add_n(all_weights) self.reg_loss = self.loss + l2_coef * loss_l2 else: self.reg_loss = self.loss """ # optimization tvars = tf.trainable_variables() grads, _ = tf.clip_by_global_norm(tf.gradients(self.loss, tvars), 5.0) optimizer = tf.train.GradientDescentOptimizer(self.learning_rate) self.train_ops = optimizer.apply_gradients(zip(grads, tvars)) train_log_dir = os.path.join(log_dir, "train") valid_log_dir = os.path.join(log_dir, "valid") print "Save summary to %s" % log_dir self.train_summary_writer = tf.train.SummaryWriter( train_log_dir, sess.graph) self.valid_summary_writer = tf.train.SummaryWriter( valid_log_dir, sess.graph) self.saver = tf.train.Saver(tf.all_variables())
def __init__(self, config, is_training=False): self.config = config self.batch_size = batch_size = config.batch_size self.num_steps = num_steps = config.num_steps self.hidden_size = hidden_size = config.hidden_size self.num_layers = 1 vocab_size = config.vocab_size self.max_grad_norm = config.max_grad_norm self.use_lstm = config.use_lstm # Placeholders for inputs. self.input_data = tf.placeholder(tf.int32, [batch_size, num_steps]) self.targets = tf.placeholder(tf.int32, [batch_size, num_steps]) self.initial_state = array_ops.zeros( array_ops.pack([self.batch_size, self.num_steps]), dtype=tf.float32).set_shape([None, self.num_steps]) embedding = tf.get_variable( 'embedding', [self.config.vocab_size, self.config.hidden_size]) # Set up ACT cell and inner rnn-type cell for use inside the ACT cell. with tf.variable_scope("rnn"): if self.use_lstm: inner_cell = rnn_cell.BasicLSTMCell(self.config.hidden_size) else: inner_cell = rnn_cell.GRUCell(self.config.hidden_size) with tf.variable_scope("ACT"): act = ACTCell(self.config.hidden_size, inner_cell, config.epsilon, max_computation=config.max_computation, batch_size=self.batch_size) inputs = tf.nn.embedding_lookup(embedding, self.input_data) inputs = [ tf.squeeze(single_input, [1]) for single_input in tf.split(1, self.config.num_steps, inputs) ] self.outputs, final_state = rnn(act, inputs, dtype=tf.float32) # Softmax to get probability distribution over vocab. output = tf.reshape(tf.concat(1, self.outputs), [-1, hidden_size]) softmax_w = tf.get_variable("softmax_w", [hidden_size, vocab_size]) softmax_b = tf.get_variable("softmax_b", [vocab_size]) self.logits = tf.matmul( output, softmax_w) + softmax_b # dim (numsteps*batchsize, vocabsize) loss = seq2seq.sequence_loss_by_example( [self.logits], [tf.reshape(self.targets, [-1])], [tf.ones([batch_size * num_steps])], vocab_size) # Add up loss and retrieve batch-normalised ponder cost: sum N + sum Remainder. ponder_cost = act.calculate_ponder_cost( time_penalty=self.config.ponder_time_penalty) self.cost = (tf.reduce_sum(loss) / batch_size) + ponder_cost self.final_state = self.outputs[-1] if is_training: self.lr = tf.Variable(0.0, trainable=False) tvars = tf.trainable_variables() grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, tvars), self.max_grad_norm) optimizer = tf.train.AdamOptimizer(self.config.learning_rate) self.train_op = optimizer.apply_gradients(zip(grads, tvars))
def __init__(self, num_symbols, num_embed_units, num_units, num_labels, batch_size, embed, learning_rate=0.001, max_gradient_norm=5.0 ): # todo: implement placeholders self.texts1 = tf.placeholder(tf.string, [batch_size, None], name='texts1') self.texts2 = tf.placeholder(tf.string, [batch_size, None], name='texts2') # shape: batch*len self.texts_length = tf.placeholder(tf.int32, [None], name='texts_length') # shape: batch self.len = tf.constant(1.0, shape=[batch_size]) self.labels = tf.placeholder( tf.int64, [None], name='labels') # shape: batch self.keep_prob = tf.placeholder(tf.float32, name='keep_prob') self.embed_units = num_embed_units self.batch_size = batch_size self._initializer = tf.truncated_normal_initializer(stddev=0.1) self.symbol2index = MutableHashTable( key_dtype=tf.string, value_dtype=tf.int64, default_value=UNK_ID, shared_name="in_table", name="in_table", checkpoint=True) # build the vocab table (string to index) # initialize the training process self.learning_rate = tf.Variable( float(learning_rate), trainable=False, dtype=tf.float32) self.global_step = tf.Variable(0, trainable=False) self.epoch = tf.Variable(0, trainable=False) self.epoch_add_op = self.epoch.assign(self.epoch + 1) self.index_input1 = self.symbol2index.lookup(self.texts1) # batch*len self.index_input2 = self.symbol2index.lookup(self.texts2) ''' self.h_s1 = tf.Variable(tf.constant(0.0,shape=[num_units+1, batch_size, num_embed_units]), trainable=False) self.h_s2 = tf.Variable(tf.constant(0.0,shape=[num_units+1, batch_size, num_embed_units]), trainable=False) self.h_r = tf.Variable(tf.constant(0.0,shape=[num_units+1, batch_size, num_embed_units]), trainable=False) self.a1 = tf.Variable(tf.constant(0.0,shape=[num_units+1, batch_size, num_embed_units]), trainable=False) self.a2 = tf.Variable(tf.constant(0.0,shape=[num_units+1, batch_size, num_embed_units]), trainable=False) ''' self.h_s1 = [] self.h_s2 = [] self.h_r = [] self.a1 = [] self.a2 = [] # build the embedding table (index to vector) if embed is None: # initialize the embedding randomly self.embed = tf.get_variable( 'embed', [num_symbols, num_embed_units], tf.float32) else: # initialize the embedding by pre-trained word vectors self.embed = tf.get_variable( 'embed', dtype=tf.float32, initializer=embed) self.embed_input1 = tf.nn.embedding_lookup( self.embed, self.index_input1) # batch*len*embed_unit self.embed_input2 = tf.nn.embedding_lookup( self.embed, self.index_input2) with tf.variable_scope('lstm_s'): self.lstm_s = rnn_cell.BasicLSTMCell(num_units=num_embed_units, forget_bias=0) ''' out_s1, state_s1 = tf.nn.dynamic_rnn(self.lstm_s, self.embed_input1, self.texts_length, dtype=tf.float32) out_s2, state_s2 = tf.nn.dynamic_rnn(self.lstm_s, self.embed_input2, self.texts_length, dtype=tf.float32) self.h_s1 = state_s1 self.h_s2 = state_s2 ''' with tf.variable_scope('lstm_r'): self.lstm_r = rnn_cell.BasicLSTMCell(num_units=num_embed_units, forget_bias=0) ''' self.ini_op1 = tf.assign(self.h_s1[0], self.lstm_s.zero_state(batch_size=batch_size, dtype=tf.float32)) self.ini_op2 = tf.assign(self.h_s2[0], self.lstm_s.zero_state(batch_size=batch_size, dtype=tf.float32)) self.ini_op3 = tf.assign(self.h_r[0], self.lstm_r.zero_state(batch_size=batch_size, dtype=tf.float32)) self.ini_op4 = tf.assign(self.a1[0], self.lstm_r.zero_state(batch_size=batch_size, dtype=tf.float32)) self.ini_op5 = tf.assign(self.a2[0], self.lstm_r.zero_state(batch_size=batch_size, dtype=tf.float32)) ''' self.h_s1.append(self.lstm_s.zero_state(batch_size=batch_size, dtype=tf.float32)) self.h_s2.append(self.lstm_s.zero_state(batch_size=batch_size, dtype=tf.float32)) self.h_r.append(self.lstm_s.zero_state(batch_size=batch_size, dtype=tf.float32)) self.a1.append(self.lstm_r.zero_state(batch_size=batch_size, dtype=tf.float32)) self.a2.append(self.lstm_r.zero_state(batch_size=batch_size, dtype=tf.float32)) W = tf.Variable(self._initializer(shape=[num_embed_units, num_labels],dtype=tf.float32)) bias = tf.Variable(tf.constant(0.0, shape=[num_labels]), dtype=tf.float32) i = tf.constant(1, dtype=tf.int64) print self.index_input1[1].get_shape() length = self._length(self.index_input1[1]) self.ind = 1 state_s1 = self.lstm_s.zero_state(batch_size=batch_size, dtype=tf.float32) state_s2 = self.lstm_s.zero_state(batch_size=batch_size, dtype=tf.float32) state_r = self.lstm_r.zero_state(batch_size=batch_size, dtype=tf.float32) def c(t, s1, s2, sr): return tf.less(t, length+1) def b(t, s1, s2, sr): return self.attention(t, s1, s2, sr) i, state_s1, state_s2, state_r = tf.while_loop(cond=c, body=b, loop_vars=(i, state_s1, state_s2, state_r)) logits = tf.matmul(state_r.h, W) + bias #logits = tf.layers.dense(outputs, num_labels) # todo: implement unfinished networks self.loss = tf.reduce_sum(tf.nn.sparse_softmax_cross_entropy_with_logits( labels=self.labels, logits=logits), name='loss') mean_loss = self.loss / \ tf.cast(tf.shape(self.labels)[0], dtype=tf.float32) predict_labels = tf.argmax(logits, 1, 'predict_labels') self.accuracy = tf.reduce_sum( tf.cast(tf.equal(self.labels, predict_labels), tf.int32), name='accuracy') self.params = tf.trainable_variables() # calculate the gradient of parameters ''' opt = tf.train.GradientDescentOptimizer(self.learning_rate) gradients = tf.gradients(mean_loss, self.params) clipped_gradients, self.gradient_norm = tf.clip_by_global_norm( gradients, max_gradient_norm) self.update = opt.apply_gradients( zip(clipped_gradients, self.params), global_step=self.global_step) ''' self.global_step = tf.Variable(0, trainable=False) self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(mean_loss, global_step=self.global_step, var_list=self.params) tf.summary.scalar('loss/step', self.loss) for each in tf.trainable_variables(): tf.summary.histogram(each.name, each) self.merged_summary_op = tf.summary.merge_all() self.saver = tf.train.Saver(write_version=tf.train.SaverDef.V2, max_to_keep=3, pad_step_number=True, keep_checkpoint_every_n_hours=1.0)
def __init__(self, args, is_training=True): if not is_training: seq_length = 1 else: seq_length = args.seq_length if args.model == 'rnn': cell_gen = rnn_cell.BasicRNNCell(args.rnn_size) cell_dis = rnn_cell.BasicRNNCell(args.rnn_size) elif args.model == 'gru': cell_gen = rnn_cell.GRUCell(args.rnn_size) cell_dis = rnn_cell.GRUCell(args.rnn_size) elif args.model == 'lstm': cell_gen = rnn_cell.BasicLSTMCell(args.rnn_size) cell_dis = rnn_cell.BasicLSTMCell(args.rnn_size) else: raise Exception('model type not supported: {}'.format(args.model)) # Pass the generated sequences and targets (1) with tf.name_scope('input'): with tf.name_scope('data'): self.input_data = tf.placeholder(tf.int32, [args.batch_size, seq_length]) with tf.name_scope('targets'): self.targets = tf.placeholder(tf.int32, [args.batch_size, seq_length]) ############ # Generator ############ with tf.variable_scope('generator'): self.cell_gen = rnn_cell.MultiRNNCell([cell_gen] * args.num_layers) self.initial_state_gen = self.cell_gen.zero_state(args.batch_size, tf.float32) with tf.variable_scope('rnn'): softmax_w = tf.get_variable('softmax_w', [args.rnn_size, args.vocab_size]) softmax_b = tf.get_variable('softmax_b', [args.vocab_size]) with tf.device('/cpu:0'): embedding = tf.get_variable('embedding', [args.vocab_size, args.rnn_size]) inputs_gen = tf.split(1, seq_length, tf.nn.embedding_lookup( embedding, self.input_data)) inputs_gen = [tf.squeeze(i, [1]) for i in inputs_gen] outputs_gen, last_state_gen = seq2seq.rnn_decoder(inputs_gen, self.initial_state_gen, self.cell_gen, loop_function=None) self.logits_sequence = [] for output_gen in outputs_gen: logits_gen = tf.nn.xw_plus_b(output_gen, softmax_w, softmax_b) self.logits_sequence.append(logits_gen) self.final_state_gen = last_state_gen ################ # Discriminator ################ with tf.variable_scope('discriminator'): self.cell_dis = rnn_cell.MultiRNNCell([cell_dis] * args.num_layers) self.initial_state_dis = self.cell_dis.zero_state(args.batch_size, tf.float32) with tf.variable_scope('rnn'): softmax_w = tf.get_variable('softmax_w', [args.rnn_size, 2]) softmax_b = tf.get_variable('softmax_b', [2]) inputs_dis = [] embedding = tf.get_variable('embedding', [args.vocab_size, args.rnn_size]) for logit in self.logits_sequence: inputs_dis.append(tf.matmul(logit, embedding)) # inputs_dis.append(tf.matmul(tf.nn.softmax(logit), embedding)) outputs_dis, last_state_dis = seq2seq.rnn_decoder(inputs_dis, self.initial_state_dis, self.cell_dis, loop_function=None) probs, logits = [], [] for output_dis in outputs_dis: logit = tf.nn.xw_plus_b(output_dis, softmax_w, softmax_b) prob = tf.nn.softmax(logit) logits.append(logit) probs.append(prob) with tf.name_scope('summary'): probs = tf.pack(probs) probs_real = tf.slice(probs, [0,0,1], [args.seq_length, args.batch_size, 1]) variable_summaries(probs_real, 'probability of real') self.final_state_dis = last_state_dis ######### # Train ######### with tf.name_scope('train'): gen_loss = seq2seq.sequence_loss_by_example( logits, tf.unpack(tf.transpose(self.targets)), tf.unpack(tf.transpose(tf.ones_like(self.targets, dtype=tf.float32)))) self.gen_cost = tf.reduce_sum(gen_loss) / args.batch_size tf.scalar_summary('training loss', self.gen_cost) self.lr_gen = tf.Variable(0.0, trainable = False) self.tvars = tf.trainable_variables() gen_vars = [v for v in self.tvars if not v.name.startswith("discriminator/")] if is_training: gen_grads = tf.gradients(self.gen_cost, gen_vars) self.all_grads = tf.gradients(self.gen_cost, self.tvars) gen_grads_clipped, _ = tf.clip_by_global_norm(gen_grads, args.grad_clip) gen_optimizer = tf.train.AdamOptimizer(self.lr_gen) self.gen_train_op = gen_optimizer.apply_gradients( zip(gen_grads_clipped, gen_vars)) with tf.name_scope('summary'): with tf.name_scope('weight_summary'): for v in self.tvars: variable_summaries(v, v.op.name) if is_training: with tf.name_scope('grad_summary'): for var, grad in zip(self.tvars, self.all_grads): variable_summaries(grad, 'grad/' + var.op.name) self.merged = tf.merge_all_summaries()
def __init__(self, embed_size, lstm_size, vocab_size, \ batch_size, seq_length, learn_rate, \ keep_prob=1.0, num_layers=2, name='char_lstm'): ''' Initialize a character-level multilayer LSTM language model. Arguments: @embed_size: dimensions of embedding space @vocab_size: number of things in vocabulary (characters!) @batch_size: sequences per training batch @seq_length: length of sequences in each training batch @learn_rate: AdamOptimizer step size @keep_prob: 1 - dropout probability @num_layers: number of LSTM cells to stack ''' # store params self.embed_size, self.lstm_size = embed_size, lstm_size self.vocab_size, self.seq_length = vocab_size, seq_length self.batch_size, self.learn_rate = batch_size, learn_rate self.kp, self.num_layers = keep_prob, num_layers self.name = name # Placeholders for input/output and dropout self.train_inputs = tf.placeholder(tf.int32, shape=[batch_size, seq_length]) self.train_targets = tf.placeholder(tf.int32, shape=[batch_size, seq_length]) self.sample_inputs = tf.placeholder(tf.int32, shape=[1, 1]) # Set up embeddings E = weight('embedding', [vocab_size, embed_size]) train_embeddings = tf.nn.embedding_lookup(E, self.train_inputs, name='train_embeddings') dropped_train_embeddings = (tf.nn.dropout(train_embeddings, self.kp) if self.kp < 1.0 else train_embeddings) sample_embeddings = tf.nn.embedding_lookup(E, self.sample_inputs, name='sample_embeddings') # Set up 2-layer LSTM # Use dynamic_rnn to run the cells with tf.variable_scope('lstm') as scope: single_cell = rnn_cell.BasicLSTMCell(lstm_size) self.cell = rnn_cell.MultiRNNCell([single_cell] * num_layers, state_is_tuple=True) self.train_init_state = self.cell.zero_state( batch_size, tf.float32) self.sample_init_state = self.cell.zero_state(1, tf.float32) train_outputs, self.train_state = dynamic_rnn( self.cell, dropped_train_embeddings, initial_state=self.train_init_state) scope.reuse_variables() sample_outputs, self.sample_state = dynamic_rnn( self.cell, sample_embeddings, initial_state=self.sample_init_state) reshaped_train_outputs = tf.reshape( train_outputs, (batch_size * seq_length, lstm_size)) reshaped_sample_outputs = tf.reshape(sample_outputs, (1, lstm_size)) # final feedforward layer (model logits) with tf.variable_scope('ff') as scope: ff_weights = weight('ff_weights', (lstm_size, vocab_size)) ff_biases = weight('ff_biases', (vocab_size, )) self.ls = train_logits = tf.add( ff_biases, tf.matmul(reshaped_train_outputs, ff_weights)) scope.reuse_variables() sample_logits = tf.add( ff_biases, tf.matmul(reshaped_sample_outputs, ff_weights)) self.probs = softmax(sample_logits) # softmax and loss for training log_perps = tf.nn.seq2seq.sequence_loss_by_example( [train_logits], [tf.reshape(self.train_targets, [-1])], [tf.ones([batch_size * seq_length])]) self.loss = tf.reduce_mean(log_perps) / batch_size # define trainer, saver, inits self.train_op = tf.train.AdamOptimizer(learn_rate).minimize(self.loss) self.init_op = tf.initialize_all_variables() self.saver = tf.train.Saver()
def __init__(self, num_symbols, num_embed_units, num_units, num_labels, batch_size, embed, learning_rate=0.001, max_gradient_norm=5.0): self.texts1 = tf.placeholder(tf.string, [batch_size, None], name='texts1') self.texts2 = tf.placeholder(tf.string, [batch_size, None], name='texts2') # shape: batch*len self.texts_length = tf.placeholder(tf.int32, [None], name='texts_length') # shape: batch learning_rate_decay_factor = 0.9 self.len = tf.constant(1.0, shape=[batch_size]) self.labels = tf.placeholder(tf.int64, [None], name='labels') # shape: batch self.keep_prob = tf.placeholder(tf.float32, name='keep_prob') self.embed_units = num_embed_units self.num_units = num_units self.batch_size = batch_size self._initializer = tf.truncated_normal_initializer(stddev=0.1) self.symbol2index = MutableHashTable(key_dtype=tf.string, value_dtype=tf.int64, default_value=UNK_ID, shared_name="in_table", name="in_table", checkpoint=True) # build the vocab table (string to index) # initialize the training process self.learning_rate = tf.Variable(float(learning_rate), trainable=False, dtype=tf.float32) self.global_step = tf.Variable(0, trainable=False) self.epoch = tf.Variable(0, trainable=False) self.epoch_add_op = self.epoch.assign(self.epoch + 1) self.learning_rate = tf.train.exponential_decay( self.learning_rate, self.epoch, 1, 0.95) self.index_input1 = self.symbol2index.lookup(self.texts1) # batch*len self.index_input2 = self.symbol2index.lookup(self.texts2) # build the embedding table (index to vector) if embed is None: # initialize the embedding randomly self.embed = tf.get_variable('embed', [num_symbols, num_embed_units], tf.float32) else: # initialize the embedding by pre-trained word vectors self.embed = tf.get_variable('embed', dtype=tf.float32, initializer=embed) self.embed_input1 = tf.nn.embedding_lookup( self.embed, self.index_input1) # batch*len*embed_unit self.embed_input2 = tf.nn.embedding_lookup(self.embed, self.index_input2) with tf.variable_scope('lstm_s'): self.lstm_s = rnn_cell.BasicLSTMCell(num_units=num_units, forget_bias=0) with tf.variable_scope('lstm_r'): self.lstm_r = rnn_cell.BasicLSTMCell(num_units=num_units, forget_bias=0) out_s1, state_s1 = dynamic_rnn(self.lstm_s, self.embed_input1, self.texts_length, dtype=tf.float32, scope='rnn') out_s2, state_s2 = dynamic_rnn(self.lstm_s, self.embed_input2, self.texts_length, dtype=tf.float32, scope='rnn') self.h_s1 = out_s1 self.h_s2 = out_s2 reshaped_s1 = tf.reshape(self.h_s1, [-1, self.num_units]) reshaped_s2 = tf.reshape(self.h_s2, [-1, self.num_units]) with tf.variable_scope('Attn_'): W_s = tf.get_variable(shape=[self.num_units, self.num_units], initializer=self._initializer, name='W_s') self.s_1 = tf.matmul(reshaped_s1, W_s) self.s_1 = tf.transpose( tf.reshape(self.s_1, [self.batch_size, -1, self.num_units]), [1, 2, 0]) i = tf.constant(0) self.length = tf.reduce_max(self.texts_length) print(self.length) state_r = self.lstm_r.zero_state(batch_size=batch_size, dtype=tf.float32) def c(t, sr): return tf.less(t, self.length) def b(t, sr): return self.attention(t, sr) i, state_r = tf.while_loop(cond=c, body=b, loop_vars=(i, state_r)) with tf.variable_scope('fully_connect'): w_fc = tf.get_variable(shape=[self.num_units, num_labels], initializer=self._initializer, name='w_fc') b_fc = tf.get_variable(shape=[num_labels], initializer=self._initializer, name='b_fc') logits = tf.matmul(state_r.h, w_fc) + b_fc self.loss = tf.reduce_sum( tf.nn.sparse_softmax_cross_entropy_with_logits(labels=self.labels, logits=logits), name='loss') mean_loss = self.loss / \ tf.cast(tf.shape(self.labels)[0], dtype=tf.float32) predict_labels = tf.argmax(logits, 1, 'predict_labels') self.accuracy = tf.reduce_sum(tf.cast( tf.equal(self.labels, predict_labels), tf.int64), name='accuracy') self.params = tf.trainable_variables() # calculate the gradient of parameters for item in tf.global_variables(): print(item) opt = tf.train.GradientDescentOptimizer(self.learning_rate) gradients = tf.gradients(mean_loss, self.params) clipped_gradients, self.gradient_norm = tf.clip_by_global_norm( gradients, max_gradient_norm) self.update = opt.apply_gradients(zip(clipped_gradients, self.params), global_step=self.global_step) #self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(mean_loss, global_step=self.global_step, #var_list=self.params) tf.summary.scalar('loss/step', self.loss) for each in tf.trainable_variables(): tf.summary.histogram(each.name, each) self.merged_summary_op = tf.summary.merge_all() self.saver = tf.train.Saver(write_version=tf.train.SaverDef.V2, max_to_keep=3, pad_step_number=True, keep_checkpoint_every_n_hours=1.0)