def build_cell(self,residual_velocities=True): """Create recurrent cell.""" cell = tf.contrib.rnn.LSTMCell(self.cell_size,reuse=self.reuse,initializer=tf.orthogonal_initializer(),forget_bias=0.2) cell = tf.contrib.rnn.AttentionCellWrapper(cell, attn_length=64, state_is_tuple=True) cell = tf.nn.rnn_cell.DropoutWrapper(cell, output_keep_prob=1-0.5) #cell = tf.contrib.rnn.BidirectionalGridLSTMCell(self.cell_size,reuse=self.reuse,initializer=tf.orthogonal_initializer(),forget_bias=0.2) if self.num_layers > 1: cell = tf.contrib.rnn.MultiRNNCell( [tf.contrib.rnn.LSTMCell(1024,reuse=self.reuse,initializer=tf.orthogonal_initializer(),forget_bias=0.2) for _ in range(self.num_layers)] ) cell = tf.nn.rnn_cell.DropoutWrapper(cell, output_keep_prob=1-0.5) cell = rnn_cell_extensions.LinearSpaceDecoderWrapper( cell, self.input_size,reuse=self.reuse) if residual_velocities: cell = rnn_cell_extensions.ResidualWrapper(cell) ''' with tf.variable_scope("rnn_cell", reuse=self.reuse): if self.cell_type == C.LSTM: cell = tf.nn.rnn_cell.LSTMCell(self.cell_size, reuse=self.reuse) elif self.cell_type == C.GRU: cell = tf.nn.rnn_cell.GRUCell(self.cell_size, reuse=self.reuse) else: raise ValueError("Cell type '{}' unknown".format(self.cell_type)) ''' self.cell = cell
def __init__( self, architecture, source_seq_len, target_seq_len, rnn_size, # hidden recurrent layer size num_layers, max_gradient_norm, batch_size, learning_rate, learning_rate_decay_factor, summaries_dir, loss_to_use, number_of_actions, one_hot=True, residual_velocities=False, dtype=tf.float32, custom_opt=False, cgru=True, fft=True, window_size=30, step_size=10, window_fun='hann', gaussian_scaling=False): """Create the model. Args: architecture: [basic, tied] whether to tie the decoder and decoder. source_seq_len: lenght of the input sequence. target_seq_len: lenght of the target sequence. rnn_size: number of units in the rnn. num_layers: number of rnns to stack. max_gradient_norm: gradients will be clipped to maximally this norm. batch_size: the size of the batches used during training; the model construction is independent of batch_size, so it can be changed after initialization if this is convenient, e.g., for decoding. learning_rate: learning rate to start with. learning_rate_decay_factor: decay learning rate by this much when needed. summaries_dir: where to log progress for tensorboard. loss_to_use: [supervised, sampling_based]. Whether to use ground truth in each timestep to compute the loss after decoding, or to feed back the prediction from the previous time-step. number_of_actions: number of classes we have. one_hot: whether to use one_hot encoding during train/test (sup models). residual_velocities: whether to use a residual connection that models velocities. dtype: the data type to use to store internal variables. """ if fft: assert cgru == True if custom_opt: assert cgru == True self.HUMAN_SIZE = 54 self.input_size = self.HUMAN_SIZE + number_of_actions if one_hot else self.HUMAN_SIZE print("One hot is ", one_hot) print("Input size is %d" % self.input_size) # Summary writers for train and test runs self.train_writer = tf.summary.FileWriter( os.path.normpath(os.path.join(summaries_dir, 'train'))) self.test_writer = tf.summary.FileWriter( os.path.normpath(os.path.join(summaries_dir, 'test'))) self.source_seq_len = source_seq_len self.target_seq_len = target_seq_len self.rnn_size = rnn_size self.batch_size = batch_size self.learning_rate = tf.Variable(float(learning_rate), trainable=False, dtype=dtype) self.learning_rate_decay_op = self.learning_rate.assign( self.learning_rate * learning_rate_decay_factor) self.global_step = tf.Variable(0, trainable=False) # === Transform the inputs === with tf.name_scope("inputs"): enc_in = tf.placeholder( dtype, shape=[None, source_seq_len - 1, self.input_size], name="enc_in") dec_in = tf.placeholder( dtype, shape=[None, target_seq_len, self.input_size], name="dec_in") dec_out = tf.placeholder( dtype, shape=[None, target_seq_len, self.input_size], name="dec_out") self.encoder_inputs = enc_in self.decoder_inputs = dec_in self.decoder_outputs = dec_out enc_in = tf.transpose(enc_in, [1, 0, 2]) dec_in = tf.transpose(dec_in, [1, 0, 2]) dec_out = tf.transpose(dec_out, [1, 0, 2]) enc_in = tf.reshape(enc_in, [-1, self.input_size]) dec_in = tf.reshape(dec_in, [-1, self.input_size]) dec_out = tf.reshape(dec_out, [-1, self.input_size]) enc_in = tf.split(enc_in, source_seq_len - 1, axis=0) dec_in = tf.split(dec_in, target_seq_len, axis=0) dec_out = tf.split(dec_out, target_seq_len, axis=0) if fft: assert cgru == True # if true do centering to avoid boundary problems. center = True if center: pad_enc_in = tf.stack(enc_in, axis=-1) pad_amount = 2 * (window_size - step_size) print('padding with', [pad_amount // 2, pad_amount // 2]) # debug_here() pad_enc_in = tf.pad( pad_enc_in, [[0, 0], [0, 0], [pad_amount // 2, pad_amount // 2]], 'REFLECT') else: pad_enc_in = tf.stack(enc_in, axis=-1) # transform input and output. if window_fun == 'hann': w = functools.partial(tf.contrib.signal.hann_window, periodic=True) elif window_fun == 'hamming': w = functools.partial(tf.contrib.signal.hamming_window, periodic=True) elif window_fun == 'None': w = None else: raise ValueError("unknown window function.") fft_enc_in = tfsignal.stft(pad_enc_in, window_size, step_size, window_fn=w) print('fft_enc_in.shape', fft_enc_in.shape) batch_size = tf.shape(fft_enc_in)[0] freq_tensor_shape = fft_enc_in.get_shape().as_list() frames_in = freq_tensor_shape[2] fft_dim_in = freq_tensor_shape[1] * freq_tensor_shape[-1] fft_enc_in = tf.transpose(fft_enc_in, [0, 2, 1, 3]) fft_enc_in = tf.reshape(fft_enc_in, [batch_size, frames_in, fft_dim_in], name='fft_enc_in_reshape') fft_enc_in = tf.unstack(fft_enc_in, axis=1) if center is True: pad_dec_in = tf.stack(dec_in, axis=-1) pad_dec_in = tf.pad( pad_dec_in, [[0, 0], [0, 0], [pad_amount // 2, pad_amount // 2]], 'REFLECT') else: pad_dec_in = tf.stack(dec_in, axis=-1) fft_dec_in = tfsignal.stft(pad_dec_in, window_size, step_size, window_fn=w) print('fft_dec_in.shape', fft_dec_in.shape) batch_size = tf.shape(fft_dec_in)[0] freq_tensor_shape = fft_dec_in.get_shape().as_list() frames_dec = freq_tensor_shape[2] fft_unique_bins_dec = freq_tensor_shape[3] assert self.input_size == freq_tensor_shape[1] fft_dim_out = self.input_size * fft_unique_bins_dec fft_dec_in = tf.transpose(fft_dec_in, [0, 2, 1, 3]) fft_dec_in = tf.reshape(fft_dec_in, [batch_size, frames_dec, fft_dim_out], name='fft_dec_in_reshape') fft_dec_in = tf.unstack(fft_dec_in, axis=1) enc_in = fft_enc_in dec_in = fft_dec_in assert fft_dim_in == fft_dim_out # === Create the RNN that will keep the state === print('rnn_size = {0}'.format(rnn_size)) if cgru: if not fft: cell = rnn_cell_extensions.ComplexGatedRecurrentUnit( self.rnn_size) else: # num_proj = self.input_size * (window_size//2+1) cell = rnn_cell_extensions.ComplexGatedRecurrentUnit( self.rnn_size, complex_out=fft, num_proj=fft_dim_in) print(cell.to_string()) else: cell = tf.contrib.rnn.GRUCell(self.rnn_size) if num_layers > 1: cell = tf.contrib.rnn.MultiRNNCell([ tf.contrib.rnn.GRUCell(self.rnn_size) for _ in range(num_layers) ]) # === Add space decoder === if not fft: cell = rnn_cell_extensions.LinearSpaceDecoderWrapper( cell, self.input_size) # Finally, wrap everything in a residual layer if we want to model velocities if residual_velocities: assert fft is False print('using resudial_velocities') cell = rnn_cell_extensions.ResidualWrapper(cell) # Store the outputs here outputs = [] # Define the loss function lf = None if loss_to_use == "sampling_based": def lf(prev, i): # function for sampling_based loss return prev elif loss_to_use == "supervised": pass else: raise (ValueError, "unknown loss: %s" % loss_to_use) # Build the RNN if architecture == "basic": # Basic RNN does not have a loop function in its API, so copying here. with vs.variable_scope("basic_rnn_seq2seq"): _, enc_state = tf.contrib.rnn.static_rnn( cell, enc_in, dtype=tf.float32) # Encoder outputs, self.states = tf.contrib.legacy_seq2seq.rnn_decoder( dec_in, enc_state, cell, loop_function=lf) # Decoder elif architecture == "tied": outputs, self.states = tf.contrib.legacy_seq2seq.tied_rnn_seq2seq( enc_in, dec_in, cell, loop_function=lf) else: raise (ValueError, "Unknown architecture: %s" % architecture) if fft: # compute the inverse fft on the outputs and restore the shape. spec_out = tf.reshape(tf.stack(outputs, -1), [ batch_size, self.input_size, fft_unique_bins_dec, len(outputs) ]) spec_out = tf.transpose(spec_out, [0, 1, 3, 2]) if w: iw = tf.contrib.signal.inverse_stft_window_fn( step_size, forward_window_fn=w) else: iw = None outputs = tfsignal.inverse_stft(spec_out, window_size, step_size, window_fn=iw) if center and pad_amount > 0: outputs = outputs[:, :, pad_amount // 2:-pad_amount // 2] outputs.set_shape([None, self.input_size, target_seq_len]) outputs = tf.unstack(outputs, axis=-1, name='result_unstack') self.outputs = outputs with tf.name_scope("loss_angles"): loss_angles = tf.reduce_mean( tf.square(tf.subtract(dec_out, outputs))) self.loss = loss_angles self.loss_summary = tf.summary.scalar('loss/loss', self.loss) # Gradients and SGD update operation for training the model. params = tf.trainable_variables() # Original algorithm. if custom_opt: # Wisdoms modification. opt = RMSpropNatGrad(self.learning_rate, global_step=self.global_step) else: opt = tf.train.GradientDescentOptimizer(self.learning_rate) # Update all the trainable parameters gradients = tf.gradients(self.loss, params) clipped_gradients, norm = tf.clip_by_global_norm( gradients, max_gradient_norm) self.gradient_norms = norm self.updates = opt.apply_gradients(zip(clipped_gradients, params), global_step=self.global_step) # Keep track of the learning rate self.learning_rate_summary = tf.summary.scalar( 'learning_rate/learning_rate', self.learning_rate) # === variables for loss in Euler Angles -- for each action with tf.name_scope("euler_error_walking"): self.walking_err80 = tf.placeholder(tf.float32, name="walking_srnn_seeds_0080") self.walking_err160 = tf.placeholder( tf.float32, name="walking_srnn_seeds_0160") self.walking_err320 = tf.placeholder( tf.float32, name="walking_srnn_seeds_0320") self.walking_err400 = tf.placeholder( tf.float32, name="walking_srnn_seeds_0400") self.walking_err560 = tf.placeholder( tf.float32, name="walking_srnn_seeds_0560") self.walking_err1000 = tf.placeholder( tf.float32, name="walking_srnn_seeds_1000") self.walking_err80_summary = tf.summary.scalar( 'euler_error_walking/srnn_seeds_0080', self.walking_err80) self.walking_err160_summary = tf.summary.scalar( 'euler_error_walking/srnn_seeds_0160', self.walking_err160) self.walking_err320_summary = tf.summary.scalar( 'euler_error_walking/srnn_seeds_0320', self.walking_err320) self.walking_err400_summary = tf.summary.scalar( 'euler_error_walking/srnn_seeds_0400', self.walking_err400) self.walking_err560_summary = tf.summary.scalar( 'euler_error_walking/srnn_seeds_0560', self.walking_err560) self.walking_err1000_summary = tf.summary.scalar( 'euler_error_walking/srnn_seeds_1000', self.walking_err1000) with tf.name_scope("euler_error_eating"): self.eating_err80 = tf.placeholder(tf.float32, name="eating_srnn_seeds_0080") self.eating_err160 = tf.placeholder(tf.float32, name="eating_srnn_seeds_0160") self.eating_err320 = tf.placeholder(tf.float32, name="eating_srnn_seeds_0320") self.eating_err400 = tf.placeholder(tf.float32, name="eating_srnn_seeds_0400") self.eating_err560 = tf.placeholder(tf.float32, name="eating_srnn_seeds_0560") self.eating_err1000 = tf.placeholder(tf.float32, name="eating_srnn_seeds_1000") self.eating_err80_summary = tf.summary.scalar( 'euler_error_eating/srnn_seeds_0080', self.eating_err80) self.eating_err160_summary = tf.summary.scalar( 'euler_error_eating/srnn_seeds_0160', self.eating_err160) self.eating_err320_summary = tf.summary.scalar( 'euler_error_eating/srnn_seeds_0320', self.eating_err320) self.eating_err400_summary = tf.summary.scalar( 'euler_error_eating/srnn_seeds_0400', self.eating_err400) self.eating_err560_summary = tf.summary.scalar( 'euler_error_eating/srnn_seeds_0560', self.eating_err560) self.eating_err1000_summary = tf.summary.scalar( 'euler_error_eating/srnn_seeds_1000', self.eating_err1000) with tf.name_scope("euler_error_smoking"): self.smoking_err80 = tf.placeholder(tf.float32, name="smoking_srnn_seeds_0080") self.smoking_err160 = tf.placeholder( tf.float32, name="smoking_srnn_seeds_0160") self.smoking_err320 = tf.placeholder( tf.float32, name="smoking_srnn_seeds_0320") self.smoking_err400 = tf.placeholder( tf.float32, name="smoking_srnn_seeds_0400") self.smoking_err560 = tf.placeholder( tf.float32, name="smoking_srnn_seeds_0560") self.smoking_err1000 = tf.placeholder( tf.float32, name="smoking_srnn_seeds_1000") self.smoking_err80_summary = tf.summary.scalar( 'euler_error_smoking/srnn_seeds_0080', self.smoking_err80) self.smoking_err160_summary = tf.summary.scalar( 'euler_error_smoking/srnn_seeds_0160', self.smoking_err160) self.smoking_err320_summary = tf.summary.scalar( 'euler_error_smoking/srnn_seeds_0320', self.smoking_err320) self.smoking_err400_summary = tf.summary.scalar( 'euler_error_smoking/srnn_seeds_0400', self.smoking_err400) self.smoking_err560_summary = tf.summary.scalar( 'euler_error_smoking/srnn_seeds_0560', self.smoking_err560) self.smoking_err1000_summary = tf.summary.scalar( 'euler_error_smoking/srnn_seeds_1000', self.smoking_err1000) with tf.name_scope("euler_error_discussion"): self.discussion_err80 = tf.placeholder( tf.float32, name="discussion_srnn_seeds_0080") self.discussion_err160 = tf.placeholder( tf.float32, name="discussion_srnn_seeds_0160") self.discussion_err320 = tf.placeholder( tf.float32, name="discussion_srnn_seeds_0320") self.discussion_err400 = tf.placeholder( tf.float32, name="discussion_srnn_seeds_0400") self.discussion_err560 = tf.placeholder( tf.float32, name="discussion_srnn_seeds_0560") self.discussion_err1000 = tf.placeholder( tf.float32, name="discussion_srnn_seeds_1000") self.discussion_err80_summary = tf.summary.scalar( 'euler_error_discussion/srnn_seeds_0080', self.discussion_err80) self.discussion_err160_summary = tf.summary.scalar( 'euler_error_discussion/srnn_seeds_0160', self.discussion_err160) self.discussion_err320_summary = tf.summary.scalar( 'euler_error_discussion/srnn_seeds_0320', self.discussion_err320) self.discussion_err400_summary = tf.summary.scalar( 'euler_error_discussion/srnn_seeds_0400', self.discussion_err400) self.discussion_err560_summary = tf.summary.scalar( 'euler_error_discussion/srnn_seeds_0560', self.discussion_err560) self.discussion_err1000_summary = tf.summary.scalar( 'euler_error_discussion/srnn_seeds_1000', self.discussion_err1000) with tf.name_scope("euler_error_directions"): self.directions_err80 = tf.placeholder( tf.float32, name="directions_srnn_seeds_0080") self.directions_err160 = tf.placeholder( tf.float32, name="directions_srnn_seeds_0160") self.directions_err320 = tf.placeholder( tf.float32, name="directions_srnn_seeds_0320") self.directions_err400 = tf.placeholder( tf.float32, name="directions_srnn_seeds_0400") self.directions_err560 = tf.placeholder( tf.float32, name="directions_srnn_seeds_0560") self.directions_err1000 = tf.placeholder( tf.float32, name="directions_srnn_seeds_1000") self.directions_err80_summary = tf.summary.scalar( 'euler_error_directions/srnn_seeds_0080', self.directions_err80) self.directions_err160_summary = tf.summary.scalar( 'euler_error_directions/srnn_seeds_0160', self.directions_err160) self.directions_err320_summary = tf.summary.scalar( 'euler_error_directions/srnn_seeds_0320', self.directions_err320) self.directions_err400_summary = tf.summary.scalar( 'euler_error_directions/srnn_seeds_0400', self.directions_err400) self.directions_err560_summary = tf.summary.scalar( 'euler_error_directions/srnn_seeds_0560', self.directions_err560) self.directions_err1000_summary = tf.summary.scalar( 'euler_error_directions/srnn_seeds_1000', self.directions_err1000) with tf.name_scope("euler_error_greeting"): self.greeting_err80 = tf.placeholder( tf.float32, name="greeting_srnn_seeds_0080") self.greeting_err160 = tf.placeholder( tf.float32, name="greeting_srnn_seeds_0160") self.greeting_err320 = tf.placeholder( tf.float32, name="greeting_srnn_seeds_0320") self.greeting_err400 = tf.placeholder( tf.float32, name="greeting_srnn_seeds_0400") self.greeting_err560 = tf.placeholder( tf.float32, name="greeting_srnn_seeds_0560") self.greeting_err1000 = tf.placeholder( tf.float32, name="greeting_srnn_seeds_1000") self.greeting_err80_summary = tf.summary.scalar( 'euler_error_greeting/srnn_seeds_0080', self.greeting_err80) self.greeting_err160_summary = tf.summary.scalar( 'euler_error_greeting/srnn_seeds_0160', self.greeting_err160) self.greeting_err320_summary = tf.summary.scalar( 'euler_error_greeting/srnn_seeds_0320', self.greeting_err320) self.greeting_err400_summary = tf.summary.scalar( 'euler_error_greeting/srnn_seeds_0400', self.greeting_err400) self.greeting_err560_summary = tf.summary.scalar( 'euler_error_greeting/srnn_seeds_0560', self.greeting_err560) self.greeting_err1000_summary = tf.summary.scalar( 'euler_error_greeting/srnn_seeds_1000', self.greeting_err1000) with tf.name_scope("euler_error_phoning"): self.phoning_err80 = tf.placeholder(tf.float32, name="phoning_srnn_seeds_0080") self.phoning_err160 = tf.placeholder( tf.float32, name="phoning_srnn_seeds_0160") self.phoning_err320 = tf.placeholder( tf.float32, name="phoning_srnn_seeds_0320") self.phoning_err400 = tf.placeholder( tf.float32, name="phoning_srnn_seeds_0400") self.phoning_err560 = tf.placeholder( tf.float32, name="phoning_srnn_seeds_0560") self.phoning_err1000 = tf.placeholder( tf.float32, name="phoning_srnn_seeds_1000") self.phoning_err80_summary = tf.summary.scalar( 'euler_error_phoning/srnn_seeds_0080', self.phoning_err80) self.phoning_err160_summary = tf.summary.scalar( 'euler_error_phoning/srnn_seeds_0160', self.phoning_err160) self.phoning_err320_summary = tf.summary.scalar( 'euler_error_phoning/srnn_seeds_0320', self.phoning_err320) self.phoning_err400_summary = tf.summary.scalar( 'euler_error_phoning/srnn_seeds_0400', self.phoning_err400) self.phoning_err560_summary = tf.summary.scalar( 'euler_error_phoning/srnn_seeds_0560', self.phoning_err560) self.phoning_err1000_summary = tf.summary.scalar( 'euler_error_phoning/srnn_seeds_1000', self.phoning_err1000) with tf.name_scope("euler_error_posing"): self.posing_err80 = tf.placeholder(tf.float32, name="posing_srnn_seeds_0080") self.posing_err160 = tf.placeholder(tf.float32, name="posing_srnn_seeds_0160") self.posing_err320 = tf.placeholder(tf.float32, name="posing_srnn_seeds_0320") self.posing_err400 = tf.placeholder(tf.float32, name="posing_srnn_seeds_0400") self.posing_err560 = tf.placeholder(tf.float32, name="posing_srnn_seeds_0560") self.posing_err1000 = tf.placeholder(tf.float32, name="posing_srnn_seeds_1000") self.posing_err80_summary = tf.summary.scalar( 'euler_error_posing/srnn_seeds_0080', self.posing_err80) self.posing_err160_summary = tf.summary.scalar( 'euler_error_posing/srnn_seeds_0160', self.posing_err160) self.posing_err320_summary = tf.summary.scalar( 'euler_error_posing/srnn_seeds_0320', self.posing_err320) self.posing_err400_summary = tf.summary.scalar( 'euler_error_posing/srnn_seeds_0400', self.posing_err400) self.posing_err560_summary = tf.summary.scalar( 'euler_error_posing/srnn_seeds_0560', self.posing_err560) self.posing_err1000_summary = tf.summary.scalar( 'euler_error_posing/srnn_seeds_1000', self.posing_err1000) with tf.name_scope("euler_error_purchases"): self.purchases_err80 = tf.placeholder( tf.float32, name="purchases_srnn_seeds_0080") self.purchases_err160 = tf.placeholder( tf.float32, name="purchases_srnn_seeds_0160") self.purchases_err320 = tf.placeholder( tf.float32, name="purchases_srnn_seeds_0320") self.purchases_err400 = tf.placeholder( tf.float32, name="purchases_srnn_seeds_0400") self.purchases_err560 = tf.placeholder( tf.float32, name="purchases_srnn_seeds_0560") self.purchases_err1000 = tf.placeholder( tf.float32, name="purchases_srnn_seeds_1000") self.purchases_err80_summary = tf.summary.scalar( 'euler_error_purchases/srnn_seeds_0080', self.purchases_err80) self.purchases_err160_summary = tf.summary.scalar( 'euler_error_purchases/srnn_seeds_0160', self.purchases_err160) self.purchases_err320_summary = tf.summary.scalar( 'euler_error_purchases/srnn_seeds_0320', self.purchases_err320) self.purchases_err400_summary = tf.summary.scalar( 'euler_error_purchases/srnn_seeds_0400', self.purchases_err400) self.purchases_err560_summary = tf.summary.scalar( 'euler_error_purchases/srnn_seeds_0560', self.purchases_err560) self.purchases_err1000_summary = tf.summary.scalar( 'euler_error_purchases/srnn_seeds_1000', self.purchases_err1000) with tf.name_scope("euler_error_sitting"): self.sitting_err80 = tf.placeholder(tf.float32, name="sitting_srnn_seeds_0080") self.sitting_err160 = tf.placeholder( tf.float32, name="sitting_srnn_seeds_0160") self.sitting_err320 = tf.placeholder( tf.float32, name="sitting_srnn_seeds_0320") self.sitting_err400 = tf.placeholder( tf.float32, name="sitting_srnn_seeds_0400") self.sitting_err560 = tf.placeholder( tf.float32, name="sitting_srnn_seeds_0560") self.sitting_err1000 = tf.placeholder( tf.float32, name="sitting_srnn_seeds_1000") self.sitting_err80_summary = tf.summary.scalar( 'euler_error_sitting/srnn_seeds_0080', self.sitting_err80) self.sitting_err160_summary = tf.summary.scalar( 'euler_error_sitting/srnn_seeds_0160', self.sitting_err160) self.sitting_err320_summary = tf.summary.scalar( 'euler_error_sitting/srnn_seeds_0320', self.sitting_err320) self.sitting_err400_summary = tf.summary.scalar( 'euler_error_sitting/srnn_seeds_0400', self.sitting_err400) self.sitting_err560_summary = tf.summary.scalar( 'euler_error_sitting/srnn_seeds_0560', self.sitting_err560) self.sitting_err1000_summary = tf.summary.scalar( 'euler_error_sitting/srnn_seeds_1000', self.sitting_err1000) with tf.name_scope("euler_error_sittingdown"): self.sittingdown_err80 = tf.placeholder( tf.float32, name="sittingdown_srnn_seeds_0080") self.sittingdown_err160 = tf.placeholder( tf.float32, name="sittingdown_srnn_seeds_0160") self.sittingdown_err320 = tf.placeholder( tf.float32, name="sittingdown_srnn_seeds_0320") self.sittingdown_err400 = tf.placeholder( tf.float32, name="sittingdown_srnn_seeds_0400") self.sittingdown_err560 = tf.placeholder( tf.float32, name="sittingdown_srnn_seeds_0560") self.sittingdown_err1000 = tf.placeholder( tf.float32, name="sittingdown_srnn_seeds_1000") self.sittingdown_err80_summary = tf.summary.scalar( 'euler_error_sittingdown/srnn_seeds_0080', self.sittingdown_err80) self.sittingdown_err160_summary = tf.summary.scalar( 'euler_error_sittingdown/srnn_seeds_0160', self.sittingdown_err160) self.sittingdown_err320_summary = tf.summary.scalar( 'euler_error_sittingdown/srnn_seeds_0320', self.sittingdown_err320) self.sittingdown_err400_summary = tf.summary.scalar( 'euler_error_sittingdown/srnn_seeds_0400', self.sittingdown_err400) self.sittingdown_err560_summary = tf.summary.scalar( 'euler_error_sittingdown/srnn_seeds_0560', self.sittingdown_err560) self.sittingdown_err1000_summary = tf.summary.scalar( 'euler_error_sittingdown/srnn_seeds_1000', self.sittingdown_err1000) with tf.name_scope("euler_error_takingphoto"): self.takingphoto_err80 = tf.placeholder( tf.float32, name="takingphoto_srnn_seeds_0080") self.takingphoto_err160 = tf.placeholder( tf.float32, name="takingphoto_srnn_seeds_0160") self.takingphoto_err320 = tf.placeholder( tf.float32, name="takingphoto_srnn_seeds_0320") self.takingphoto_err400 = tf.placeholder( tf.float32, name="takingphoto_srnn_seeds_0400") self.takingphoto_err560 = tf.placeholder( tf.float32, name="takingphoto_srnn_seeds_0560") self.takingphoto_err1000 = tf.placeholder( tf.float32, name="takingphoto_srnn_seeds_1000") self.takingphoto_err80_summary = tf.summary.scalar( 'euler_error_takingphoto/srnn_seeds_0080', self.takingphoto_err80) self.takingphoto_err160_summary = tf.summary.scalar( 'euler_error_takingphoto/srnn_seeds_0160', self.takingphoto_err160) self.takingphoto_err320_summary = tf.summary.scalar( 'euler_error_takingphoto/srnn_seeds_0320', self.takingphoto_err320) self.takingphoto_err400_summary = tf.summary.scalar( 'euler_error_takingphoto/srnn_seeds_0400', self.takingphoto_err400) self.takingphoto_err560_summary = tf.summary.scalar( 'euler_error_takingphoto/srnn_seeds_0560', self.takingphoto_err560) self.takingphoto_err1000_summary = tf.summary.scalar( 'euler_error_takingphoto/srnn_seeds_1000', self.takingphoto_err1000) with tf.name_scope("euler_error_waiting"): self.waiting_err80 = tf.placeholder(tf.float32, name="waiting_srnn_seeds_0080") self.waiting_err160 = tf.placeholder( tf.float32, name="waiting_srnn_seeds_0160") self.waiting_err320 = tf.placeholder( tf.float32, name="waiting_srnn_seeds_0320") self.waiting_err400 = tf.placeholder( tf.float32, name="waiting_srnn_seeds_0400") self.waiting_err560 = tf.placeholder( tf.float32, name="waiting_srnn_seeds_0560") self.waiting_err1000 = tf.placeholder( tf.float32, name="waiting_srnn_seeds_1000") self.waiting_err80_summary = tf.summary.scalar( 'euler_error_waiting/srnn_seeds_0080', self.waiting_err80) self.waiting_err160_summary = tf.summary.scalar( 'euler_error_waiting/srnn_seeds_0160', self.waiting_err160) self.waiting_err320_summary = tf.summary.scalar( 'euler_error_waiting/srnn_seeds_0320', self.waiting_err320) self.waiting_err400_summary = tf.summary.scalar( 'euler_error_waiting/srnn_seeds_0400', self.waiting_err400) self.waiting_err560_summary = tf.summary.scalar( 'euler_error_waiting/srnn_seeds_0560', self.waiting_err560) self.waiting_err1000_summary = tf.summary.scalar( 'euler_error_waiting/srnn_seeds_1000', self.waiting_err1000) with tf.name_scope("euler_error_walkingdog"): self.walkingdog_err80 = tf.placeholder( tf.float32, name="walkingdog_srnn_seeds_0080") self.walkingdog_err160 = tf.placeholder( tf.float32, name="walkingdog_srnn_seeds_0160") self.walkingdog_err320 = tf.placeholder( tf.float32, name="walkingdog_srnn_seeds_0320") self.walkingdog_err400 = tf.placeholder( tf.float32, name="walkingdog_srnn_seeds_0400") self.walkingdog_err560 = tf.placeholder( tf.float32, name="walkingdog_srnn_seeds_0560") self.walkingdog_err1000 = tf.placeholder( tf.float32, name="walkingdog_srnn_seeds_1000") self.walkingdog_err80_summary = tf.summary.scalar( 'euler_error_walkingdog/srnn_seeds_0080', self.walkingdog_err80) self.walkingdog_err160_summary = tf.summary.scalar( 'euler_error_walkingdog/srnn_seeds_0160', self.walkingdog_err160) self.walkingdog_err320_summary = tf.summary.scalar( 'euler_error_walkingdog/srnn_seeds_0320', self.walkingdog_err320) self.walkingdog_err400_summary = tf.summary.scalar( 'euler_error_walkingdog/srnn_seeds_0400', self.walkingdog_err400) self.walkingdog_err560_summary = tf.summary.scalar( 'euler_error_walkingdog/srnn_seeds_0560', self.walkingdog_err560) self.walkingdog_err1000_summary = tf.summary.scalar( 'euler_error_walkingdog/srnn_seeds_1000', self.walkingdog_err1000) with tf.name_scope("euler_error_walkingtogether"): self.walkingtogether_err80 = tf.placeholder( tf.float32, name="walkingtogether_srnn_seeds_0080") self.walkingtogether_err160 = tf.placeholder( tf.float32, name="walkingtogether_srnn_seeds_0160") self.walkingtogether_err320 = tf.placeholder( tf.float32, name="walkingtogether_srnn_seeds_0320") self.walkingtogether_err400 = tf.placeholder( tf.float32, name="walkingtogether_srnn_seeds_0400") self.walkingtogether_err560 = tf.placeholder( tf.float32, name="walkingtogether_srnn_seeds_0560") self.walkingtogether_err1000 = tf.placeholder( tf.float32, name="walkingtogether_srnn_seeds_1000") self.walkingtogether_err80_summary = tf.summary.scalar( 'euler_error_walkingtogether/srnn_seeds_0080', self.walkingtogether_err80) self.walkingtogether_err160_summary = tf.summary.scalar( 'euler_error_walkingtogether/srnn_seeds_0160', self.walkingtogether_err160) self.walkingtogether_err320_summary = tf.summary.scalar( 'euler_error_walkingtogether/srnn_seeds_0320', self.walkingtogether_err320) self.walkingtogether_err400_summary = tf.summary.scalar( 'euler_error_walkingtogether/srnn_seeds_0400', self.walkingtogether_err400) self.walkingtogether_err560_summary = tf.summary.scalar( 'euler_error_walkingtogether/srnn_seeds_0560', self.walkingtogether_err560) self.walkingtogether_err1000_summary = tf.summary.scalar( 'euler_error_walkingtogether/srnn_seeds_1000', self.walkingtogether_err1000) self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=10)
def __init__(self, architecture, source_seq_len, target_seq_len, rnn_size, # hidden recurrent layer size num_layers, max_gradient_norm, batch_size, learning_rate, learning_rate_decay_factor, summaries_dir, loss_to_use, number_of_actions, one_hot=True, residual_velocities=False, dtype=tf.float32): """Create the model. Args: architecture: [basic, tied] whether to tie the decoder and decoder. source_seq_len: lenght of the input sequence. target_seq_len: lenght of the target sequence. rnn_size: number of units in the rnn. num_layers: number of rnns to stack. max_gradient_norm: gradients will be clipped to maximally this norm. batch_size: the size of the batches used during training; the model construction is independent of batch_size, so it can be changed after initialization if this is convenient, e.g., for decoding. learning_rate: learning rate to start with. learning_rate_decay_factor: decay learning rate by this much when needed. summaries_dir: where to log progress for tensorboard. loss_to_use: [supervised, sampling_based]. Whether to use ground truth in each timestep to compute the loss after decoding, or to feed back the prediction from the previous time-step. number_of_actions: number of classes we have. one_hot: whether to use one_hot encoding during train/test (sup models). residual_velocities: whether to use a residual connection that models velocities. dtype: the data type to use to store internal variables. """ self.HUMAN_SIZE = 54 self.input_size = self.HUMAN_SIZE + number_of_actions if one_hot else self.HUMAN_SIZE print( "One hot is ", one_hot ) print( "Input size is %d" % self.input_size ) # Summary writers for train and test runs self.train_writer = tf.summary.FileWriter(os.path.join( summaries_dir, 'train')) self.test_writer = tf.summary.FileWriter(os.path.join( summaries_dir, 'test')) self.source_seq_len = source_seq_len self.target_seq_len = target_seq_len self.rnn_size = rnn_size self.batch_size = batch_size self.learning_rate = tf.Variable( float(learning_rate), trainable=False, dtype=dtype ) self.learning_rate_decay_op = self.learning_rate.assign( self.learning_rate * learning_rate_decay_factor ) self.global_step = tf.Variable(0, trainable=False) # === Create the RNN that will keep the state === print('rnn_size = {0}'.format( rnn_size )) single_cell = tf.contrib.rnn.GRUCell( self.rnn_size ) if num_layers > 1: # Create multiple rnn layers cell = tf.contrib.rnn.MultiRNNCell( [single_cell] * num_layers ) else: cell = single_cell # === Transform the inputs === with tf.name_scope("inputs"): enc_in = tf.placeholder(dtype, shape=[None, source_seq_len-1, self.input_size], name="enc_in") dec_in = tf.placeholder(dtype, shape=[None, target_seq_len, self.input_size], name="dec_in") dec_out = tf.placeholder(dtype, shape=[None, target_seq_len, self.input_size], name="dec_out") self.encoder_inputs = enc_in self.decoder_inputs = dec_in self.decoder_outputs = dec_out enc_in = tf.transpose(enc_in, [1, 0, 2]) dec_in = tf.transpose(dec_in, [1, 0, 2]) dec_out = tf.transpose(dec_out, [1, 0, 2]) enc_in = tf.reshape(enc_in, [-1, self.input_size]) dec_in = tf.reshape(dec_in, [-1, self.input_size]) dec_out = tf.reshape(dec_out, [-1, self.input_size]) enc_in = tf.split(enc_in, source_seq_len-1, axis=0) dec_in = tf.split(dec_in, target_seq_len, axis=0) dec_out = tf.split(dec_out, target_seq_len, axis=0) # === Add space decoder === cell = rnn_cell_extensions.LinearSpaceDecoderWrapper( cell, self.input_size ) # Finally, wrap everything in a residual layer if we want to model velocities if residual_velocities: cell = rnn_cell_extensions.ResidualWrapper( cell ) # Store the outputs here outputs = [] # Define the loss function lf = None if loss_to_use == "sampling_based": def lf(prev, i): # function for sampling_based loss return prev elif loss_to_use == "supervised": pass else: raise(ValueError, "unknown loss: %s" % loss_to_use) # Build the RNN if architecture == "basic": # Basic RNN does not have a loop function in its API, so copying here. with vs.variable_scope("basic_rnn_seq2seq"): _, enc_state = tf.contrib.rnn.static_rnn(cell, enc_in, dtype=tf.float32) # Encoder outputs, self.states = tf.contrib.legacy_seq2seq.rnn_decoder( dec_in, enc_state, cell, loop_function=lf ) # Decoder elif architecture == "tied": outputs, self.states = tf.contrib.legacy_seq2seq.tied_rnn_seq2seq( enc_in, dec_in, cell, loop_function=lf ) else: raise(ValueError, "Uknown architecture: %s" % architecture ) self.outputs = outputs with tf.name_scope("loss_angles"): loss_angles = tf.reduce_mean(tf.square(tf.subtract(dec_out, outputs))) self.loss = loss_angles self.loss_summary = tf.summary.scalar('loss/loss', self.loss) # Gradients and SGD update operation for training the model. params = tf.trainable_variables() opt = tf.train.GradientDescentOptimizer( self.learning_rate ) # Update all the trainable parameters gradients = tf.gradients( self.loss, params ) clipped_gradients, norm = tf.clip_by_global_norm(gradients, max_gradient_norm) self.gradient_norms = norm self.updates = opt.apply_gradients( zip(clipped_gradients, params), global_step=self.global_step) # Keep track of the learning rate self.learning_rate_summary = tf.summary.scalar('learning_rate/learning_rate', self.learning_rate) # === variables for loss in Euler Angles -- for each action with tf.name_scope( "euler_error_walking" ): self.walking_err80 = tf.placeholder( tf.float32, name="walking_srnn_seeds_0080" ) self.walking_err160 = tf.placeholder( tf.float32, name="walking_srnn_seeds_0160" ) self.walking_err320 = tf.placeholder( tf.float32, name="walking_srnn_seeds_0320" ) self.walking_err400 = tf.placeholder( tf.float32, name="walking_srnn_seeds_0400" ) self.walking_err560 = tf.placeholder( tf.float32, name="walking_srnn_seeds_0560" ) self.walking_err1000 = tf.placeholder( tf.float32, name="walking_srnn_seeds_1000" ) self.walking_err80_summary = tf.summary.scalar( 'euler_error_walking/srnn_seeds_0080', self.walking_err80 ) self.walking_err160_summary = tf.summary.scalar( 'euler_error_walking/srnn_seeds_0160', self.walking_err160 ) self.walking_err320_summary = tf.summary.scalar( 'euler_error_walking/srnn_seeds_0320', self.walking_err320 ) self.walking_err400_summary = tf.summary.scalar( 'euler_error_walking/srnn_seeds_0400', self.walking_err400 ) self.walking_err560_summary = tf.summary.scalar( 'euler_error_walking/srnn_seeds_0560', self.walking_err560 ) self.walking_err1000_summary = tf.summary.scalar( 'euler_error_walking/srnn_seeds_1000', self.walking_err1000 ) with tf.name_scope( "euler_error_eating" ): self.eating_err80 = tf.placeholder( tf.float32, name="eating_srnn_seeds_0080" ) self.eating_err160 = tf.placeholder( tf.float32, name="eating_srnn_seeds_0160" ) self.eating_err320 = tf.placeholder( tf.float32, name="eating_srnn_seeds_0320" ) self.eating_err400 = tf.placeholder( tf.float32, name="eating_srnn_seeds_0400" ) self.eating_err560 = tf.placeholder( tf.float32, name="eating_srnn_seeds_0560" ) self.eating_err1000 = tf.placeholder( tf.float32, name="eating_srnn_seeds_1000" ) self.eating_err80_summary = tf.summary.scalar( 'euler_error_eating/srnn_seeds_0080', self.eating_err80 ) self.eating_err160_summary = tf.summary.scalar( 'euler_error_eating/srnn_seeds_0160', self.eating_err160 ) self.eating_err320_summary = tf.summary.scalar( 'euler_error_eating/srnn_seeds_0320', self.eating_err320 ) self.eating_err400_summary = tf.summary.scalar( 'euler_error_eating/srnn_seeds_0400', self.eating_err400 ) self.eating_err560_summary = tf.summary.scalar( 'euler_error_eating/srnn_seeds_0560', self.eating_err560 ) self.eating_err1000_summary = tf.summary.scalar( 'euler_error_eating/srnn_seeds_1000', self.eating_err1000 ) with tf.name_scope( "euler_error_smoking" ): self.smoking_err80 = tf.placeholder( tf.float32, name="smoking_srnn_seeds_0080" ) self.smoking_err160 = tf.placeholder( tf.float32, name="smoking_srnn_seeds_0160" ) self.smoking_err320 = tf.placeholder( tf.float32, name="smoking_srnn_seeds_0320" ) self.smoking_err400 = tf.placeholder( tf.float32, name="smoking_srnn_seeds_0400" ) self.smoking_err560 = tf.placeholder( tf.float32, name="smoking_srnn_seeds_0560" ) self.smoking_err1000 = tf.placeholder( tf.float32, name="smoking_srnn_seeds_1000" ) self.smoking_err80_summary = tf.summary.scalar( 'euler_error_smoking/srnn_seeds_0080', self.smoking_err80 ) self.smoking_err160_summary = tf.summary.scalar( 'euler_error_smoking/srnn_seeds_0160', self.smoking_err160 ) self.smoking_err320_summary = tf.summary.scalar( 'euler_error_smoking/srnn_seeds_0320', self.smoking_err320 ) self.smoking_err400_summary = tf.summary.scalar( 'euler_error_smoking/srnn_seeds_0400', self.smoking_err400 ) self.smoking_err560_summary = tf.summary.scalar( 'euler_error_smoking/srnn_seeds_0560', self.smoking_err560 ) self.smoking_err1000_summary = tf.summary.scalar( 'euler_error_smoking/srnn_seeds_1000', self.smoking_err1000 ) with tf.name_scope( "euler_error_discussion" ): self.discussion_err80 = tf.placeholder( tf.float32, name="discussion_srnn_seeds_0080" ) self.discussion_err160 = tf.placeholder( tf.float32, name="discussion_srnn_seeds_0160" ) self.discussion_err320 = tf.placeholder( tf.float32, name="discussion_srnn_seeds_0320" ) self.discussion_err400 = tf.placeholder( tf.float32, name="discussion_srnn_seeds_0400" ) self.discussion_err560 = tf.placeholder( tf.float32, name="discussion_srnn_seeds_0560" ) self.discussion_err1000 = tf.placeholder( tf.float32, name="discussion_srnn_seeds_1000" ) self.discussion_err80_summary = tf.summary.scalar( 'euler_error_discussion/srnn_seeds_0080', self.discussion_err80 ) self.discussion_err160_summary = tf.summary.scalar( 'euler_error_discussion/srnn_seeds_0160', self.discussion_err160 ) self.discussion_err320_summary = tf.summary.scalar( 'euler_error_discussion/srnn_seeds_0320', self.discussion_err320 ) self.discussion_err400_summary = tf.summary.scalar( 'euler_error_discussion/srnn_seeds_0400', self.discussion_err400 ) self.discussion_err560_summary = tf.summary.scalar( 'euler_error_discussion/srnn_seeds_0560', self.discussion_err560 ) self.discussion_err1000_summary = tf.summary.scalar( 'euler_error_discussion/srnn_seeds_1000', self.discussion_err1000 ) with tf.name_scope( "euler_error_directions" ): self.directions_err80 = tf.placeholder( tf.float32, name="directions_srnn_seeds_0080" ) self.directions_err160 = tf.placeholder( tf.float32, name="directions_srnn_seeds_0160" ) self.directions_err320 = tf.placeholder( tf.float32, name="directions_srnn_seeds_0320" ) self.directions_err400 = tf.placeholder( tf.float32, name="directions_srnn_seeds_0400" ) self.directions_err560 = tf.placeholder( tf.float32, name="directions_srnn_seeds_0560" ) self.directions_err1000 = tf.placeholder( tf.float32, name="directions_srnn_seeds_1000" ) self.directions_err80_summary = tf.summary.scalar( 'euler_error_directions/srnn_seeds_0080', self.directions_err80 ) self.directions_err160_summary = tf.summary.scalar( 'euler_error_directions/srnn_seeds_0160', self.directions_err160 ) self.directions_err320_summary = tf.summary.scalar( 'euler_error_directions/srnn_seeds_0320', self.directions_err320 ) self.directions_err400_summary = tf.summary.scalar( 'euler_error_directions/srnn_seeds_0400', self.directions_err400 ) self.directions_err560_summary = tf.summary.scalar( 'euler_error_directions/srnn_seeds_0560', self.directions_err560 ) self.directions_err1000_summary = tf.summary.scalar( 'euler_error_directions/srnn_seeds_1000', self.directions_err1000 ) with tf.name_scope( "euler_error_greeting" ): self.greeting_err80 = tf.placeholder( tf.float32, name="greeting_srnn_seeds_0080" ) self.greeting_err160 = tf.placeholder( tf.float32, name="greeting_srnn_seeds_0160" ) self.greeting_err320 = tf.placeholder( tf.float32, name="greeting_srnn_seeds_0320" ) self.greeting_err400 = tf.placeholder( tf.float32, name="greeting_srnn_seeds_0400" ) self.greeting_err560 = tf.placeholder( tf.float32, name="greeting_srnn_seeds_0560" ) self.greeting_err1000 = tf.placeholder( tf.float32, name="greeting_srnn_seeds_1000" ) self.greeting_err80_summary = tf.summary.scalar( 'euler_error_greeting/srnn_seeds_0080', self.greeting_err80 ) self.greeting_err160_summary = tf.summary.scalar( 'euler_error_greeting/srnn_seeds_0160', self.greeting_err160 ) self.greeting_err320_summary = tf.summary.scalar( 'euler_error_greeting/srnn_seeds_0320', self.greeting_err320 ) self.greeting_err400_summary = tf.summary.scalar( 'euler_error_greeting/srnn_seeds_0400', self.greeting_err400 ) self.greeting_err560_summary = tf.summary.scalar( 'euler_error_greeting/srnn_seeds_0560', self.greeting_err560 ) self.greeting_err1000_summary = tf.summary.scalar( 'euler_error_greeting/srnn_seeds_1000', self.greeting_err1000 ) with tf.name_scope( "euler_error_phoning" ): self.phoning_err80 = tf.placeholder( tf.float32, name="phoning_srnn_seeds_0080" ) self.phoning_err160 = tf.placeholder( tf.float32, name="phoning_srnn_seeds_0160" ) self.phoning_err320 = tf.placeholder( tf.float32, name="phoning_srnn_seeds_0320" ) self.phoning_err400 = tf.placeholder( tf.float32, name="phoning_srnn_seeds_0400" ) self.phoning_err560 = tf.placeholder( tf.float32, name="phoning_srnn_seeds_0560" ) self.phoning_err1000 = tf.placeholder( tf.float32, name="phoning_srnn_seeds_1000" ) self.phoning_err80_summary = tf.summary.scalar( 'euler_error_phoning/srnn_seeds_0080', self.phoning_err80 ) self.phoning_err160_summary = tf.summary.scalar( 'euler_error_phoning/srnn_seeds_0160', self.phoning_err160 ) self.phoning_err320_summary = tf.summary.scalar( 'euler_error_phoning/srnn_seeds_0320', self.phoning_err320 ) self.phoning_err400_summary = tf.summary.scalar( 'euler_error_phoning/srnn_seeds_0400', self.phoning_err400 ) self.phoning_err560_summary = tf.summary.scalar( 'euler_error_phoning/srnn_seeds_0560', self.phoning_err560 ) self.phoning_err1000_summary = tf.summary.scalar( 'euler_error_phoning/srnn_seeds_1000', self.phoning_err1000 ) with tf.name_scope( "euler_error_posing" ): self.posing_err80 = tf.placeholder( tf.float32, name="posing_srnn_seeds_0080" ) self.posing_err160 = tf.placeholder( tf.float32, name="posing_srnn_seeds_0160" ) self.posing_err320 = tf.placeholder( tf.float32, name="posing_srnn_seeds_0320" ) self.posing_err400 = tf.placeholder( tf.float32, name="posing_srnn_seeds_0400" ) self.posing_err560 = tf.placeholder( tf.float32, name="posing_srnn_seeds_0560" ) self.posing_err1000 = tf.placeholder( tf.float32, name="posing_srnn_seeds_1000" ) self.posing_err80_summary = tf.summary.scalar( 'euler_error_posing/srnn_seeds_0080', self.posing_err80 ) self.posing_err160_summary = tf.summary.scalar( 'euler_error_posing/srnn_seeds_0160', self.posing_err160 ) self.posing_err320_summary = tf.summary.scalar( 'euler_error_posing/srnn_seeds_0320', self.posing_err320 ) self.posing_err400_summary = tf.summary.scalar( 'euler_error_posing/srnn_seeds_0400', self.posing_err400 ) self.posing_err560_summary = tf.summary.scalar( 'euler_error_posing/srnn_seeds_0560', self.posing_err560 ) self.posing_err1000_summary = tf.summary.scalar( 'euler_error_posing/srnn_seeds_1000', self.posing_err1000 ) with tf.name_scope( "euler_error_purchases" ): self.purchases_err80 = tf.placeholder( tf.float32, name="purchases_srnn_seeds_0080" ) self.purchases_err160 = tf.placeholder( tf.float32, name="purchases_srnn_seeds_0160" ) self.purchases_err320 = tf.placeholder( tf.float32, name="purchases_srnn_seeds_0320" ) self.purchases_err400 = tf.placeholder( tf.float32, name="purchases_srnn_seeds_0400" ) self.purchases_err560 = tf.placeholder( tf.float32, name="purchases_srnn_seeds_0560" ) self.purchases_err1000 = tf.placeholder( tf.float32, name="purchases_srnn_seeds_1000" ) self.purchases_err80_summary = tf.summary.scalar( 'euler_error_purchases/srnn_seeds_0080', self.purchases_err80 ) self.purchases_err160_summary = tf.summary.scalar( 'euler_error_purchases/srnn_seeds_0160', self.purchases_err160 ) self.purchases_err320_summary = tf.summary.scalar( 'euler_error_purchases/srnn_seeds_0320', self.purchases_err320 ) self.purchases_err400_summary = tf.summary.scalar( 'euler_error_purchases/srnn_seeds_0400', self.purchases_err400 ) self.purchases_err560_summary = tf.summary.scalar( 'euler_error_purchases/srnn_seeds_0560', self.purchases_err560 ) self.purchases_err1000_summary = tf.summary.scalar( 'euler_error_purchases/srnn_seeds_1000', self.purchases_err1000 ) with tf.name_scope( "euler_error_sitting" ): self.sitting_err80 = tf.placeholder( tf.float32, name="sitting_srnn_seeds_0080" ) self.sitting_err160 = tf.placeholder( tf.float32, name="sitting_srnn_seeds_0160" ) self.sitting_err320 = tf.placeholder( tf.float32, name="sitting_srnn_seeds_0320" ) self.sitting_err400 = tf.placeholder( tf.float32, name="sitting_srnn_seeds_0400" ) self.sitting_err560 = tf.placeholder( tf.float32, name="sitting_srnn_seeds_0560" ) self.sitting_err1000 = tf.placeholder( tf.float32, name="sitting_srnn_seeds_1000" ) self.sitting_err80_summary = tf.summary.scalar( 'euler_error_sitting/srnn_seeds_0080', self.sitting_err80 ) self.sitting_err160_summary = tf.summary.scalar( 'euler_error_sitting/srnn_seeds_0160', self.sitting_err160 ) self.sitting_err320_summary = tf.summary.scalar( 'euler_error_sitting/srnn_seeds_0320', self.sitting_err320 ) self.sitting_err400_summary = tf.summary.scalar( 'euler_error_sitting/srnn_seeds_0400', self.sitting_err400 ) self.sitting_err560_summary = tf.summary.scalar( 'euler_error_sitting/srnn_seeds_0560', self.sitting_err560 ) self.sitting_err1000_summary = tf.summary.scalar( 'euler_error_sitting/srnn_seeds_1000', self.sitting_err1000 ) with tf.name_scope( "euler_error_sittingdown" ): self.sittingdown_err80 = tf.placeholder( tf.float32, name="sittingdown_srnn_seeds_0080" ) self.sittingdown_err160 = tf.placeholder( tf.float32, name="sittingdown_srnn_seeds_0160" ) self.sittingdown_err320 = tf.placeholder( tf.float32, name="sittingdown_srnn_seeds_0320" ) self.sittingdown_err400 = tf.placeholder( tf.float32, name="sittingdown_srnn_seeds_0400" ) self.sittingdown_err560 = tf.placeholder( tf.float32, name="sittingdown_srnn_seeds_0560" ) self.sittingdown_err1000 = tf.placeholder( tf.float32, name="sittingdown_srnn_seeds_1000" ) self.sittingdown_err80_summary = tf.summary.scalar( 'euler_error_sittingdown/srnn_seeds_0080', self.sittingdown_err80 ) self.sittingdown_err160_summary = tf.summary.scalar( 'euler_error_sittingdown/srnn_seeds_0160', self.sittingdown_err160 ) self.sittingdown_err320_summary = tf.summary.scalar( 'euler_error_sittingdown/srnn_seeds_0320', self.sittingdown_err320 ) self.sittingdown_err400_summary = tf.summary.scalar( 'euler_error_sittingdown/srnn_seeds_0400', self.sittingdown_err400 ) self.sittingdown_err560_summary = tf.summary.scalar( 'euler_error_sittingdown/srnn_seeds_0560', self.sittingdown_err560 ) self.sittingdown_err1000_summary = tf.summary.scalar( 'euler_error_sittingdown/srnn_seeds_1000', self.sittingdown_err1000 ) with tf.name_scope( "euler_error_takingphoto" ): self.takingphoto_err80 = tf.placeholder( tf.float32, name="takingphoto_srnn_seeds_0080" ) self.takingphoto_err160 = tf.placeholder( tf.float32, name="takingphoto_srnn_seeds_0160" ) self.takingphoto_err320 = tf.placeholder( tf.float32, name="takingphoto_srnn_seeds_0320" ) self.takingphoto_err400 = tf.placeholder( tf.float32, name="takingphoto_srnn_seeds_0400" ) self.takingphoto_err560 = tf.placeholder( tf.float32, name="takingphoto_srnn_seeds_0560" ) self.takingphoto_err1000 = tf.placeholder( tf.float32, name="takingphoto_srnn_seeds_1000" ) self.takingphoto_err80_summary = tf.summary.scalar( 'euler_error_takingphoto/srnn_seeds_0080', self.takingphoto_err80 ) self.takingphoto_err160_summary = tf.summary.scalar( 'euler_error_takingphoto/srnn_seeds_0160', self.takingphoto_err160 ) self.takingphoto_err320_summary = tf.summary.scalar( 'euler_error_takingphoto/srnn_seeds_0320', self.takingphoto_err320 ) self.takingphoto_err400_summary = tf.summary.scalar( 'euler_error_takingphoto/srnn_seeds_0400', self.takingphoto_err400 ) self.takingphoto_err560_summary = tf.summary.scalar( 'euler_error_takingphoto/srnn_seeds_0560', self.takingphoto_err560 ) self.takingphoto_err1000_summary = tf.summary.scalar( 'euler_error_takingphoto/srnn_seeds_1000', self.takingphoto_err1000 ) with tf.name_scope( "euler_error_waiting" ): self.waiting_err80 = tf.placeholder( tf.float32, name="waiting_srnn_seeds_0080" ) self.waiting_err160 = tf.placeholder( tf.float32, name="waiting_srnn_seeds_0160" ) self.waiting_err320 = tf.placeholder( tf.float32, name="waiting_srnn_seeds_0320" ) self.waiting_err400 = tf.placeholder( tf.float32, name="waiting_srnn_seeds_0400" ) self.waiting_err560 = tf.placeholder( tf.float32, name="waiting_srnn_seeds_0560" ) self.waiting_err1000 = tf.placeholder( tf.float32, name="waiting_srnn_seeds_1000" ) self.waiting_err80_summary = tf.summary.scalar( 'euler_error_waiting/srnn_seeds_0080', self.waiting_err80 ) self.waiting_err160_summary = tf.summary.scalar( 'euler_error_waiting/srnn_seeds_0160', self.waiting_err160 ) self.waiting_err320_summary = tf.summary.scalar( 'euler_error_waiting/srnn_seeds_0320', self.waiting_err320 ) self.waiting_err400_summary = tf.summary.scalar( 'euler_error_waiting/srnn_seeds_0400', self.waiting_err400 ) self.waiting_err560_summary = tf.summary.scalar( 'euler_error_waiting/srnn_seeds_0560', self.waiting_err560 ) self.waiting_err1000_summary = tf.summary.scalar( 'euler_error_waiting/srnn_seeds_1000', self.waiting_err1000 ) with tf.name_scope( "euler_error_walkingdog" ): self.walkingdog_err80 = tf.placeholder( tf.float32, name="walkingdog_srnn_seeds_0080" ) self.walkingdog_err160 = tf.placeholder( tf.float32, name="walkingdog_srnn_seeds_0160" ) self.walkingdog_err320 = tf.placeholder( tf.float32, name="walkingdog_srnn_seeds_0320" ) self.walkingdog_err400 = tf.placeholder( tf.float32, name="walkingdog_srnn_seeds_0400" ) self.walkingdog_err560 = tf.placeholder( tf.float32, name="walkingdog_srnn_seeds_0560" ) self.walkingdog_err1000 = tf.placeholder( tf.float32, name="walkingdog_srnn_seeds_1000" ) self.walkingdog_err80_summary = tf.summary.scalar( 'euler_error_walkingdog/srnn_seeds_0080', self.walkingdog_err80 ) self.walkingdog_err160_summary = tf.summary.scalar( 'euler_error_walkingdog/srnn_seeds_0160', self.walkingdog_err160 ) self.walkingdog_err320_summary = tf.summary.scalar( 'euler_error_walkingdog/srnn_seeds_0320', self.walkingdog_err320 ) self.walkingdog_err400_summary = tf.summary.scalar( 'euler_error_walkingdog/srnn_seeds_0400', self.walkingdog_err400 ) self.walkingdog_err560_summary = tf.summary.scalar( 'euler_error_walkingdog/srnn_seeds_0560', self.walkingdog_err560 ) self.walkingdog_err1000_summary = tf.summary.scalar( 'euler_error_walkingdog/srnn_seeds_1000', self.walkingdog_err1000 ) with tf.name_scope( "euler_error_walkingtogether" ): self.walkingtogether_err80 = tf.placeholder( tf.float32, name="walkingtogether_srnn_seeds_0080" ) self.walkingtogether_err160 = tf.placeholder( tf.float32, name="walkingtogether_srnn_seeds_0160" ) self.walkingtogether_err320 = tf.placeholder( tf.float32, name="walkingtogether_srnn_seeds_0320" ) self.walkingtogether_err400 = tf.placeholder( tf.float32, name="walkingtogether_srnn_seeds_0400" ) self.walkingtogether_err560 = tf.placeholder( tf.float32, name="walkingtogether_srnn_seeds_0560" ) self.walkingtogether_err1000 = tf.placeholder( tf.float32, name="walkingtogether_srnn_seeds_1000" ) self.walkingtogether_err80_summary = tf.summary.scalar( 'euler_error_walkingtogether/srnn_seeds_0080', self.walkingtogether_err80 ) self.walkingtogether_err160_summary = tf.summary.scalar( 'euler_error_walkingtogether/srnn_seeds_0160', self.walkingtogether_err160 ) self.walkingtogether_err320_summary = tf.summary.scalar( 'euler_error_walkingtogether/srnn_seeds_0320', self.walkingtogether_err320 ) self.walkingtogether_err400_summary = tf.summary.scalar( 'euler_error_walkingtogether/srnn_seeds_0400', self.walkingtogether_err400 ) self.walkingtogether_err560_summary = tf.summary.scalar( 'euler_error_walkingtogether/srnn_seeds_0560', self.walkingtogether_err560 ) self.walkingtogether_err1000_summary = tf.summary.scalar( 'euler_error_walkingtogether/srnn_seeds_1000', self.walkingtogether_err1000 ) self.saver = tf.train.Saver( tf.global_variables(), max_to_keep=10 )
def __init__( self, architecture, max_seq_len, human_size, rnn_size, # hidden recurrent layer size num_layers, max_gradient_norm, stddev, batch_size, learning_rate, learning_rate_decay_factor, summaries_dir, loss_to_use, number_of_actions, one_hot=True, residual_velocities=False, dtype=tf.float32): """Create the model. Args: architecture: [basic, tied] whether to tie the decoder and decoder. source_seq_len: lenght of the input sequence. #target_seq_len: lenght of the target sequence. rnn_size: number of units in the rnn. num_layers: number of rnns to stack. max_gradient_norm: gradients will be clipped to maximally this norm. batch_size: the size of the batches used during training; the model construction is independent of batch_size, so it can be changed after initialization if this is convenient, e.g., for decoding. learning_rate: learning rate to start with. learning_rate_decay_factor: decay learning rate by this much when needed. summaries_dir: where to log progress for tensorboard. loss_to_use: [supervised, sampling_based]. Whether to use ground truth in each timestep to compute the loss after decoding, or to feed back the prediction from the previous time-step. number_of_actions: number of classes we have. one_hot: whether to use one_hot encoding during train/test (sup models). residual_velocities: whether to use a residual connection that models velocities. dtype: the data type to use to store internal variables. """ self.HUMAN_SIZE = human_size self.input_size = self.HUMAN_SIZE + number_of_actions if one_hot else self.HUMAN_SIZE print("One hot is ", one_hot) print("Input size is %d" % self.input_size) # Summary writers for train and test runs self.train_writer = tf.summary.FileWriter( os.path.normpath(os.path.join(summaries_dir, 'train'))) self.test_writer = tf.summary.FileWriter( os.path.normpath(os.path.join(summaries_dir, 'test'))) self.max_seq_len = max_seq_len self.rnn_size = rnn_size self.batch_size = batch_size self.learning_rate = tf.Variable(float(learning_rate), trainable=False, dtype=dtype) self.learning_rate_decay_op = self.learning_rate.assign( self.learning_rate * learning_rate_decay_factor) self.global_step = tf.Variable(0, trainable=False) # === Create the RNN that will keep the state === print('rnn_size = {0}'.format(rnn_size)) cell = tf.contrib.rnn.GRUCell(self.rnn_size) if num_layers > 1: cell = tf.contrib.rnn.MultiRNNCell([ tf.contrib.rnn.GRUCell(self.rnn_size) for _ in range(num_layers) ]) # === Transform the inputs === with tf.name_scope("inputs_gts"): inputs = tf.placeholder( dtype, shape=[None, self.max_seq_len + 1, self.input_size], name="inputs") gts = tf.placeholder( dtype, shape=[None, self.max_seq_len, self.input_size], name="gts") seq_len = tf.placeholder(tf.int32, shape=[None], name="seq_len") self.inputs = inputs self.gts = gts self.seq_len = seq_len ''' inputs = tf.transpose(inputs, [1, 0, 2]) gts = tf.transpose(gts, [1, 0, 2]) inputs = tf.reshape(inputs, [-1, self.input_size]) gts = tf.reshape(gts, [-1, self.input_size]) inputs = tf.split(inputs, self.max_seq_len, axis=0) gts = tf.split(gts, self.max_seq_len, axis=0) ''' inputs = _transpose_batch_time(inputs) gts = _transpose_batch_time(gts) # === Add space decoder === cell = rnn_cell_extensions.LinearSpaceDecoderWrapper( cell, self.input_size) # Finally, wrap everything in a residual layer if we want to model velocities if residual_velocities: cell = rnn_cell_extensions.ResidualWrapper(cell) # Store the outputs here outputs = [] self.stddev = stddev def addGN(inputs): noise = tf.random_normal(shape=tf.shape(inputs), mean=0.0, stddev=self.stddev, dtype=tf.float32) return inputs + noise self.is_training = tf.placeholder(dtype=tf.bool) # Build the RNN if architecture == "basic": cell_init_state = tf.Variable(np.zeros([1, cell.state_size]), trainable=True, dtype=tf.float32) init_input = tf.Variable(np.zeros([63]), trainable=True, dtype=tf.float32) output_ta = tf.TensorArray(size=self.max_seq_len, dtype=tf.float32) def loop_fn(time, cell_output, cell_state, loop_state): emit_output = cell_output if cell_output is None: #next_cell_state = cell.zero_state(self.batch_size, tf.float32) next_cell_state = tf.tile(cell_init_state, [tf.shape(inputs[0])[0], 1]) next_input = tf.cond( self.is_training, lambda: tf.concat([ tf.tile(tf.expand_dims(init_input, 0), [tf.shape(inputs[0])[0], 1]), addGN(inputs[time]) ], axis=1), lambda: tf.concat([ tf.tile(tf.expand_dims(init_input, 0), [tf.shape(inputs[0])[0], 1]), inputs[time] ], axis=1)) next_loop_state = output_ta else: next_cell_state = cell_state next_input = tf.cond( self.is_training, lambda: tf.concat( [cell_output, addGN(inputs[time])], axis=1), lambda: tf.concat([cell_output, inputs[time]], axis=1)) next_loop_state = loop_state.write(time - 1, cell_output) finished = (time > self.max_seq_len - 1) #finished = False return (finished, next_input, next_cell_state, emit_output, next_loop_state) # Basic RNN does not have a loop function in its API, so copying here. with vs.variable_scope("raw_rnn"): _, _, loop_state_ta = tf.nn.raw_rnn(cell, loop_fn) #outputs = _transpose_batch_time(loop_state_ta.stack()) outputs = loop_state_ta.stack() self.outputs = outputs mask1 = tf.tile( tf.expand_dims( tf.transpose( tf.sequence_mask(self.seq_len, dtype=tf.float32, maxlen=self.max_seq_len)), -1), [1, 1, self.input_size]) mask2 = tf.tile( tf.expand_dims( tf.transpose( tf.sequence_mask(self.seq_len - 1, dtype=tf.float32, maxlen=self.max_seq_len - 1)), -1), [1, 1, self.input_size]) with tf.name_scope("loss_pos"): loss_pos = tf.reduce_mean( tf.square( tf.subtract(tf.multiply(outputs, mask1), tf.multiply(gts, mask1)))) with tf.name_scope("loss_smooth"): loss_smooth = tf.reduce_mean( tf.square( tf.multiply(tf.subtract(outputs[1:], outputs[:-1]), mask2))) #self.loss = tf.add(loss_pos, loss_smooth*1000) self.loss = loss_pos self.loss_summary = tf.summary.scalar('loss/loss', self.loss) self.loss_each_data = tf.reduce_mean(tf.square(tf.subtract(tf.multiply(gts,mask1), tf.multiply(outputs,mask1))), axis=[0,2]) \ + tf.reduce_mean(tf.square(tf.multiply(tf.subtract( outputs[1:], outputs[:-1]),mask2)),axis=[0,2]) # Gradients and SGD update operation for training the model. params = tf.trainable_variables() opt = tf.train.AdamOptimizer(learning_rate=learning_rate) # Update all the trainable parameters gradients = tf.gradients(self.loss, params) clipped_gradients, norm = tf.clip_by_global_norm( gradients, max_gradient_norm) self.gradient_norms = norm self.updates = opt.apply_gradients(zip(clipped_gradients, params), global_step=self.global_step) self.learning_rate_summary = tf.summary.scalar( 'learning_rate/learning_rate', self.learning_rate) self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=10)