def sample(nodes, nodesPresent, grid, args, net, true_nodes, true_nodesPresent, true_grid, saved_args, dimensions): ''' The sample function params: nodes: Input positions nodesPresent: Peds present in each frame args: arguments net: The model true_nodes: True positions true_nodesPresent: The true peds present in each frame true_grid: The true grid masks saved_args: Training arguments dimensions: The dimensions of the dataset ''' # Number of peds in the sequence numNodes = nodes.size()[1] # Construct variables for hidden and cell states hidden_states = Variable(torch.zeros(numNodes, net.args.rnn_size), volatile=True).cuda() cell_states = Variable(torch.zeros(numNodes, net.args.rnn_size), volatile=True).cuda() # For the observed part of the trajectory for tstep in range(args.obs_length-1): # Do a forward prop out_obs, hidden_states, cell_states = net(nodes[tstep].view(1, numNodes, 2), [grid[tstep]], [nodesPresent[tstep]], hidden_states, cell_states) # loss_obs = Gaussian2DLikelihood(out_obs, nodes[tstep+1].view(1, numNodes, 2), [nodesPresent[tstep+1]]) # Initialize the return data structure ret_nodes = Variable(torch.zeros(args.obs_length+args.pred_length, numNodes, 2), volatile=True).cuda() ret_nodes[:args.obs_length, :, :] = nodes.clone() # Last seen grid prev_grid = grid[-1].clone() # For the predicted part of the trajectory for tstep in range(args.obs_length-1, args.pred_length + args.obs_length - 1): # Do a forward prop outputs, hidden_states, cell_states = net(ret_nodes[tstep].view(1, numNodes, 2), [prev_grid], [nodesPresent[args.obs_length-1]], hidden_states,cell_states) # loss_pred = Gaussian2DLikelihoodInference(outputs, true_nodes[tstep+1].view(1, numNodes, 2), nodesPresent[args.obs_length-1], [true_nodesPresent[tstep+1]]) # Extract the mean, std and corr of the bivariate Gaussian mux, muy, sx, sy, corr = getCoef(outputs) # Sample from the bivariate Gaussian next_x, next_y = sample_gaussian_2d(mux.data, muy.data, sx.data, sy.data, corr.data, nodesPresent[args.obs_length-1]) # Store the predicted position ret_nodes[tstep + 1, :, 0] = next_x ret_nodes[tstep + 1, :, 1] = next_y # List of nodes at the last time-step (assuming they exist until the end) list_of_nodes = Variable(torch.LongTensor(nodesPresent[args.obs_length-1]), volatile=True).cuda() # Get their predicted positions current_nodes = torch.index_select(ret_nodes[tstep+1], 0, list_of_nodes) # Compute the new grid masks with the predicted positions prev_grid = getGridMaskInference(current_nodes.data.cpu().numpy(), dimensions, saved_args.neighborhood_size, saved_args.grid_size) prev_grid = Variable(torch.from_numpy(prev_grid).float(), volatile=True).cuda() return ret_nodes
def Gaussian2DLikelihood(outputs, targets, nodesPresent, pred_length): ''' Computes the likelihood of predicted locations under a bivariate Gaussian distribution params: outputs: Torch variable containing tensor of shape seq_length x numNodes x output_size (e.g. 20*10*5) targets: Torch variable containing tensor of shape seq_length x numNodes x input_size nodesPresent : A list of lists, of size seq_length. Each list contains the nodeIDs that are present in the frame ''' # Get the sequence length seq_length = outputs.size()[0] # Get the observed length obs_length = seq_length - pred_length # Extract mean, std devs and correlation mux, muy, sx, sy, corr = getCoef(outputs) # print('outputs={},size={}'.format(outputs,outputs.size)) # print('input={},size={}'.format(targets, outputs.size)) #print('mux, muy, sx, sy, corr:{}{}{}{}'.format(mux, muy, sx, sy, corr)) # Compute factors normx = targets[:, :, 0] - mux normy = targets[:, :, 1] - muy sxsy = sx * sy z = torch.pow((normx / sx), 2) + torch.pow( (normy / sy), 2) - 2 * ((corr * normx * normy) / sxsy) negRho = 1 - torch.pow(corr, 2) # Numerator result = torch.exp(-z / (2 * negRho)) # Normalization factor denom = 2 * np.pi * (sxsy * torch.sqrt(negRho)) # Final PDF calculation result = result / denom # Numerical stability epsilon = 1e-20 result = -torch.log(torch.clamp(result, min=epsilon)) #print('result={}'.format(result)) # Compute the loss across all frames and all nodes loss = 0 counter = 0 for framenum in range(obs_length, seq_length): nodeIDs = nodesPresent[framenum] for nodeID in nodeIDs: loss = loss + result[framenum, nodeID] counter = counter + 1 if counter != 0: return loss / counter else: return loss
def Gaussian2DLikelihoodInference(outputs, targets, assumedNodesPresent, nodesPresent, use_cuda): ''' Computes the likelihood of predicted locations under a bivariate Gaussian distribution at test time params: outputs : predicted locations targets : true locations assumedNodesPresent : Nodes assumed to be present in each frame in the sequence nodesPresent : True nodes present in each frame in the sequence ''' # Extract mean, std devs and correlation mux, muy, sx, sy, corr = getCoef(outputs) # Compute factors normx = targets[:, :, 0] - mux normy = targets[:, :, 1] - muy sxsy = sx * sy z = (normx / sx)**2 + (normy / sy)**2 - 2 * ((corr * normx * normy) / sxsy) negRho = 1 - corr**2 # Numerator result = torch.exp(-z / (2 * negRho)) # Normalization factor denom = 2 * np.pi * (sxsy * torch.sqrt(negRho)) # Final PDF calculation result = result / denom # Numerical stability epsilon = 1e-20 result = -torch.log(torch.clamp(result, min=epsilon)) # Compute the loss loss = Variable(torch.zeros(1)) if use_cuda: loss = loss.cuda() counter = 0 for framenum in range(outputs.size()[0]): nodeIDs = nodesPresent[framenum] for nodeID in nodeIDs: if nodeID not in assumedNodesPresent: # If the node wasn't assumed to be present, don't compute loss for it continue loss = loss + result[framenum, nodeID] counter = counter + 1 if counter != 0: return loss / counter else: return loss
def Gaussian2DLikelihood(outputs, targets, nodesPresent, pred_length): ''' Parameters: outputs: Torch variable containing tensor of shape seq_length x numNodes x 1 x output_size targets: Torch variable containing tensor of shape seq_length x numNodes x 1 x input_size nodesPresent : A list of lists, of size seq_length. Each list contains the nodeIDs that are present in the frame ''' seq_length = outputs.size()[0] obs_length = seq_length - pred_length # Extract mean, std devs and correlation mux, muy, sx, sy, corr = getCoef(outputs) # Compute factors normx = targets[:, :, 0] - mux normy = targets[:, :, 1] - muy sxsy = sx * sy z = (normx / sx)**2 + (normy / sy)**2 - 2 * ((corr * normx * normy) / sxsy) negRho = 1 - corr**2 # Numerator result = torch.exp(-z / (2 * negRho)) # Normalization factor denom = 2 * np.pi * (sxsy * torch.sqrt(negRho)) # Final PDF calculation result = result / denom # Numerical stability epsilon = 1e-20 result = -torch.log(torch.clamp(result, min=epsilon)) loss = 0 counter = 0 for framenum in range(obs_length, seq_length): nodeIDs = nodesPresent[framenum] for nodeID in nodeIDs: loss = loss + result[framenum, nodeID] counter = counter + 1 if counter != 0: return loss / counter else: return loss
def sample( nodes, edges, nodesPresent, edgesPresent, args, net, true_nodes, true_edges, true_nodesPresent, ): """ Sample function Parameters ========== nodes : A tensor of shape obs_length x numNodes x 2 Each row contains (x, y) edges : A tensor of shape obs_length x numNodes x numNodes x 2 Each row contains the vector representing the edge If edge doesn't exist, then the row contains zeros nodesPresent : A list of lists, of size obs_length Each list contains the nodeIDs that are present in the frame edgesPresent : A list of lists, of size obs_length Each list contains tuples of nodeIDs that have edges in the frame args : Sampling Arguments net : The network Returns ======= ret_nodes : A tensor of shape (obs_length + pred_length) x numNodes x 2 Contains the true and predicted positions of all the nodes """ # Number of nodes numNodes = nodes.size()[1] # Initialize hidden states for the nodes h_nodes = Variable(torch.zeros(numNodes, net.args.node_rnn_size), volatile=True) h_edges = Variable(torch.zeros(numNodes * numNodes, net.args.edge_rnn_size), volatile=True) c_nodes = Variable(torch.zeros(numNodes, net.args.node_rnn_size), volatile=True) c_edges = Variable(torch.zeros(numNodes * numNodes, net.args.edge_rnn_size), volatile=True) h_super_node = Variable(torch.zeros(3, net.args.node_rnn_size), volatile=True) c_super_node = Variable(torch.zeros(3, net.args.node_rnn_size), volatile=True) h_super_edges = Variable(torch.zeros(3, net.args.edge_rnn_size), volatile=True) c_super_edges = Variable(torch.zeros(3, net.args.edge_rnn_size), volatile=True) if args.use_cuda: h_nodes = h_nodes.cuda() h_edges = h_edges.cuda() c_nodes = c_nodes.cuda() c_edges = c_edges.cuda() h_super_node = h_super_node.cuda() c_super_node = c_super_node.cuda() h_super_edges = h_super_edges.cuda() c_super_edges = c_super_edges.cuda() # Propagate the observed length of the trajectory for tstep in range(args.obs_length - 1): # Forward prop out_obs, h_nodes, h_edges, c_nodes, c_edges, h_super_node, h_super_edges, c_super_node, c_super_edges, _ = net( nodes[tstep].view(1, numNodes, 2), edges[tstep].view(1, numNodes * numNodes, 2), [nodesPresent[tstep]], [edgesPresent[tstep]], h_nodes, h_edges, c_nodes, c_edges, h_super_node, h_super_edges, c_super_node, c_super_edges, ) # loss_obs = Gaussian2DLikelihood(out_obs, nodes[tstep+1].view(1, numNodes, 2), [nodesPresent[tstep+1]]) # Initialize the return data structures ret_nodes = Variable(torch.zeros(args.obs_length + args.pred_length, numNodes, 2), volatile=True) if args.use_cuda: ret_nodes = ret_nodes.cuda() ret_nodes[:args.obs_length, :, :] = nodes.clone() ret_edges = Variable( torch.zeros((args.obs_length + args.pred_length), numNodes * numNodes, 2), volatile=True, ) if args.use_cuda: ret_edges = ret_edges.cuda() ret_edges[:args.obs_length, :, :] = edges.clone() ret_attn = [] # Propagate the predicted length of trajectory (sampling from previous prediction) for tstep in range(args.obs_length - 1, args.pred_length + args.obs_length - 1): # TODO Not keeping track of nodes leaving the frame (or new nodes entering the frame, which I don't think we can do anyway) # Forward prop outputs, h_nodes, h_edges, c_nodes, c_edges, h_super_node, h_super_edges, c_super_node, c_super_edges, attn_w = net( ret_nodes[tstep].view(1, numNodes, 2), ret_edges[tstep].view(1, numNodes * numNodes, 2), [nodesPresent[args.obs_length - 1]], [edgesPresent[args.obs_length - 1]], h_nodes, h_edges, c_nodes, c_edges, h_super_node, h_super_edges, c_super_node, c_super_edges, ) mux, muy, sx, sy, corr = getCoef(outputs) next_x, next_y = sample_gaussian_2d( mux.data, muy.data, sx.data, sy.data, corr.data, nodesPresent[args.obs_length - 1], ) ret_nodes[tstep + 1, :, 0] = next_x ret_nodes[tstep + 1, :, 1] = next_y # Compute edges # TODO Currently, assuming edges from the last observed time-step will stay for the entire prediction length ret_edges[tstep + 1, :, :] = compute_edges( ret_nodes.data, tstep + 1, edgesPresent[args.obs_length - 1], args.use_cuda) # Store computed attention weights ret_attn.append(attn_w[0]) return ret_nodes, ret_attn
def sample(x_seq, Pedlist, args, net, true_x_seq, true_Pedlist, saved_args, dimensions, dataloader, look_up, num_pedlist, is_gru, grid=None): ''' The sample function params: x_seq: Input positions Pedlist: Peds present in each frame args: arguments net: The model true_x_seq: True positions true_Pedlist: The true peds present in each frame saved_args: Training arguments dimensions: The dimensions of the dataset target_id: ped_id number that try to predict in this sequence ''' # Number of peds in the sequence numx_seq = len(look_up) with torch.no_grad(): # Construct variables for hidden and cell states hidden_states = Variable(torch.zeros(numx_seq, net.args.rnn_size)) if args.use_cuda: hidden_states = hidden_states.cuda() if not is_gru: cell_states = Variable(torch.zeros(numx_seq, net.args.rnn_size)) if args.use_cuda: cell_states = cell_states.cuda() else: cell_states = None ret_x_seq = Variable( torch.zeros(args.obs_length + args.pred_length, numx_seq, 2)) # Initialize the return data structure if args.use_cuda: ret_x_seq = ret_x_seq.cuda() # For the observed part of the trajectory for tstep in range(args.obs_length - 1): if grid is None: #vanilla lstm # Do a forward prop out_obs, hidden_states, cell_states = net( x_seq[tstep].view(1, numx_seq, 2), hidden_states, cell_states, [Pedlist[tstep]], [num_pedlist[tstep]], dataloader, look_up) else: # Do a forward prop out_obs, hidden_states, cell_states = net( x_seq[tstep].view(1, numx_seq, 2), [grid[tstep]], hidden_states, cell_states, [Pedlist[tstep]], [num_pedlist[tstep]], dataloader, look_up) # loss_obs = Gaussian2DLikelihood(out_obs, x_seq[tstep+1].view(1, numx_seq, 2), [Pedlist[tstep+1]]) # Extract the mean, std and corr of the bivariate Gaussian mux, muy, sx, sy, corr = getCoef(out_obs) # Sample from the bivariate Gaussian next_x, next_y = sample_gaussian_2d(mux.data, muy.data, sx.data, sy.data, corr.data, true_Pedlist[tstep], look_up) ret_x_seq[tstep + 1, :, 0] = next_x ret_x_seq[tstep + 1, :, 1] = next_y ret_x_seq[:args.obs_length, :, :] = x_seq.clone() # Last seen grid if grid is not None: #no vanilla lstm prev_grid = grid[-1].clone() #assign last position of observed data to temp #temp_last_observed = ret_x_seq[args.obs_length-1].clone() #ret_x_seq[args.obs_length-1] = x_seq[args.obs_length-1] # For the predicted part of the trajectory for tstep in range(args.obs_length - 1, args.pred_length + args.obs_length - 1): # Do a forward prop if grid is None: #vanilla lstm outputs, hidden_states, cell_states = net( ret_x_seq[tstep].view(1, numx_seq, 2), hidden_states, cell_states, [true_Pedlist[tstep]], [num_pedlist[tstep]], dataloader, look_up) else: outputs, hidden_states, cell_states = net( ret_x_seq[tstep].view(1, numx_seq, 2), [prev_grid], hidden_states, cell_states, [true_Pedlist[tstep]], [num_pedlist[tstep]], dataloader, look_up) # Extract the mean, std and corr of the bivariate Gaussian mux, muy, sx, sy, corr = getCoef(outputs) # Sample from the bivariate Gaussian next_x, next_y = sample_gaussian_2d(mux.data, muy.data, sx.data, sy.data, corr.data, true_Pedlist[tstep], look_up) # Store the predicted position ret_x_seq[tstep + 1, :, 0] = next_x ret_x_seq[tstep + 1, :, 1] = next_y # List of x_seq at the last time-step (assuming they exist until the end) true_Pedlist[tstep + 1] = [ int(_x_seq) for _x_seq in true_Pedlist[tstep + 1] ] next_ped_list = copy.deepcopy(true_Pedlist[tstep + 1]) converted_pedlist = [look_up[_x_seq] for _x_seq in next_ped_list] list_of_x_seq = Variable(torch.LongTensor(converted_pedlist)) if args.use_cuda: list_of_x_seq = list_of_x_seq.cuda() #Get their predicted positions current_x_seq = torch.index_select(ret_x_seq[tstep + 1], 0, list_of_x_seq) if grid is not None: #no vanilla lstm # Compute the new grid masks with the predicted positions if args.method == 2: #obstacle lstm prev_grid = getGridMask(current_x_seq.data.cpu(), dimensions, len(true_Pedlist[tstep + 1]), saved_args.neighborhood_size, saved_args.grid_size, True) elif args.method == 1: #social lstm prev_grid = getGridMask(current_x_seq.data.cpu(), dimensions, len(true_Pedlist[tstep + 1]), saved_args.neighborhood_size, saved_args.grid_size) prev_grid = Variable(torch.from_numpy(prev_grid).float()) if args.use_cuda: prev_grid = prev_grid.cuda() #ret_x_seq[args.obs_length-1] = temp_last_observed return ret_x_seq
def sample( nodes, edges, nodesPresent, edgesPresent, args, net, true_nodes, true_edges, true_nodesPresent, ): """ Sample function Parameters ========== nodes : A tensor of shape obs_length x numNodes x 2 Each row contains (x, y) edges : A tensor of shape obs_length x numNodes x numNodes x 2 Each row contains the vector representing the edge If edge doesn't exist, then the row contains zeros nodesPresent : A list of lists, of size obs_length Each list contains the nodeIDs that are present in the frame edgesPresent : A list of lists, of size obs_length Each list contains tuples of nodeIDs that have edges in the frame args : Sampling Arguments net : The network Returns ======= ret_nodes : A tensor of shape (obs_length + pred_length) x numNodes x 2 Contains the true and predicted positions of all the nodes """ # Number of nodes numNodes = nodes.size()[1] # Initialize hidden states for the nodes h_nodes = Variable(torch.zeros(numNodes, net.args.node_rnn_size), volatile=True) h_edges = Variable(torch.zeros(numNodes * numNodes, net.args.edge_rnn_size), volatile=True) c_nodes = Variable(torch.zeros(numNodes, net.args.node_rnn_size), volatile=True) c_edges = Variable(torch.zeros(numNodes * numNodes, net.args.edge_rnn_size), volatile=True) h_super_node = Variable(torch.zeros(3, net.args.node_rnn_size), volatile=True) c_super_node = Variable(torch.zeros(3, net.args.node_rnn_size), volatile=True) h_super_edges = Variable(torch.zeros(3, net.args.edge_rnn_size), volatile=True) c_super_edges = Variable(torch.zeros(3, net.args.edge_rnn_size), volatile=True) if args.use_cuda: h_nodes = h_nodes.cuda() h_edges = h_edges.cuda() c_nodes = c_nodes.cuda() c_edges = c_edges.cuda() h_super_node = h_super_node.cuda() c_super_node = c_super_node.cuda() h_super_edges = h_super_edges.cuda() c_super_edges = c_super_edges.cuda() # NOTE: Propagate the observed length of the trajectory// I think this step is for obtaining '''{hidden, cell}''' states for tstep in range(args.obs_length - 1): #till observed length # Forward prop out_obs, h_nodes, h_edges, c_nodes, c_edges, h_super_node, h_super_edges, c_super_node, c_super_edges, _ = net( nodes[tstep].view(1, numNodes, 2), edges[tstep].view(1, numNodes * numNodes, 2), [nodesPresent[tstep]], [edgesPresent[tstep]], h_nodes, h_edges, c_nodes, c_edges, h_super_node, h_super_edges, c_super_node, c_super_edges, ) # loss_obs = Gaussian2DLikelihood(out_obs, nodes[tstep+1].view(1, numNodes, 2), [nodesPresent[tstep+1]]) # Initialize the return data structures ret_nodes = Variable(torch.zeros(args.obs_length + args.pred_length, numNodes, 2), volatile=True) if args.use_cuda: ret_nodes = ret_nodes.cuda() ret_nodes[:args.obs_length, :, :] = nodes.clone() #### Ground Truth ret_edges = Variable( torch.zeros((args.obs_length + args.pred_length), numNodes * numNodes, 2), volatile=True, ) if args.use_cuda: ret_edges = ret_edges.cuda() ret_edges[:args.obs_length, :, :] = edges.clone() #### Ground Truth ret_attn = [] # Propagate the predicted length of trajectory (sampling from previous prediction) # WHICH IS PREDICTION STEP! for tstep in range(args.obs_length - 1, args.pred_length + args.obs_length - 1): # TODO Not keeping track of nodes leaving the frame (or new nodes entering the frame, which I don't think we can do anyway) # Forward prop// Recursive// # 이 알고리즘은 recursive하게 현 state{t}에서 다음 state를 단순히 잇는 LSTM cell을 학습시키는 것 이기에 sequence 데이터가 들어오게 되면 # cell을 이용하여 다음 output_{t+1}을 예측하고, 이 output_{t+1}이 다시 cell로 들어가서 output_{t+2}를 내뱉는 구조. outputs, h_nodes, h_edges, c_nodes, c_edges, h_super_node, h_super_edges, c_super_node, c_super_edges, attn_w = net( ret_nodes[tstep].view(1, numNodes, 2), ret_edges[tstep].view(1, numNodes * numNodes, 2), [nodesPresent[args.obs_length - 1]], [edgesPresent[args.obs_length - 1]], h_nodes, h_edges, c_nodes, c_edges, h_super_node, h_super_edges, c_super_node, c_super_edges, ) ####이때 output은 현재 frame에 존재하는 모든 node들의 (mux,muy,std,corr) mux, muy, sx, sy, corr = getCoef(outputs) #TODO next_x, next_y = sample_gaussian_2d( #gaussian 2d를 구축한 후, 여기서 sampling된 한 점을 x,y로 하는데 이게 맞나...? mux.data, muy.data, sx.data, sy.data, corr.data, nodesPresent[args.obs_length - 1], #오직 현scene에 존재하는 object들의 trajectory를 예측하는 것. ) ret_nodes[tstep + 1, :, 0] = next_x ret_nodes[tstep + 1, :, 1] = next_y # Compute edges # TODO Currently, assuming edges from the last observed time-step will stay for the entire prediction length ret_edges[tstep + 1, :, :] = compute_edges( ret_nodes.data, tstep + 1, edgesPresent[args.obs_length - 1], args.use_cuda) # Store computed attention weights ret_attn.append(attn_w[0]) return ret_nodes, ret_attn