def sample(self, sess, traj, grid, dimensions, true_traj, num=10):
# traj is a sequence of frames (of length obs_length)
# so traj shape is (obs_length x maxNumPeds x 3)
# grid is a tensor of shape obs_length x maxNumPeds x maxNumPeds x (gs**2)
states = sess.run(self.LSTM_states)
# print "Fitting"
# For each frame in the sequence
for index, frame in enumerate(traj[:-1]):
data = np.reshape(frame, (1, self.maxNumPeds, 3))
target_data = np.reshape(traj[index + 1], (1, self.maxNumPeds, 3))
grid_data = np.reshape(grid[index, :],
(1, self.maxNumPeds, self.maxNumPeds, self.grid_size * self.grid_size))
feed = {self.input_data: data, self.LSTM_states: states, self.grid_data: grid_data,
self.target_data: target_data}
[states, cost] = sess.run([self.final_states, self.cost], feed)
# print cost
ret = traj
last_frame = traj[-1]
prev_data = np.reshape(last_frame, (1, self.maxNumPeds, 3))
prev_grid_data = np.reshape(grid[-1], (1, self.maxNumPeds, self.maxNumPeds, self.grid_size * self.grid_size))
prev_target_data = np.reshape(true_traj[traj.shape[0]], (1, self.maxNumPeds, 3))
# Prediction
for t in range(num):
# print "**** NEW PREDICTION TIME STEP", t, "****"
feed = {self.input_data: prev_data, self.LSTM_states: states, self.grid_data: prev_grid_data,
self.target_data: prev_target_data}
[output, states, cost] = sess.run([self.final_output, self.final_states, self.cost], feed)
# print "Cost", cost
# Output is a list of lists where the inner lists contain matrices of shape 1x5. The outer list contains only one element (since seq_length=1) and the inner list contains maxNumPeds elements
# output = output[0]
newpos = np.zeros((1, self.maxNumPeds, 3))
for pedindex, pedoutput in enumerate(output):
[o_mux, o_muy, o_sx, o_sy, o_corr] = np.split(pedoutput[0], 5, 0)
mux, muy, sx, sy, corr = o_mux[0], o_muy[0], np.exp(o_sx[0]), np.exp(o_sy[0]), np.tanh(o_corr[0])
next_x, next_y = self.sample_gaussian_2d(mux, muy, sx, sy, corr)
# if prev_data[0, pedindex, 0] != 0:
# print "Pedestrian ID", prev_data[0, pedindex, 0]
# print "Predicted parameters", mux, muy, sx, sy, corr
# print "New Position", next_x, next_y
# print "Target Position", prev_target_data[0, pedindex, 1], prev_target_data[0, pedindex, 2]
# print
newpos[0, pedindex, :] = [prev_data[0, pedindex, 0], next_x, next_y]
ret = np.vstack((ret, newpos))
prev_data = newpos
prev_grid_data = getSequenceGridMask(prev_data, dimensions, self.args.neighborhood_size, self.grid_size)
if t != num - 1:
prev_target_data = np.reshape(true_traj[traj.shape[0] + t + 1], (1, self.maxNumPeds, 3))
# The returned ret is of shape (obs_length+pred_length) x maxNumPeds x 3
return ret
def sample_one_step(self, sess, prev_data, prev_grid_data, dimensions):
'''
FUNC:
Arguments:
sess: tf session to be run on
prev_data: data of previous step, shape = (1,MNP,3)
prev_grid_data: grid data of previous step, shape = (1,MNP,MNP,grid_size**2)
dimensions: a list of [width,height], specifying shape of background
Returns:
next_data: data of predicted (next) step, shape = (1,MNP,3)
next_grid_data: grid data of predicted (next) step, shape = (1,MNP,MNP,grid_size**2)
'''
# predict one step according to last frame in traj
feed = {
self.input_data: prev_data,
self.LSTM_states: self.stored_states,
self.grid_data: prev_grid_data,
self.target_data: prev_data # does not matter
}
fetch = [self.final_output, self.final_states]
[output, states] = sess.run(fetch, feed)
# store states --> used for following-on prediction
self.stored_states = states
# convert raw ouput to positions
next_data = np.zeros((1, self.maxNumPeds, 3))
for pedindex, pedoutput in enumerate(output):
# convert 5 distribution parameters
[o_mux, o_muy, o_sx, o_sy, o_corr] = np.split(pedoutput[0], 5, 0)
mux, muy, sx, sy, corr = o_mux[0], o_muy[0], np.exp(
o_sx[0]), np.exp(o_sy[0]), np.tanh(o_corr[0])
# sample from Gaussian parameterized by the 5 parameters
next_x, next_y = self.sample_gaussian_2d(mux, muy, sx, sy, corr)
#DEBUG
'''
if prev_data[0, pedindex, 0] != 0:
print "Pedestrian ID", prev_data[0, pedindex, 0]
print "Predicted parameters", mux, muy, sx, sy, corr
print "New Position", next_x, next_y
print "Target Position", prev_target_data[0, pedindex, 1], prev_target_data[0, pedindex, 2]
print
'''
# store to matrix
next_data[0, pedindex, :] = [
prev_data[0, pedindex, 0], next_x, next_y
]
# obtain grid data of predicted (next) step
next_grid_data = getSequenceGridMask(next_data, dimensions,
self.args.neighborhood_size,
self.grid_size)
return next_data, next_grid_data
def sample(self, sess, traj, grid, dimensions, true_traj, num=10):
states = sess.run(self.LSTM_states)
for index, frame in enumerate(traj[:-1]):
#data ora ha dimensione 1 X maxNumPeds X 5 invece che X 3 cosi' da poter contenere anche il goal
# lo stesso vale per target_data
data = np.reshape(frame, (1, self.maxNumPeds, 5))
target_data = np.reshape(traj[index+1], (1, self.maxNumPeds, 5))
grid_data = np.reshape(grid[index, :], (1, self.maxNumPeds, self.maxNumPeds, self.grid_size*self.grid_size))
feed = {self.input_data: data, self.LSTM_states: states, self.grid_data: grid_data, self.target_data: target_data}
[states, cost] = sess.run([self.final_states, self.cost], feed)
ret = traj
last_frame = traj[-1]
#come prima sono stati ampliati anche prev_data e prev_grid_data da 3 a 5 per potre contenere anche il goal
prev_data = np.reshape(last_frame, (1, self.maxNumPeds, 5))
prev_grid_data = np.reshape(grid[-1], (1, self.maxNumPeds, self.maxNumPeds, self.grid_size*self.grid_size))
prev_target_data = np.reshape(true_traj[traj.shape[0]], (1, self.maxNumPeds, 5))
for t in range(num):
feed = {self.input_data: prev_data, self.LSTM_states: states, self.grid_data: prev_grid_data, self.target_data: prev_target_data}
[output, states, cost] = sess.run([self.final_output, self.final_states, self.cost], feed)
#le nuove posizioni hanno dimensione 1 X 5 invece che 1 X 3 per fare spazio al goal
newpos = np.zeros((1, self.maxNumPeds, 5))
for pedindex, pedoutput in enumerate(output):
[o_mux, o_muy, o_sx, o_sy, o_corr] = np.split(pedoutput[0], 5, 0)
mux, muy, sx, sy, corr = o_mux[0], o_muy[0], np.exp(o_sx[0]), np.exp(o_sy[0]), np.tanh(o_corr[0])
next_x, next_y = self.sample_gaussian_2d(mux, muy, sx, sy, corr)
newpos[0, pedindex, :] = [prev_data[0, pedindex, 0], next_x, next_y, 0, 0]
ret = np.vstack((ret, newpos))
prev_data = newpos
prev_grid_data = getSequenceGridMask(prev_data, dimensions, self.args.neighborhood_size, self.grid_size)
if t != num - 1:
prev_target_data = np.reshape(true_traj[traj.shape[0] + t + 1], (1, self.maxNumPeds, 5))
return ret
def train(args):
datasets = list(range(5))
#从数据集中删除leaveDataset
datasets.remove(args.leaveDataset)
# Create the SocialDataLoader object
data_loader = SocialDataLoader(args.batch_size,
args.seq_length,
args.maxNumPeds,
datasets,
forcePreProcess=True,
infer=False)
# 日志目录
log_directory = 'log/'
log_directory += str(args.leaveDataset) + '/'
# Logging files
log_file_curve = open(os.path.join(log_directory, 'log_curve.txt'), 'w')
log_file = open(os.path.join(log_directory, 'val.txt'), 'w')
#存储目录
save_directory = 'save/'
save_directory += str(args.leaveDataset) + '/'
with open(os.path.join(save_directory, 'social_config.pkl'), 'wb') as f:
pickle.dump(args, f)
# Create a SocialModel object with the arguments
model = SocialModel(args)
with tf.Session() as sess:
# Initialize all variables in the graph
sess.run(tf.global_variables_initializer())
# Initialize a saver that saves all the variables in the graph
saver = tf.train.Saver(tf.global_variables(), max_to_keep=None)
# summary_writer = tf.train.SummaryWriter('/tmp/lstm/logs', graph_def=sess.graph_def)
print('Training begin')
best_val_loss = 100
best_epoch = 0
# For each epoch
for e in range(args.num_epochs):
# Assign the learning rate value for this epoch
sess.run(
tf.assign(model.lr, args.learning_rate * (args.decay_rate**e)))
# Reset the data pointers in the data_loader
data_loader.reset_batch_pointer(valid=False)
loss_epoch = 0
# For each batch
for b in range(data_loader.num_batches):
start = time.time()
# Get the source, target and dataset data for the next batch
# 获取下一批的源，目标和数据集数据
# x, y are input and target data which are lists containing numpy arrays of size seq_length x maxNumPeds x 3
# x，y是输入和目标数据，它们是包含大小为seq_length x maxNumPeds x 3的numpy数组的列表
# d is the list of dataset indices from which each batch is generated (used to differentiate between datasets)
# d是生成每个批次的数据集索引列表（用于区分数据集）
x, y, d = data_loader.next_batch()
# variable to store the loss for this batch
# 变量来存储此批次的损失
loss_batch = 0
# For each sequence in the batch
for batch in range(data_loader.batch_size):
# x_batch, y_batch and d_batch contains the source, target and dataset index data for
# x_batch，y_batch和d_batch包含源，目标和数据集索引数据
# seq_length long consecutive frames in the dataset
# seq_length数据集中的长连续帧
# x_batch, y_batch would be numpy arrays of size seq_length x maxNumPeds x 3
# d_batch would be a scalar identifying the dataset from which this sequence is extracted
# d_batch将是一个标量，用于标识从中提取此序列的数据集
x_batch, y_batch, d_batch = x[batch], y[batch], d[batch]
if d_batch == 0 and datasets[0] == 0:
dataset_data = [640, 480]
else:
dataset_data = [720, 576]
grid_batch = getSequenceGridMask(x_batch, dataset_data,
args.neighborhood_size,
args.grid_size)
# Feed the source, target data
# 提供源，目标数据
feed = {
model.input_data: x_batch,
model.target_data: y_batch,
model.grid_data: grid_batch
}
# 运行模型，跑出来一个结果
train_loss, _ = sess.run([model.cost, model.train_op],
feed)
loss_batch += train_loss
end = time.time()
loss_batch = loss_batch / data_loader.batch_size
loss_epoch += loss_batch
print(
"{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}"
.format(e * data_loader.num_batches + b,
args.num_epochs * data_loader.num_batches, e,
loss_batch, end - start))
loss_epoch /= data_loader.num_batches
log_file_curve.write(str(e) + ',' + str(loss_epoch) + ',')
print('*****************')
# 验证模型
data_loader.reset_batch_pointer(valid=True)
loss_epoch = 0
for b in range(data_loader.num_batches):
# Get the source, target and dataset data for the next batch
# x, y are input and target data which are lists containing numpy arrays of size seq_length x maxNumPeds x 3
# d is the list of dataset indices from which each batch is generated (used to differentiate between datasets)
x, y, d = data_loader.next_valid_batch()
# variable to store the loss for this batch
loss_batch = 0
# For each sequence in the batch
for batch in range(data_loader.batch_size):
# x_batch, y_batch and d_batch contains the source, target and dataset index data for
# seq_length long consecutive frames in the dataset
# x_batch, y_batch would be numpy arrays of size seq_length x maxNumPeds x 3
# d_batch would be a scalar identifying the dataset from which this sequence is extracted
x_batch, y_batch, d_batch = x[batch], y[batch], d[batch]
if d_batch == 0 and datasets[0] == 0:
dataset_data = [640, 480]
else:
dataset_data = [720, 576]
grid_batch = getSequenceGridMask(x_batch, dataset_data,
args.neighborhood_size,
args.grid_size)
# Feed the source, target data
feed = {
model.input_data: x_batch,
model.target_data: y_batch,
model.grid_data: grid_batch
}
train_loss = sess.run(model.cost, feed)
loss_batch += train_loss
loss_batch = loss_batch / data_loader.batch_size
loss_epoch += loss_batch
loss_epoch /= data_loader.valid_num_batches
# Update best validation loss until now
if loss_epoch < best_val_loss:
best_val_loss = loss_epoch
best_epoch = e
print('(epoch {}), valid_loss = {:.3f}'.format(e, loss_epoch))
print('Best epoch', best_epoch, 'Best validation loss',
best_val_loss)
log_file_curve.write(str(loss_epoch) + '\n')
print('*****************')
# Save the model after each epoch
print('Saving model')
checkpoint_path = os.path.join(save_directory, 'social_model.ckpt')
saver.save(sess, checkpoint_path, global_step=e)
print("model saved to {}".format(checkpoint_path))
print('Best epoch', best_epoch, 'Best validation loss', best_val_loss)
log_file.write(str(best_epoch) + ',' + str(best_val_loss))
# CLose logging files
log_file.close()
log_file_curve.close()
def sample(self, sess, traj, grid, dimensions, true_traj, num=10):
states = sess.run(self.LSTM_states)
for index, frame in enumerate(traj[:-1]):
data = np.reshape(frame, (1, self.maxNumPeds, 3))
target_data = np.reshape(traj[index + 1], (1, self.maxNumPeds, 3))
#grid_data ha dimensioni 1 X NMP X grid_size*2 a differenza di prima che aveva dimensioni 1 X MNP X MNP X grid_size*grid_size
grid_data = np.reshape(grid[index, :],
(1, self.maxNumPeds, self.grid_size * 2))
feed = {
self.input_data: data,
self.LSTM_states: states,
self.grid_data: grid_data,
self.target_data: target_data
}
[states, cost] = sess.run([self.final_states, self.cost], feed)
# print cost
ret = traj
last_frame = traj[-1]
prev_data = np.reshape(last_frame, (1, self.maxNumPeds, 3))
#come grid_data e' stata modificata la dimensione anche di prev_grid_data
prev_grid_data = np.reshape(grid[-1],
(1, self.maxNumPeds, self.grid_size * 2))
prev_target_data = np.reshape(true_traj[traj.shape[0]],
(1, self.maxNumPeds, 3))
for t in range(num):
feed = {
self.input_data: prev_data,
self.LSTM_states: states,
self.grid_data: prev_grid_data,
self.target_data: prev_target_data
}
[output, states, cost
] = sess.run([self.final_output, self.final_states, self.cost],
feed)
newpos = np.zeros((1, self.maxNumPeds, 3))
for pedindex, pedoutput in enumerate(output):
[o_mux, o_muy, o_sx, o_sy,
o_corr] = np.split(pedoutput[0], 5, 0)
mux, muy, sx, sy, corr = o_mux[0], o_muy[0], np.exp(
o_sx[0]), np.exp(o_sy[0]), np.tanh(o_corr[0])
next_x, next_y = self.sample_gaussian_2d(
mux, muy, sx, sy, corr)
newpos[0, pedindex, :] = [
prev_data[0, pedindex, 0], next_x, next_y
]
ret = np.vstack((ret, newpos))
prev_data = newpos
prev_grid_data = getSequenceGridMask(prev_data, dimensions,
self.args.neighborhood_size,
self.grid_size)
if t != num - 1:
prev_target_data = np.reshape(true_traj[traj.shape[0] + t + 1],
(1, self.maxNumPeds, 3))
return ret
def train(args):
datasets = range(5)
# Remove the leaveDataset from datasets
datasets.remove(args.leaveDataset)
# Create the SocialDataLoader object
data_loader = SocialDataLoader(args.batch_size,
args.seq_length,
args.maxNumPeds,
datasets,
forcePreProcess=True,
infer=False)
# Log directory
log_directory = 'log/'
log_directory += str(args.leaveDataset) + '/'
# Logging files
log_file_curve = open(os.path.join(log_directory, 'log_curve.txt'), 'w')
log_file = open(os.path.join(log_directory, 'val.txt'), 'w')
# Save directory
save_directory = 'save/'
save_directory += str(args.leaveDataset) + '/'
with open(os.path.join(save_directory, 'social_config.pkl'), 'wb') as f:
pickle.dump(args, f)
# Create a SocialModel object with the arguments
model = SocialModel(args)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config = tf.ConfigProto(
log_device_placement=True
) # Showing which device is allocated (in case of multiple GPUs)
config.gpu_options.per_process_gpu_memory_fraction = 0.8 # Allocating 20% of memory in each GPU with 0.5
# Initialize a TensorFlow session
with tf.Session() as sess:
# Initialize all variables in the graph
sess.run(tf.global_variables_initializer())
# Initialize a saver that saves all the variables in the graph
saver = tf.train.Saver(tf.global_variables(), max_to_keep=None)
# summary_writer = tf.train.SummaryWriter('/tmp/lstm/logs', graph_def=sess.graph_def)
print('Training begin')
best_val_loss = 100
best_epoch = 0
# For each epoch
for e in range(args.num_epochs):
# Assign the learning rate value for this epoch
sess.run(
tf.assign(model.lr, args.learning_rate * (args.decay_rate**e)))
# Reset the data pointers in the data_loader
data_loader.reset_batch_pointer(valid=False)
loss_epoch = 0
# For each batch
for b in range(data_loader.num_batches):
# Tic
start = time.time()
# Get the source, target and dataset data for the next batch
# x, y are input and target data which are lists containing numpy arrays of size seq_length x maxNumPeds x 3
# d is the list of dataset indices from which each batch is generated (used to differentiate between datasets)
x, y, d = data_loader.next_batch()
# variable to store the loss for this batch
loss_batch = 0
# For each sequence in the batch
for batch in range(data_loader.batch_size):
# x_batch, y_batch and d_batch contains the source, target and dataset index data for
# seq_length long consecutive frames in the dataset
# x_batch, y_batch would be numpy arrays of size seq_length x maxNumPeds x 3
# d_batch would be a scalar identifying the dataset from which this sequence is extracted
x_batch, y_batch, d_batch = x[batch], y[batch], d[batch]
if d_batch == 0 and datasets[0] == 0:
dataset_data = [640, 480]
else:
dataset_data = [720, 576]
grid_batch = getSequenceGridMask(x_batch, dataset_data,
args.neighborhood_size,
args.grid_size)
# Feed the source, target data
feed = {
model.input_data: x_batch,
model.target_data: y_batch,
model.grid_data: grid_batch
}
train_loss, _ = sess.run([model.cost, model.train_op],
feed)
loss_batch += train_loss
end = time.time()
loss_batch = loss_batch / data_loader.batch_size
loss_epoch += loss_batch
print(
"{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}"
.format(e * data_loader.num_batches + b,
args.num_epochs * data_loader.num_batches, e,
loss_batch, end - start))
# Save the model if the current epoch and batch number match the frequency
'''
if (e * data_loader.num_batches + b) % args.save_every == 0 and ((e * data_loader.num_batches + b) > 0):
checkpoint_path = os.path.join('save', 'social_model.ckpt')
saver.save(sess, checkpoint_path, global_step=e * data_loader.num_batches + b)
print("model saved to {}".format(checkpoint_path))
'''
loss_epoch /= data_loader.num_batches
log_file_curve.write(str(e) + ',' + str(loss_epoch) + ',')
print('*****************')
# Validation
data_loader.reset_batch_pointer(valid=True)
loss_epoch = 0
for b in range(data_loader.num_batches):
# Get the source, target and dataset data for the next batch
# x, y are input and target data which are lists containing numpy arrays of size seq_length x maxNumPeds x 3
# d is the list of dataset indices from which each batch is generated (used to differentiate between datasets)
x, y, d = data_loader.next_valid_batch()
# variable to store the loss for this batch
loss_batch = 0
# For each sequence in the batch
for batch in range(data_loader.batch_size):
# x_batch, y_batch and d_batch contains the source, target and dataset index data for
# seq_length long consecutive frames in the dataset
# x_batch, y_batch would be numpy arrays of size seq_length x maxNumPeds x 3
# d_batch would be a scalar identifying the dataset from which this sequence is extracted
x_batch, y_batch, d_batch = x[batch], y[batch], d[batch]
if d_batch == 0 and datasets[0] == 0:
dataset_data = [640, 480]
else:
dataset_data = [720, 576]
grid_batch = getSequenceGridMask(x_batch, dataset_data,
args.neighborhood_size,
args.grid_size)
# Feed the source, target data
feed = {
model.input_data: x_batch,
model.target_data: y_batch,
model.grid_data: grid_batch
}
train_loss = sess.run(model.cost, feed)
loss_batch += train_loss
loss_batch = loss_batch / data_loader.batch_size
loss_epoch += loss_batch
loss_epoch /= data_loader.valid_num_batches
# Update best validation loss until now
if loss_epoch < best_val_loss:
best_val_loss = loss_epoch
best_epoch = e
print('(epoch {}), valid_loss = {:.3f}'.format(e, loss_epoch))
print('Best epoch', best_epoch, 'Best validation loss',
best_val_loss)
log_file_curve.write(str(loss_epoch) + '\n')
print('*****************')
# Save the model after each epoch
print('Saving model')
checkpoint_path = os.path.join(save_directory, 'social_model.ckpt')
saver.save(sess, checkpoint_path, global_step=e)
print("model saved to {}".format(checkpoint_path))
print('Best epoch', best_epoch, 'Best validation loss', best_val_loss)
log_file.write(str(best_epoch) + ',' + str(best_val_loss))
# CLose logging files
log_file.close()
log_file_curve.close()
def train(args):
datasets = [i for i in range(5)]
# Remove the leave out dataset from the datasets
datasets.remove(args.leaveDataset)
# Construct the DataLoader object
dataloader = DataLoader(args.batch_size,
args.seq_length + 1,
datasets,
forcePreProcess=True)
# Construct the ST-graph object
stgraph = ST_GRAPH(args.batch_size, args.seq_length + 1)
# Log directory
log_directory = 'log/'
log_directory += str(args.leaveDataset) + '/'
# Logging files
log_file_curve = open(os.path.join(log_directory, 'log_curve.txt'), 'w')
log_file = open(os.path.join(log_directory, 'val.txt'), 'w')
# Save directory
save_directory = 'save/'
save_directory += str(args.leaveDataset) + '/'
# Dump the arguments into the configuration file
with open(os.path.join(save_directory, 'config.pkl'), 'wb') as f:
pickle.dump(args, f)
# Path to store the checkpoint file
def checkpoint_path(x):
return os.path.join(save_directory,
'social_lstm_model_' + str(x) + '.tar')
# Initialize net
net = SocialLSTM(args)
net.cuda()
optimizer = torch.optim.RMSprop(net.parameters(), lr=args.learning_rate)
learning_rate = args.learning_rate
print('Training begin')
best_val_loss = 100
best_epoch = 0
# Training
for epoch in range(args.num_epochs):
dataloader.reset_batch_pointer(valid=False)
loss_epoch = 0
# For each batch
for batch in range(dataloader.num_batches):
start = time.time()
# Get batch data
x, _, d = dataloader.next_batch()
# Construct the stgraph
stgraph.readGraph(x)
loss_batch = 0
# For each sequence
for sequence in range(dataloader.batch_size):
# Get the data corresponding to the current sequence
x_seq, d_seq = x[sequence], d[sequence]
# Dataset dimensions
if d_seq == 0 and datasets[0] == 0:
dataset_data = [640, 480]
else:
dataset_data = [720, 576]
# Compute grid masks
grid_seq = getSequenceGridMask(x_seq, dataset_data,
args.neighborhood_size,
args.grid_size)
obst_seq = get_seq_mask(x_seq, d_seq, dataset_data,
args.neighborhood_size, args.grid_size)
# Get the node features and nodes present from stgraph
nodes, _, nodesPresent, _ = stgraph.getSequence(sequence)
# Construct variables
nodes = Variable(torch.from_numpy(nodes).float()).cuda()
# nodes = Variable(torch.from_numpy(nodes).float())
numNodes = nodes.size()[1]
hidden_states = Variable(torch.zeros(numNodes,
args.rnn_size)).cuda()
cell_states = Variable(torch.zeros(numNodes,
args.rnn_size)).cuda()
# hidden_states = Variable(torch.zeros(numNodes, args.rnn_size))
# cell_states = Variable(torch.zeros(numNodes, args.rnn_size))
# Zero out gradients
net.zero_grad()
optimizer.zero_grad()
# Forward prop
outputs, _, _ = net(nodes[:-1], grid_seq[:-1], obst_seq[:-1],
nodesPresent[:-1], hidden_states,
cell_states)
# Compute loss
loss = Gaussian2DLikelihood(outputs, nodes[1:],
nodesPresent[1:], args.pred_length)
loss_batch += loss.data[0]
# Compute gradients
loss.backward()
# Clip gradients
torch.nn.utils.clip_grad_norm(net.parameters(), args.grad_clip)
# Update parameters
optimizer.step()
# Reset stgraph
stgraph.reset()
end = time.time()
loss_batch = loss_batch / dataloader.batch_size
loss_epoch += loss_batch
print('{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}'.
format(epoch * dataloader.num_batches + batch,
args.num_epochs * dataloader.num_batches, epoch,
loss_batch, end - start))
loss_epoch /= dataloader.num_batches
# Log loss values
log_file_curve.write(str(epoch) + ',' + str(loss_epoch) + ',')
# Validation
dataloader.reset_batch_pointer(valid=True)
loss_epoch = 0
# For each batch
for batch in range(dataloader.valid_num_batches):
# Get batch data
x, _, d = dataloader.next_valid_batch(randomUpdate=False)
# Read the st graph from data
stgraph.readGraph(x)
# Loss for this batch
loss_batch = 0
# For each sequence
for sequence in range(dataloader.batch_size):
# Get data corresponding to the current sequence
x_seq, d_seq = x[sequence], d[sequence]
# Dataset dimensions
if d_seq == 0 and datasets[0] == 0:
dataset_data = [640, 480]
else:
dataset_data = [720, 576]
# Compute grid masks
grid_seq = getSequenceGridMask(x_seq, dataset_data,
args.neighborhood_size,
args.grid_size)
obst_seq = get_seq_mask(x_seq, d_seq, dataset_data,
args.neighborhood_size, args.grid_size)
# Get node features and nodes present from stgraph
nodes, _, nodesPresent, _ = stgraph.getSequence(sequence)
# Construct variables
nodes = Variable(torch.from_numpy(nodes).float()).cuda()
# nodes = Variable(torch.from_numpy(nodes).float())
numNodes = nodes.size()[1]
# hidden_states = Variable(torch.zeros(numNodes, args.rnn_size))
# cell_states = Variable(torch.zeros(numNodes, args.rnn_size))
hidden_states = Variable(torch.zeros(numNodes,
args.rnn_size)).cuda()
cell_states = Variable(torch.zeros(numNodes,
args.rnn_size)).cuda()
# Forward prop
outputs, _, _ = net(nodes[:-1], grid_seq[:-1], obst_seq[:-1],
nodesPresent[:-1], hidden_states,
cell_states)
# Compute loss
loss = Gaussian2DLikelihood(outputs, nodes[1:],
nodesPresent[1:], args.pred_length)
loss_batch += loss.data[0]
# Reset the stgraph
stgraph.reset()
loss_batch = loss_batch / dataloader.batch_size
loss_epoch += loss_batch
loss_epoch = loss_epoch / dataloader.valid_num_batches
# Update best validation loss until now
if loss_epoch < best_val_loss:
best_val_loss = loss_epoch
best_epoch = epoch
print('(epoch {}), valid_loss = {:.3f}'.format(epoch, loss_epoch))
print('Best epoch', best_epoch, 'Best validation loss', best_val_loss)
log_file_curve.write(str(loss_epoch) + '\n')
# Save the model after each epoch
print('Saving model')
torch.save(
{
'epoch': epoch,
'state_dict': net.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, checkpoint_path(epoch))
print('Best epoch', best_epoch, 'Best validation Loss', best_val_loss)
# Log the best epoch and best validation loss
log_file.write(str(best_epoch) + ',' + str(best_val_loss))
# Close logging files
log_file.close()
log_file_curve.close()
def predict(self, tracked_persons):
tracks = tracked_persons.tracks
track_dict = {}
for track in tracks:
#print track
#print track.pose.pose.position.x
track_dict[track.track_id] = [
track.pose.pose.position.x, track.pose.pose.position.y
]
del self.prev_frames[0]
self.prev_frames.append(track_dict)
if len(tracks) == 0:
return
input_data = self.__generate_input(tracks)
#print input_data.shape
#print input_data
grid_batch = getSequenceGridMask(input_data, self.dimensions,
self.saved_args.neighborhood_size,
self.saved_args.grid_size)
obs_traj = input_data[:self.obs_length]
obs_grid = grid_batch[:self.obs_length]
print "********************** PREDICT NEW TRAJECTORY ******************************"
complete_traj = self.social_lstm_model.sample(self.sess, obs_traj,
obs_grid,
self.dimensions,
input_data,
self.pred_length)
#print complete_traj
# Initialize the markers array
prediction_markers = MarkerArray()
# Publish them
people_predictions = PeoplePrediction()
for frame_index in range(self.pred_length):
people = People()
people.header.stamp = tracked_persons.header.stamp + rospy.Duration(
frame_index * self.time_resolution)
people.header.frame_id = tracked_persons.header.frame_id
predicted_frame_index = frame_index + self.obs_length
for person_index in range(self.max_pedestrians):
track_id = complete_traj[predicted_frame_index, person_index,
0]
x_coord = complete_traj[predicted_frame_index, person_index, 1]
y_coord = complete_traj[predicted_frame_index, person_index, 2]
if track_id == 0:
break
person = Person()
person.name = str(track_id)
point = Point()
point.x = x_coord
point.y = y_coord
person.position = point
people.people.append(person)
self.prediction_marker.header.frame_id = tracked_persons.header.frame_id
self.prediction_marker.header.stamp = tracked_persons.header.stamp
self.prediction_marker.id = int(track_id)
self.prediction_marker.pose.position.x = person.position.x
self.prediction_marker.pose.position.y = person.position.y
#self.prediction_marker.color.a = 1 - (frame_index * 1.0 / (self.pred_length * 1.0))
self.prediction_marker.color.a = 1.0
prediction_markers.markers.append(self.prediction_marker)
people_predictions.predicted_people.append(people)
#print people_predictions
self.pedestrian_prediction_pub.publish(people_predictions)
self.prediction_marker_pub.publish(prediction_markers)
def main():
parser = argparse.ArgumentParser()
# Observed length of the trajectory parameter
parser.add_argument('--obs_length',
type=int,
default=8,
help='Observed length of the trajectory')
# Predicted length of the trajectory parameter
parser.add_argument('--pred_length',
type=int,
default=12,
help='Predicted length of the trajectory')
# Model to be loaded
parser.add_argument('--epoch',
type=int,
default=14,
help='Epoch of model to be loaded')
# cuda support
parser.add_argument('--use_cuda',
action="store_true",
default=False,
help='Use GPU or not')
# drive support
parser.add_argument('--drive',
action="store_true",
default=False,
help='Use Google drive or not')
# number of iteration -> we are trying many times to get lowest test error derived from observed part and prediction of observed
# part.Currently it is useless because we are using direct copy of observed part and no use of prediction.Test error will be 0.
parser.add_argument(
'--iteration',
type=int,
default=1,
help=
'Number of iteration to create test file (smallest test errror will be selected)'
)
# gru model
parser.add_argument('--gru',
action="store_true",
default=False,
help='True : GRU cell, False: LSTM cell')
# method selection
parser.add_argument(
'--method',
type=int,
default=1,
help=
'Method of lstm will be used (1 = social lstm, 2 = obstacle lstm, 3 = vanilla lstm)'
)
# Parse the parameters
sample_args = parser.parse_args()
#for drive run
prefix = ''
f_prefix = '.'
if sample_args.drive is True:
prefix = 'drive/semester_project/social_lstm_final/'
f_prefix = 'drive/semester_project/social_lstm_final'
#run sh file for folder creation
if not os.path.isdir("log/"):
print("Directory creation script is running...")
subprocess.call([f_prefix + '/make_directories.sh'])
method_name = get_method_name(sample_args.method)
model_name = "LSTM"
save_tar_name = method_name + "_lstm_model_"
if sample_args.gru:
model_name = "GRU"
save_tar_name = method_name + "_gru_model_"
print("Selected method name: ", method_name, " model name: ", model_name)
# Save directory
save_directory = os.path.join(f_prefix, 'model/', method_name, model_name)
#plot directory for plotting in the future
plot_directory = os.path.join(f_prefix, 'plot/', method_name, model_name)
result_directory = os.path.join(f_prefix, 'result/', method_name)
plot_test_file_directory = 'test'
# Define the path for the config file for saved args
with open(os.path.join(save_directory, 'config.pkl'), 'rb') as f:
saved_args = pickle.load(f)
seq_lenght = sample_args.pred_length + sample_args.obs_length
# Create the DataLoader object
dataloader = DataLoader(f_prefix,
1,
seq_lenght,
forcePreProcess=True,
infer=True)
create_directories(os.path.join(result_directory, model_name),
dataloader.get_all_directory_namelist())
create_directories(plot_directory, [plot_test_file_directory])
dataloader.reset_batch_pointer()
dataset_pointer_ins = dataloader.dataset_pointer
smallest_err = 100000
smallest_err_iter_num = -1
origin = (0, 0)
reference_point = (0, 1)
submission_store = [] # store submission data points (txt)
result_store = [] # store points for plotting
for iteration in range(sample_args.iteration):
# Initialize net
net = get_model(sample_args.method, saved_args, True)
if sample_args.use_cuda:
net = net.cuda()
# Get the checkpoint path
checkpoint_path = os.path.join(
save_directory, save_tar_name + str(sample_args.epoch) + '.tar')
if os.path.isfile(checkpoint_path):
print('Loading checkpoint')
checkpoint = torch.load(checkpoint_path)
model_epoch = checkpoint['epoch']
net.load_state_dict(checkpoint['state_dict'])
print('Loaded checkpoint at epoch', model_epoch)
# For each batch
iteration_submission = []
iteration_result = []
results = []
submission = []
# Variable to maintain total error
total_error = 0
final_error = 0
for batch in range(dataloader.num_batches):
start = time.time()
# Get data
x, y, d, numPedsList, PedsList, target_ids = dataloader.next_batch(
)
# Get the sequence
x_seq, d_seq, numPedsList_seq, PedsList_seq, target_id = x[0], d[
0], numPedsList[0], PedsList[0], target_ids[0]
dataloader.clean_test_data(x_seq, target_id,
sample_args.obs_length,
sample_args.pred_length)
dataloader.clean_ped_list(x_seq, PedsList_seq, target_id,
sample_args.obs_length,
sample_args.pred_length)
#get processing file name and then get dimensions of file
folder_name = dataloader.get_directory_name_with_pointer(d_seq)
dataset_data = dataloader.get_dataset_dimension(folder_name)
#dense vector creation
x_seq, lookup_seq = dataloader.convert_proper_array(
x_seq, numPedsList_seq, PedsList_seq)
#will be used for error calculation
orig_x_seq = x_seq.clone()
# target_id_values = orig_x_seq[0][lookup_seq[target_id], 0:2]
#grid mask calculation
if sample_args.method == 2: #obstacle lstm
grid_seq = getSequenceGridMask(x_seq, dataset_data,
PedsList_seq,
saved_args.neighborhood_size,
saved_args.grid_size,
saved_args.use_cuda, True)
elif sample_args.method == 1: #social lstm
grid_seq = getSequenceGridMask(x_seq, dataset_data,
PedsList_seq,
saved_args.neighborhood_size,
saved_args.grid_size,
saved_args.use_cuda)
#vectorize datapoints
x_seq, first_values_dict = vectorize_seq(x_seq, PedsList_seq,
lookup_seq)
# <------------- Experimental block ---------------->
# x_seq = translate(x_seq, PedsList_seq, lookup_seq ,target_id_values)
# angle = angle_between(reference_point, (x_seq[1][lookup_seq[target_id], 0].data.numpy(), x_seq[1][lookup_seq[target_id], 1].data.numpy()))
# x_seq = rotate_traj_with_target_ped(x_seq, angle, PedsList_seq, lookup_seq)
# grid_seq = getSequenceGridMask(x_seq[:sample_args.obs_length], dataset_data, PedsList_seq, saved_args.neighborhood_size, saved_args.grid_size, sample_args.use_cuda)
# x_seq, first_values_dict = vectorize_seq(x_seq, PedsList_seq, lookup_seq)
if sample_args.use_cuda:
x_seq = x_seq.cuda()
# The sample function
if sample_args.method == 3: #vanilla lstm
# Extract the observed part of the trajectories
obs_traj, obs_PedsList_seq = x_seq[:sample_args.
obs_length], PedsList_seq[:
sample_args
.
obs_length]
ret_x_seq = sample(obs_traj, obs_PedsList_seq, sample_args,
net, x_seq, PedsList_seq, saved_args,
dataset_data, dataloader, lookup_seq,
numPedsList_seq, sample_args.gru)
else:
# Extract the observed part of the trajectories
obs_traj, obs_PedsList_seq, obs_grid = x_seq[:sample_args.
obs_length], PedsList_seq[:
sample_args
.
obs_length], grid_seq[:
sample_args
.
obs_length]
ret_x_seq = sample(obs_traj, obs_PedsList_seq, sample_args,
net, x_seq, PedsList_seq, saved_args,
dataset_data, dataloader, lookup_seq,
numPedsList_seq, sample_args.gru, obs_grid)
#revert the points back to original space
ret_x_seq = revert_seq(ret_x_seq, PedsList_seq, lookup_seq,
first_values_dict)
# <--------------------- Experimental inverse block ---------------------->
# ret_x_seq = revert_seq(ret_x_seq, PedsList_seq, lookup_seq, target_id_values, first_values_dict)
# ret_x_seq = rotate_traj_with_target_ped(ret_x_seq, -angle, PedsList_seq, lookup_seq)
# ret_x_seq = translate(ret_x_seq, PedsList_seq, lookup_seq ,-target_id_values)
# Record the mean and final displacement error
print(PedsList_seq[-sample_args.pred_length:])
total_error += get_mean_error(
ret_x_seq[-sample_args.pred_length:].data,
orig_x_seq[-sample_args.pred_length:].data,
PedsList_seq[-sample_args.pred_length:],
PedsList_seq[-sample_args.pred_length:], sample_args.use_cuda,
lookup_seq)
final_error += get_final_error(
ret_x_seq[-sample_args.pred_length:].data,
orig_x_seq[-sample_args.pred_length:].data,
PedsList_seq[-sample_args.pred_length:],
PedsList_seq[-sample_args.pred_length:], lookup_seq)
end = time.time()
print('Current file : ', dataloader.get_file_name(0),
' Processed trajectory number : ', batch + 1, 'out of',
dataloader.num_batches, 'trajectories in time', end - start)
# if dataset_pointer_ins is not dataloader.dataset_pointer:
# if dataloader.dataset_pointer is not 0:
# iteration_submission.append(submission)
# iteration_result.append(results)
# dataset_pointer_ins = dataloader.dataset_pointer
# submission = []
# results = []
# submission.append(submission_preprocess(dataloader, ret_x_seq.data[sample_args.obs_length:, lookup_seq[target_id], :].numpy(), sample_args.pred_length, sample_args.obs_length, target_id))
# results.append((x_seq.data.cpu().numpy(), ret_x_seq.data.cpu().numpy(), PedsList_seq, lookup_seq , dataloader.get_frame_sequence(seq_lenght), target_id, sample_args.obs_length))
# iteration_submission.append(submission)
# iteration_result.append(results)
# submission_store.append(iteration_submission)
# result_store.append(iteration_result)
if total_error < smallest_err:
print("**********************************************************")
print('Best iteration has been changed. Previous best iteration: ',
smallest_err_iter_num + 1, 'Error: ',
smallest_err / dataloader.num_batches)
print('New best iteration : ', iteration + 1, 'Error: ',
total_error / dataloader.num_batches)
smallest_err_iter_num = iteration
smallest_err = total_error
print('Iteration:', iteration + 1,
' Total training (observed part) mean error of the model is ',
total_error / dataloader.num_batches)
print('Iteration:', iteration + 1,
'Total training (observed part) final error of the model is ',
final_error / dataloader.num_batches)
#print(submission)
print('Smallest error iteration:', smallest_err_iter_num + 1)
def _step(self):
### a certain chance of a new pedstrain poping up
# only possible if current number of pedestrians doesn't exceed the maximum number
if self._cur_num_peds < self._max_num_peds:
new_ped_pops_up = self._pops_up_fn(self._cur_num_peds,
self._max_num_peds)
if new_ped_pops_up:
# create initial data for the new pedestrain
new_data = self._init_ped_data()
# add data of the new pedestrian self._data and self._grid_data
for i in range(self._max_num_peds):
if self._data[0, i, 0] == 0: # an unoccupied element
newly_exist_pedID = i + 1
self._init_data[:, i, 0] = newly_exist_pedID
self._init_data[:, i, 1:3] = new_data
break
assert (newly_exist_pedID == i + 1) #DEBUG
self._init_grid_data = getSequenceGridMask(
self._init_data, self._bg_shape,
self._social_conf.neighborhood_size,
self._social_conf.grid_size)
# reinitialize LSTM model
self._model.sample_init(self._sess, self._init_data,
self._init_grid_data)
self._data[0, i, :] = self._init_data[
-1,
i, :] #np.reshape(self._init_data[-1,:,:], self._data.shape)
self._grid_data = np.reshape(self._init_grid_data[-1, :, :, :],
self._grid_data.shape)
# update current number of pedestrians and pedestrian existence list
self._cur_num_peds += 1
self._peds_exist[i] = True
if self.verbose:
print('A new pedestrian with ID {} pops up at ({},{})'\
.format(newly_exist_pedID,
int(new_data[0,0]*self._bg_shape[0]),
int(new_data[0,1]*self._bg_shape[1])))
### predict next step of all existing pedestrians
self._data, self._grid_data = self._model.sample_one_step(
self._sess, self._data, self._grid_data, self._bg_shape)
### remove pedestrians out-of-bound (not in the background area)
for i in range(self._max_num_peds):
pedID = self._data[0, i, 0]
# if pedID==0 --> nonexisting pedestrian
if pedID != 0:
x = self._data[0, i, 1]
y = self._data[0, i, 2]
if (x < -0.1) or (x > 1.1) or (y < -0.1) or (y > 1.1):
# remove data of current pedstrian from self._data and self._grid_data
self._init_data[:, i, :] = 0
self._init_grid_data = getSequenceGridMask(
self._init_data, self._bg_shape,
self._social_conf.neighborhood_size,
self._social_conf.grid_size)
# reinitialize social LSTM model
self._model.sample_init(self._sess, self._init_data,
self._init_grid_data)
self._data[0, i, :] = self._init_data[
-1,
i, :] #np.reshape(self._init_data[-1,:,:], self._data.shape)
self._grid_data = np.reshape(
self._init_grid_data[-1, :, :, :],
self._grid_data.shape)
# update current number of pedestrian and pedestrian existence list
self._cur_num_peds -= 1
self._peds_exist[i] = False
if self.verbose:
print('A pedestrian with ID {} is out-of-bound at ({},{}) and is removed.'\
.format(int(pedID),
int(x*self._bg_shape[0]),
int(y*self._bg_shape[1])))
def main():
# Set random seed
np.random.seed(1)
parser = argparse.ArgumentParser()
# Observed length of the trajectory parameter
parser.add_argument('--obs_length', type=int, default=8,
help='Observed length of the trajectory')
# Predicted length of the trajectory parameter
parser.add_argument('--pred_length', type=int, default=12,
help='Predicted length of the trajectory')
# Test dataset
parser.add_argument('--test_dataset', type=str,
help='Dataset to be tested on')
parser.add_argument('--visible',type=str,
required=False, default=None, help='GPU to run on')
parser.add_argument('--model_path', type=str)
# Parse the parameters
sample_args = parser.parse_args()
if sample_args.visible:
os.environ["CUDA_VISIBLE_DEVICES"] = sample_args.visible
save_path = sample_args.model_path
# Define the path for the config file for saved args
with open(os.path.join(save_path, 'social_config.pkl'), 'rb') as f:
saved_args = pickle.load(f)
# Create a SocialModel object with the saved_args and infer set to true
model = SocialModel(saved_args, True)
# Initialize a TensorFlow session
sess = tf.InteractiveSession()
# Initialize a saver
saver = tf.train.Saver()
# Get the checkpoint state for the model
ckpt = tf.train.get_checkpoint_state(save_path)
print ('loading model: ', ckpt.model_checkpoint_path)
# Restore the model at the checkpoint
saver.restore(sess, ckpt.model_checkpoint_path)
# Create a SocialDataLoader object with batch_size 1 and seq_length equal to observed_length + pred_length
data_loader = SocialDataLoader(1, sample_args.pred_length +
sample_args.obs_length, saved_args.maxNumPeds, sample_args.test_dataset, True)
# Reset all pointers of the data_loader
data_loader.reset_batch_pointer()
results = []
# Variable to maintain total error
total_error = 0
# For each batch
for b in range(data_loader.num_batches):
# Get the source, target and dataset data for the next batch
x, y, d = data_loader.next_batch(randomUpdate=False)
# Batch size is 1
x_batch, y_batch, d_batch = x[0], y[0], d[0]
'''
if d_batch == 0 and dataset[0] == 0:
dimensions = [640, 480]
else:
dimensions = [720, 576]
'''
grid_batch = getSequenceGridMask(x_batch, [0,0], saved_args.neighborhood_size, saved_args.grid_size)
obs_traj = x_batch[:sample_args.obs_length]
obs_grid = grid_batch[:sample_args.obs_length]
# obs_traj is an array of shape obs_length x maxNumPeds x 3
print "********************** SAMPLING A NEW TRAJECTORY", b, "******************************"
complete_traj = model.sample(sess, obs_traj, obs_grid, [0,0], x_batch, sample_args.pred_length)
# ipdb.set_trace()
# complete_traj is an array of shape (obs_length+pred_length) x maxNumPeds x 3
total_error += get_mean_error(complete_traj, x[0], sample_args.obs_length, saved_args.maxNumPeds)
print "Processed trajectory number : ", b, "out of ", data_loader.num_batches, " trajectories"
# plot_trajectories(x[0], complete_traj, sample_args.obs_length)
# return
results.append((x[0], complete_traj, sample_args.obs_length))
# Print the mean error across all the batches
print "Total mean error of the model is ", total_error/data_loader.num_batches
print "Saving results"
with open(os.path.join(save_path, 'social_results.pkl'), 'wb') as f:
pickle.dump(results, f)
def main():
parser = argparse.ArgumentParser()
# Observed length of the trajectory parameter
parser.add_argument('--obs_length',
type=int,
default=8,
help='Observed length of the trajectory')
# Predicted length of the trajectory parameter
parser.add_argument('--pred_length',
type=int,
default=12,
help='Predicted length of the trajectory')
# Model to be loaded
parser.add_argument('--epoch',
type=int,
default=14,
help='Epoch of model to be loaded')
# cuda support
parser.add_argument('--use_cuda',
action="store_true",
default=False,
help='Use GPU or not')
# drive support
parser.add_argument('--drive',
action="store_true",
default=False,
help='Use Google drive or not')
# number of iteration -> we are trying many times to get lowest test error derived from observed part and prediction of observed
# part.Currently it is useless because we are using direct copy of observed part and no use of prediction.Test error will be 0.
parser.add_argument(
'--iteration',
type=int,
default=1,
help=
'Number of iteration to create test file (smallest test errror will be selected)'
)
# gru model
parser.add_argument('--gru',
action="store_true",
default=False,
help='True : GRU cell, False: LSTM cell')
# method selection
parser.add_argument(
'--method',
type=int,
default=3,
help=
'Method of lstm will be used (1 = social lstm, 2 = obstacle lstm, 3 = vanilla lstm)'
)
# Parse the parameters
sample_args = parser.parse_args()
# for drive run
prefix = ''
f_prefix = '.'
if sample_args.drive is True:
prefix = 'drive/semester_project/social_lstm_final/'
f_prefix = 'drive/semester_project/social_lstm_final'
# run sh file for folder creation
if not os.path.isdir("log/"):
print("Directory creation script is running...")
subprocess.call([f_prefix + '/make_directories.sh'])
ofile = "./data/validation/simulation/simulation_test.txt"
method_name = get_method_name(sample_args.method)
model_name = "LSTM"
save_tar_name = method_name + "_lstm_model_"
if sample_args.gru:
model_name = "GRU"
save_tar_name = method_name + "_gru_model_"
print("Selected method name: ", method_name, " model name: ", model_name)
# Save directory
save_directory = os.path.join(f_prefix, 'model/', method_name, model_name)
# plot directory for plotting in the future
plot_directory = os.path.join(f_prefix, 'plot/', method_name, model_name)
result_directory = os.path.join(f_prefix, 'result/', method_name)
plot_test_file_directory = 'test'
# Define the path for the config file for saved args
with open(os.path.join(save_directory, 'config.pkl'), 'rb') as f:
saved_args = pickle.load(f)
seq_lenght = sample_args.pred_length + sample_args.obs_length
# Create the DataLoader object
dataloader = DataLoader(f_prefix,
1,
seq_lenght,
forcePreProcess=True,
infer=True)
odataloader = DataLoader(ofile,
1,
seq_lenght,
forcePreProcess=True,
infer=True)
create_directories(os.path.join(result_directory, model_name),
dataloader.get_all_directory_namelist())
create_directories(plot_directory, [plot_test_file_directory])
dataloader.reset_batch_pointer()
dataset_pointer_ins = dataloader.dataset_pointer
more_than_one = False
pedid = 0
smallest_err = 100000
smallest_err_iter_num = -1
origin = (0, 0)
reference_point = (0, 1)
submission_store = [] # store submission data points (txt)
result_store = [] # store points for plotting
beg = 0
lowest_ade = 1
lowest_fde = 1
highest_ade = 0
highest_fde = 0
for iteration in range(sample_args.iteration):
# Initialize net
net = get_model(sample_args.method, saved_args, True)
if sample_args.use_cuda:
net = net.cuda()
# Get the checkpoint path
checkpoint_path = os.path.join(
save_directory, save_tar_name + str(sample_args.epoch) + '.tar')
if os.path.isfile(checkpoint_path):
print('Loading checkpoint')
checkpoint = torch.load(checkpoint_path)
model_epoch = checkpoint['epoch']
net.load_state_dict(checkpoint['state_dict'])
print('Loaded checkpoint at epoch', model_epoch)
# For each batch
iteration_submission = []
iteration_result = []
results = []
submission = []
# Variable to maintain total error
total_error = 0
final_error = 0
totalade = 0
totalfde = 0
old_n_p = 0
for batch in range(dataloader.num_batches):
start = time.time()
# Get data
x, y, d, numPedsList, PedsList, target_ids = dataloader.next_batch(
)
# Get the sequence
x_seq, d_seq, numPedsList_seq, PedsList_seq, target_id = x[0], d[0], numPedsList[0], PedsList[0], \
target_ids[0]
dataloader.clean_test_data(x_seq, target_id,
sample_args.obs_length,
sample_args.pred_length)
dataloader.clean_ped_list(x_seq, PedsList_seq, target_id,
sample_args.obs_length,
sample_args.pred_length)
# get processing file name and then get dimensions of file
folder_name = dataloader.get_directory_name_with_pointer(d_seq)
dataset_data = dataloader.get_dataset_dimension(folder_name)
# dense vector creation
x_seq, lookup_seq = dataloader.convert_proper_array(
x_seq, numPedsList_seq, PedsList_seq)
# will be used for error calculation
orig_x_seq = x_seq.clone()
target_id_values = orig_x_seq[0][lookup_seq[target_id], 0:2]
# grid mask calculation
if sample_args.method == 2: # obstacle lstm
grid_seq = getSequenceGridMask(x_seq, dataset_data,
PedsList_seq,
saved_args.neighborhood_size,
saved_args.grid_size,
saved_args.use_cuda, True)
elif sample_args.method == 1: # social lstm
grid_seq = getSequenceGridMask(x_seq, dataset_data,
PedsList_seq,
saved_args.neighborhood_size,
saved_args.grid_size,
saved_args.use_cuda)
# vectorize datapoints
x_seq, first_values_dict = vectorize_seq(x_seq, PedsList_seq,
lookup_seq)
# <------------- Experimental block ---------------->
# x_seq = translate(x_seq, PedsList_seq, lookup_seq ,target_id_values)
# angle = angle_between(reference_point, (x_seq[1][lookup_seq[target_id], 0].data.numpy(), x_seq[1][lookup_seq[target_id], 1].data.numpy()))
# x_seq = rotate_traj_with_target_ped(x_seq, angle, PedsList_seq, lookup_seq)
# grid_seq = getSequenceGridMask(x_seq[:sample_args.obs_length], dataset_data, PedsList_seq, saved_args.neighborhood_size, saved_args.grid_size, sample_args.use_cuda)
# x_seq, first_values_dict = vectorize_seq(x_seq, PedsList_seq, lookup_seq)
if sample_args.use_cuda:
x_seq = x_seq.cuda()
# The sample function
if sample_args.method == 3: # vanilla lstm
# Extract the observed part of the trajectories
obs_traj, obs_PedsList_seq = x_seq[:sample_args.
obs_length], PedsList_seq[:
sample_args
.
obs_length]
ret_x_seq = sample(obs_traj, obs_PedsList_seq, sample_args,
net, x_seq, PedsList_seq, saved_args,
dataset_data, dataloader, lookup_seq,
numPedsList_seq, sample_args.gru)
else:
# Extract the observed part of the trajectories
obs_traj, obs_PedsList_seq, obs_grid = x_seq[:sample_args.
obs_length], PedsList_seq[:
sample_args
.
obs_length], grid_seq[:
sample_args
.
obs_length]
ret_x_seq = sample(obs_traj, obs_PedsList_seq, sample_args,
net, x_seq, PedsList_seq, saved_args,
dataset_data, dataloader, lookup_seq,
numPedsList_seq, sample_args.gru, obs_grid)
# revert the points back to original space
ret_x_seq = revert_seq(ret_x_seq, PedsList_seq, lookup_seq,
first_values_dict)
# <--------------------- Experimental inverse block ---------------------->
# ret_x_seq = revert_seq(ret_x_seq, PedsList_seq, lookup_seq, target_id_values, first_values_dict)
# ret_x_seq = rotate_traj_with_target_ped(ret_x_seq, -angle, PedsList_seq, lookup_seq)
# ret_x_seq = translate(ret_x_seq, PedsList_seq, lookup_seq ,-target_id_values)
# Record the mean and final displacement error
# PRUEBA
# experimment
traj_xy = np.zeros((40000, 2))
i = 0
'''
change the file to be opened
'''
if ("stpet_residence" in dataloader.train_dataset[0]):
file_validation_name = "data/validation/stpet_residence/stpet_residence_validation.txt"
if ("simulation" in dataloader.train_dataset[0]):
file_validation_name = "data/validation/simulation/simulation_validation.txt"
if ("tudresden" in dataloader.train_dataset[0]):
file_validation_name = "data/validation/tudresden/residence_real.txt"
with open(file_validation_name) as fval:
for line in fval:
t = line.split(' ')
traj_xy[i][0] = float(t[3])
traj_xy[i][1] = float(t[2])
i += 1
traj_xy_new = np.zeros((20, 2)) # 200 -- POR QUE ERA ESTO
traj_xy_new = traj_xy[beg:beg + 20]
used_ret = ret_x_seq
n_p = ret_x_seq.shape[1]
if old_n_p > n_p:
pedid = pedid - (old_n_p - n_p)
if pedid < 0:
pedid = 0
old_n_p = n_p
if more_than_one == True:
used_ret = ret_x_seq[:, pedid, :]
if n_p > 1 and more_than_one == False:
more_than_one = True
pedid = 0
used_ret = ret_x_seq[:, pedid, :]
ground_truth = traj_xy[beg:beg + 8]
beg += 20
# fin del experimento
# PRUEBA
data = np.array(traj_xy_new, dtype='float')
ground_t = np.array(ground_truth, dtype='float')
results = np.array(ret_x_seq, dtype='float')
results_julio = np.array(used_ret, dtype='float')
x, y = data.T
xx, yy = results[:, pedid].T
xxx, yyy = ground_t.T
adep = np.zeros(20)
for j in range(20):
adep[j] = (data[j][0] - results[j][pedid][0])**2 + (
data[j][1] - results[j][pedid][1])**2
fde = np.sqrt((data[19][0] - results[19][pedid][0])**2 +
(data[19][1] - results[19][pedid][1])**2)
ade = sum(adep) / 12
if ade < lowest_ade:
lowest_ade = ade
if fde < lowest_fde:
lowest_fde = fde
if ade > highest_ade:
highest_ade = ade
if fde > highest_fde:
highest_fde = fde
pedid = pedid + 1
if pedid == n_p:
pedid = 0
more_than_one = False
n_p = 1
totalade = totalade + ade
totalfde = totalfde + fde
print("ADE = ", ade)
print("FDE = ", fde)
print("LOWEST ADE: ", lowest_ade)
print("LOWEST FDE: ", lowest_fde)
print("HIGHEST ADE: ", highest_ade)
print("HIGHEST FDE: ", highest_fde)
'''
PREDICTION OF THIS TRAJECTORY
'''
plt.plot(x,
y,
color='green',
linestyle='-',
marker='+',
label='ground truth')
plt.scatter(xx,
yy,
color='b',
linestyle='-',
marker='o',
label='prediction')
plt.scatter(xxx,
yyy,
color='red',
marker='s',
label='observed steps')
plt.axis('equal')
plt.ylim((-6, 6))
plt.xlim((-6, 6))
plt.grid(True)
plt.legend(loc='best')
plt.show()
'''
ALL TRAJECTORIES GENERATED
To observe a superposition of only the ground truth of the trajectories, comment the lines of "show" , scatter "xx",
scatter "xxx" and "legend" above. Uncomment the two lines below. The files with the evolution are then stores
in "plot/Superposition"
'''
#picturename = "plot/Superposition/" + str(int(beg / 20)) + "trajectories.png"
#plt.savefig(picturename)
'''
total_error += get_mean_error(ret_x_seq[1:sample_args.obs_length].data,
orig_x_seq[1:sample_args.obs_length].data,
PedsList_seq[1:sample_args.obs_length],
PedsList_seq[1:sample_args.obs_length], sample_args.use_cuda, lookup_seq)
final_error += get_final_error(ret_x_seq[1:sample_args.obs_length].data,
orig_x_seq[1:sample_args.obs_length].data,
PedsList_seq[1:sample_args.obs_length],
PedsList_seq[1:sample_args.obs_length], lookup_seq)
print("TOTAL ERROR: ", total_error)
print("FINAL ERROR: ", final_error)
'''
end = time.time()
print('Current file : ', dataloader.get_file_name(0),
' Processed trajectory number : ', batch + 1, 'out of',
dataloader.num_batches, 'trajectories in time', end - start)
if dataset_pointer_ins is not dataloader.dataset_pointer:
if dataloader.dataset_pointer is not 0:
iteration_submission.append(submission)
iteration_result.append(results)
dataset_pointer_ins = dataloader.dataset_pointer
submission = []
results = []
submission.append(
submission_preprocess(
dataloader,
ret_x_seq.data[sample_args.obs_length:,
lookup_seq[target_id], :].numpy(),
sample_args.pred_length, sample_args.obs_length,
target_id))
# -- LO HE COMENTADO YO
# results.append((x_seq.data.cpu().numpy(), ret_x_seq.data.cpu().numpy(), PedsList_seq, lookup_seq,
# -- dataloader.get_frame_sequence(seq_lenght), target_id, sample_args.obs_length))
avade = totalade / dataloader.num_batches
avfde = totalfde / dataloader.num_batches
print("ADE AVERAGE OF ALL:", avade)
print("FDE AVERAGE OF ALL:", avfde)
iteration_submission.append(submission)
iteration_result.append(results)
submission_store.append(iteration_submission)
result_store.append(iteration_result)
print('Iteration:', iteration + 1,
' Total training (observed part) mean error of the model is ',
total_error / dataloader.num_batches)
print('Iteration:', iteration + 1,
'Total training (observed part) final error of the model is ',
final_error / dataloader.num_batches)
# print(submission)
print('Smallest error iteration:', smallest_err_iter_num + 1)
dataloader.write_to_file(submission_store[smallest_err_iter_num],
result_directory, prefix, model_name)
dataloader.write_to_plot_file(
result_store[smallest_err_iter_num],
os.path.join(plot_directory, plot_test_file_directory))
dataset_data = dataloader.get_dataset_dimension(folder_name)
# dense vector creation
x_seq, lookup_seq = dataloader.convert_proper_array(
x_seq, numPedsList_seq, PedsList_seq)
# will be used for error calculation
orig_x_seq = x_seq.clone()
target_id_values = orig_x_seq[0][lookup_seq[target_id], 0:2]
# grid mask calculation
if sample_args.method == 2: # obstacle lstm
grid_seq = getSequenceGridMask(x_seq, dataset_data,
PedsList_seq,
saved_args.neighborhood_size,
saved_args.grid_size,
saved_args.use_cuda, True)
elif sample_args.method == 1: # social lstm
grid_seq = getSequenceGridMask(x_seq, dataset_data,
PedsList_seq,
saved_args.neighborhood_size,
saved_args.grid_size,
saved_args.use_cuda)
# Vectorize data points
x_seq, first_values_dict = vectorize_seq(x_seq, PedsList_seq,
lookup_seq)
# <------------- Experimental block ----------------> x_seq = translate(x_seq, PedsList_seq, lookup_seq ,
# target_id_values) angle = angle_between(reference_point, (x_seq[1][lookup_seq[target_id],
def main():
np.random.seed(1)
parser = argparse.ArgumentParser()
# 观测轨迹长度
parser.add_argument('--obs_length',
type=int,
default=7,
help='Observed length of the trajectory')
# 预测轨迹长度
parser.add_argument('--pred_length',
type=int,
default=5,
help='Predicted length of the trajectory')
# 测试数据集
parser.add_argument('--test_dataset',
type=int,
default=2,
help='Epoch of model to be loaded')
# 导入的模型
parser.add_argument('--epoch',
type=int,
default=8,
help='Epoch of model to be loaded')
sample_args = parser.parse_args(args=[])
# 存储历史
save_directory = 'save/' + str(sample_args.test_dataset) + '/'
# Define the path for the config file for saved args
with open(os.path.join(save_directory, 'social_config.pkl'), 'rb') as f:
saved_args = pickle.load(f)
# Create a SocialModel object with the saved_args and infer set to true
model = SocialModel(saved_args, True)
# Initialize a TensorFlow session
config = tf.ConfigProto(
log_device_placement=True
) # Showing which device is allocated (in case of multiple GPUs)
config.gpu_options.per_process_gpu_memory_fraction = 0.8 # Allocating 20% of memory in each GPU
sess = tf.InteractiveSession(config=config)
# Initialize a saver
saver = tf.train.Saver()
# Get the checkpoint state for the model
ckpt = tf.train.get_checkpoint_state(save_directory)
# print ('loading model: ', ckpt.model_checkpoint_path)
print('loading model: ',
ckpt.all_model_checkpoint_paths[sample_args.epoch])
# Restore the model at the checkpoint
saver.restore(sess, ckpt.all_model_checkpoint_paths[sample_args.epoch])
# Dataset to get data from
dataset = [sample_args.test_dataset]
# Create a SocialDataLoader object with batch_size 1 and seq_length equal to observed_length + pred_length
data_loader = SocialDataLoader(1,
sample_args.pred_length +
sample_args.obs_length,
saved_args.maxNumPeds,
dataset,
True,
infer=True)
# Reset all pointers of the data_loader
data_loader.reset_batch_pointer()
results = []
# Variable to maintain total error
total_error = 0
# For each batch
for b in range(data_loader.num_batches):
# Get the source, target and dataset data for the next batch
x, y, d = data_loader.next_batch(randomUpdate=False)
# Batch size is 1
x_batch, y_batch, d_batch = x[0], y[0], d[0]
if d_batch == 0 and dataset[0] == 0:
dimensions = [640, 480]
else:
dimensions = [720, 576]
grid_batch = getSequenceGridMask(x_batch, dimensions,
saved_args.neighborhood_size,
saved_args.grid_size)
obs_traj = x_batch[:sample_args.obs_length]
obs_grid = grid_batch[:sample_args.obs_length]
# obs_traj is an array of shape obs_length x maxNumPeds x 3
print("********************** SAMPLING A NEW TRAJECTORY", b,
"******************************")
complete_traj = model.sample(sess, obs_traj, obs_grid, dimensions,
x_batch, sample_args.pred_length)
# ipdb.set_trace()
# complete_traj is an array of shape (obs_length+pred_length) x maxNumPeds x 3
total_error += get_mean_error(complete_traj, x[0],
sample_args.obs_length,
saved_args.maxNumPeds)
print("Processed trajectory number : ", b, "out of ",
data_loader.num_batches, " trajectories")
print('Model loaded: ',
ckpt.all_model_checkpoint_paths[sample_args.epoch])
# plot_trajectories(x[0], complete_traj, sample_args.obs_length)
# return
results.append((x[0], complete_traj, sample_args.obs_length))
# Print the mean error across all the batches
print("Total mean error of the model is ",
total_error / data_loader.num_batches)
print("Saving results")
with open(os.path.join(save_directory, 'social_results.pkl'), 'wb') as f:
pickle.dump(results, f)
def main():
parser = argparse.ArgumentParser()
# Observed length of the trajectory parameter
parser.add_argument('--obs_length',
type=int,
default=8,
help='Observed length of the trajectory')
# Predicted length of the trajectory parameter
parser.add_argument('--pred_length',
type=int,
default=12,
help='Predicted length of the trajectory')
# Test dataset
parser.add_argument('--test_dataset',
type=int,
default=0,
help='Dataset to be tested on')
# Parse the parameters
sample_args = parser.parse_args()
# Define the path for the config file for saved args
with open(os.path.join('save', 'social_config.pkl'), 'rb') as f:
saved_args = pickle.load(f)
# Create a SocialModel object with the saved_args and infer set to true
model = SocialModel(saved_args, True)
# Initialize a TensorFlow session
sess = tf.InteractiveSession()
# Initialize a saver
saver = tf.train.Saver()
# Get the checkpoint state for the model
ckpt = tf.train.get_checkpoint_state('save')
print('loading model: ', ckpt.model_checkpoint_path)
# Restore the model at the checkpoint
saver.restore(sess, ckpt.model_checkpoint_path)
# saver.restore(sess, 'save/social_model.ckpt-800')
# Dataset to get data from
dataset = [sample_args.test_dataset]
# Create a SocialDataLoader object with batch_size 1 and seq_length equal to observed_length + pred_length
data_loader = SocialDataLoader(
1, sample_args.pred_length + sample_args.obs_length,
saved_args.maxNumPeds, dataset, True)
# Reset all pointers of the data_loader
data_loader.reset_batch_pointer()
# Variable to maintain total error
total_error = 0
# For each batch
for b in range(data_loader.num_batches):
# Get the source, target and dataset data for the next batch
x, y, d = data_loader.next_batch()
# Batch size is 1
x_batch, y_batch, d_batch = x[0], y[0], d[0]
if d_batch == 0 and dataset[0] == 0:
dimensions = [640, 480]
else:
dimensions = [720, 576]
grid_batch = getSequenceGridMask(x_batch, dimensions,
saved_args.neighborhood_size,
saved_args.grid_size)
obs_traj = x_batch[:sample_args.obs_length]
obs_grid = grid_batch[:sample_args.obs_length]
# obs_traj is an array of shape obs_length x maxNumPeds x 3
complete_traj = model.sample(sess, obs_traj, obs_grid, dimensions,
x_batch, sample_args.pred_length)
# ipdb.set_trace()
# complete_traj is an array of shape (obs_length+pred_length) x maxNumPeds x 3
total_error += get_mean_error(complete_traj, x[0],
sample_args.obs_length,
saved_args.maxNumPeds)
print "Processed trajectory number : ", b, "out of ", data_loader.num_batches, " trajectories"
# Print the mean error across all the batches
print "Total mean error of the model is ", total_error / data_loader.num_batches
def validLoss(net, valid_loader, args):
'''
Calculates log-likelihood loss on validation dataset
:return: average log-likelihood loss
'''
with torch.no_grad():
num_seen_sequences = 0
total_loss = 0
for batch_idx, batch in enumerate(valid_loader):
loss_batch = 0
# Check if last batch is shorter that batch_size
# batch_size = len(batch) if (len(batch) < args.batch_size) else args.batch_size
if len(batch) < args.batch_size:
continue
# For each sequence
for sequence in range(args.batch_size):
# Get the data corresponding to the current sequence
x_seq, num_peds_list_seq, peds_list_seq, folder_path = batch[
sequence]
# Dense vector (tensor) creation
x_seq, lookup_seq = convertToTensor(x_seq, peds_list_seq)
# Get processing file name and then get dimensions of file
folder_name = getFolderName(folder_path, args.dataset)
dataset_data = dataset_dimensions[folder_name]
# Grid mask calculation and storage depending on grid parameter
grid_seq = getSequenceGridMask(x_seq, dataset_data,
peds_list_seq,
args.neighborhood_size,
args.grid_size, args.use_cuda)
# Vectorize trajectories in sequence
x_seq, _ = vectorizeSeq(x_seq, peds_list_seq, lookup_seq)
if args.use_cuda:
x_seq = x_seq.cuda()
# Number of peds in this sequence per frame
numNodes = len(lookup_seq)
hidden_states = Variable(torch.zeros(numNodes, args.rnn_size))
if args.use_cuda:
hidden_states = hidden_states.cuda()
cell_states = Variable(torch.zeros(numNodes, args.rnn_size))
if args.use_cuda:
cell_states = cell_states.cuda()
# Forward prop
outputs, _, _ = net(x_seq, grid_seq, hidden_states,
cell_states, peds_list_seq,
num_peds_list_seq, valid_loader,
lookup_seq)
# Increment number of seen sequences
num_seen_sequences += 1
# Compute loss
loss = Gaussian2DLikelihood(outputs, x_seq, peds_list_seq,
lookup_seq)
loss_batch += loss.item()
total_loss += loss_batch
return total_loss / num_seen_sequences
def main():
parser = argparse.ArgumentParser()
# Observed length of the trajectory parameter
parser.add_argument('--obs_length',
type=int,
default=4,
help='Observed length of the trajectory')
# Predicted length of the trajectory parameter
parser.add_argument('--pred_length',
type=int,
default=4,
help='Predicted length of the trajectory')
# Custom scenario to be tested on
parser.add_argument('--scenario',
type=str,
default='collision',
help='Custom scenario to be tested on')
sample_args = parser.parse_args()
# Define the path for the config file for saved args
with open(os.path.join('save', 'social_config.pkl'), 'rb') as f:
saved_args = pickle.load(f)
# Create a SocialModel object with the saved_args and infer set to true
model = SocialModel(saved_args, True)
# Initialize a TensorFlow session
sess = tf.InteractiveSession()
# Initialize a saver
saver = tf.train.Saver()
# Get the checkpoint state for the model
ckpt = tf.train.get_checkpoint_state('save')
print('loading model: ', ckpt.model_checkpoint_path)
# Restore the model at the checkpoint
saver.restore(sess, ckpt.model_checkpoint_path)
results = []
# Load the data
file_path = os.path.join('matlab', 'csv', sample_args.scenario + '.csv')
data = np.genfromtxt(file_path, delimiter=',')
# Reshape data
x_batch = np.reshape(data, [
sample_args.obs_length + sample_args.pred_length,
saved_args.maxNumPeds, 3
])
dimensions = [720, 576]
grid_batch = getSequenceGridMask(x_batch, [720, 576],
saved_args.neighborhood_size,
saved_args.grid_size)
obs_traj = x_batch[:sample_args.obs_length]
obs_grid = grid_batch[:sample_args.obs_length]
complete_traj = model.sample(sess, obs_traj, obs_grid, dimensions, x_batch,
sample_args.pred_length)
total_error = get_mean_error(complete_traj, x_batch,
sample_args.obs_length, saved_args.maxNumPeds)
print "Mean error of the model on this scenario is", total_error
def train(args, _run):
origin = (0, 0)
reference_point = (0, 1)
# Set directory to save the trained model
inner_dir = args.save_prefix
if inner_dir is None:
inner_dir = 'tmp' if _run._id is None else str(_run._id)
save_directory = os.path.join(args.save_dir, inner_dir)
if os.path.isdir(save_directory):
shutil.rmtree(save_directory)
train_loader, valid_loader = loadData(args.train_dataset_path,
args.orig_seq_len,
args.keep_every,
args.valid_percentage,
args.batch_size,
args.max_val_size,
args.persons_to_keep,
filename=args.dataset_filename)
model_name = "LSTM"
method_name = "SOCIALLSTM"
save_tar_name = method_name + "_lstm_model_"
if args.gru:
model_name = "GRU"
save_tar_name = method_name + "_gru_model_"
# Save the arguments int the config file
os.makedirs(save_directory, exist_ok=True) #TODO: fix this!
with open(os.path.join(save_directory, 'config.json'), 'w') as f:
json.dump(args, f)
# Path to store the checkpoint file (trained model)
def checkpoint_path(x):
return os.path.join(save_directory, save_tar_name + str(x) + '.tar')
# model creation
net = SocialModel(args)
if args.use_cuda:
net = net.cuda()
optimizer = torch.optim.Adagrad(net.parameters(),
weight_decay=args.lambda_param)
num_batch = 0
# Training
for epoch in range(args.num_epochs):
print('****************Training epoch beginning******************')
loss_epoch = 0
num_seen_sequences = 0
# For each batch
for batch_idx, batch in enumerate(train_loader):
start = time.time()
loss_batch = 0
# Check if last batch is shorter that batch_size
# batch_size = len(batch) if (len(batch) < args.batch_size) else args.batch_size
if len(batch) < args.batch_size:
continue
# For each sequence
for sequence in range(args.batch_size):
# Get the data corresponding to the current sequence
x_seq, num_peds_list_seq, peds_list_seq, folder_path = batch[
sequence]
# Dense vector (tensor) creation
x_seq, lookup_seq = convertToTensor(x_seq, peds_list_seq)
# Get processing file name and then get dimensions of file
folder_name = getFolderName(folder_path, args.dataset)
dataset_data = dataset_dimensions[folder_name]
# Grid mask calculation and storage depending on grid parameter
grid_seq = getSequenceGridMask(x_seq, dataset_data,
peds_list_seq,
args.neighborhood_size,
args.grid_size, args.use_cuda)
# Replace relative positions with true positions in x_seq
x_seq, _ = vectorizeSeq(x_seq, peds_list_seq, lookup_seq)
if args.use_cuda:
x_seq = x_seq.cuda()
# Number of peds in this sequence
numNodes = len(lookup_seq)
hidden_states = Variable(torch.zeros(numNodes, args.rnn_size))
if args.use_cuda:
hidden_states = hidden_states.cuda()
cell_states = Variable(torch.zeros(numNodes, args.rnn_size))
if args.use_cuda:
cell_states = cell_states.cuda()
# Zero out gradients
net.zero_grad()
optimizer.zero_grad()
# Forward prop
outputs, _, _ = net(x_seq, grid_seq, hidden_states,
cell_states, peds_list_seq,
num_peds_list_seq, train_loader,
lookup_seq)
# Increment number of seen sequences
num_seen_sequences += 1
# Debug
# Compute loss
loss = Gaussian2DLikelihood(outputs, x_seq, peds_list_seq,
lookup_seq)
loss_batch += loss.item()
# Free the memory
# *basketball*
del x_seq
del hidden_states
del cell_states
torch.cuda.empty_cache()
# Compute gradients
loss.backward()
# Clip gradients
torch.nn.utils.clip_grad_norm_(net.parameters(),
args.grad_clip)
# Update parameters
optimizer.step()
end = time.time()
loss_epoch += loss_batch
num_batch += 1
num_batches = math.floor(
len(train_loader.dataset) / args.batch_size)
print('{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}'.
format(epoch * num_batches + batch_idx,
args.num_epochs * num_batches, epoch, loss_batch,
end - start))
'''
if args.validate:
# Validate
if batch_idx % 5000 == 0:
if len(valid_loader) > 0:
#TEST
t_dataset, _ = torch.utils.data.random_split(all_datasets, [1000, len(all_datasets)-1000])
# Create the data loader objects
t_loader = DataLoader(t_dataset, batch_size=args.batch_size, shuffle=False, num_workers=4,
pin_memory=False,
collate_fn=lambda x: x)
t_loss = validLoss(net, t_loader, args)
_run.log_scalar(metric_name='t.loss', value=t_loss, step=epoch + batch_idx / num_batches)
ttt_loss = loss_epoch / num_seen_sequences
_run.log_scalar(metric_name='ttt.loss', value=ttt_loss, step=epoch + batch_idx / num_batches)
valid_loss = validLoss(net, valid_loader, args)
total_error, final_error, norm_l2_dists = testHelper(net, valid_loader, args, args)
total_error = total_error.item() if isinstance(total_error, torch.Tensor) else total_error
final_error = final_error.item() if isinstance(final_error, torch.Tensor) else final_error
_run.log_scalar(metric_name='valid.loss', value=valid_loss, step=epoch+batch_idx/num_batches)
_run.log_scalar(metric_name='valid.total_error', value=total_error, step=epoch+batch_idx/num_batches)
_run.log_scalar(metric_name='valid.final_error', value=final_error, step=epoch+batch_idx/num_batches)
for i, l in enumerate(norm_l2_dists):
error = norm_l2_dists[i].item() if isinstance(norm_l2_dists[i], torch.Tensor) else norm_l2_dists[i]
_run.log_scalar(metric_name=f'valid.norm_l2_dist_{i}', value=error, step=epoch+batch_idx/num_batches)
'''
loss_epoch /= num_seen_sequences
# Log loss values
#log_file_curve.write("Training epoch: " + str(epoch) + " loss: " + str(loss_epoch) + '\n')
# Sacred metrics plot
_run.log_scalar(metric_name='train.loss', value=loss_epoch, step=epoch)
if args.validate:
# Validate
if len(valid_loader) > 0:
mux, muy, sx, sy, corr = getCoef(outputs)
#import pdb; pdb.set_trace()
_run.log_scalar(metric_name='valid.mux',
value=torch.mean(mux).item(),
step=epoch)
_run.log_scalar(metric_name='valid.muy',
value=torch.mean(muy).item(),
step=epoch)
_run.log_scalar(metric_name='valid.sx',
value=torch.mean(sx).item(),
step=epoch)
_run.log_scalar(metric_name='valid.sy',
value=torch.mean(sy).item(),
step=epoch)
valid_loss = validLoss(net, valid_loader, args)
total_error, final_error, norm_l2_dists = testHelper(
net, valid_loader, args, args)
total_error = total_error.item() if isinstance(
total_error, torch.Tensor) else total_error
final_error = final_error.item() if isinstance(
final_error, torch.Tensor) else final_error
_run.log_scalar(metric_name='valid.loss',
value=valid_loss,
step=epoch)
_run.log_scalar(metric_name='valid.total_error',
value=total_error,
step=epoch)
_run.log_scalar(metric_name='valid.final_error',
value=final_error,
step=epoch)
for i, l in enumerate(norm_l2_dists):
error = norm_l2_dists[i].item() if isinstance(
norm_l2_dists[i], torch.Tensor) else norm_l2_dists[i]
_run.log_scalar(metric_name=f'valid.norm_l2_dist_{i}',
value=error,
step=epoch)
# Save the model after each epoch
print('Saving model')
torch.save(
{
'epoch': epoch,
'state_dict': net.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, checkpoint_path(epoch))
def main():
parser = argparse.ArgumentParser()
# Model to be loaded
# RNN size parameter (dimension of the output/hidden state)
parser.add_argument('--input_size', type=int, default=2)
parser.add_argument('--output_size', type=int, default=5)
parser.add_argument('--seq_length', type=int, default=20,
help='RNN sequence length')
# Size of each batch parameter
parser.add_argument('--batch_size', type=int, default=10,
help='minibatch size')
parser.add_argument('--num_samples', type=int, default=500,
help='NUmber of random configuration will be tested')
parser.add_argument('--num_epochs', type=int, default=3,
help='number of epochs')
# Maximum number of pedestrians to be considered
parser.add_argument('--maxNumPeds', type=int, default=27,
help='Maximum Number of Pedestrians')
# cuda support
parser.add_argument('--use_cuda', action="store_true", default=False,
help='Use GPU or not')
# drive support
parser.add_argument('--drive', action="store_true", default=False,
help='Use Google drive or not')
# number of validation dataset will be used
parser.add_argument('--num_validation', type=int, default=1,
help='Total number of validation dataset will be visualized')
# gru model
parser.add_argument('--gru', action="store_true", default=False,
help='True : GRU cell, False: LSTM cell')
# method selection for hyperparameter
parser.add_argument('--method', type=int, default=1,
help='Method of lstm will be used (1 = social lstm, 2 = obstacle lstm, 3 = vanilla lstm)')
# number of parameter set will be logged
parser.add_argument('--best_n', type=int, default=100,
help='Number of best n configuration will be logged')
# Parse the parameters
#sample_args = parser.parse_args()
args = parameters(parser)
args.best_n = np.clip(args.best_n, 0, args.num_samples)
#for drive run
prefix = ''
f_prefix = '.'
if args.drive is True:
prefix='drive/semester_project/social_lstm_final/'
f_prefix = 'drive/semester_project/social_lstm_final'
method_name = getMethodName(args.method)
model_name = "LSTM"
save_tar_name = method_name+"_lstm_model_"
if args.gru:
model_name = "GRU"
save_tar_name = method_name+"_gru_model_"
#plot directory for plotting in the future
param_log = os.path.join(f_prefix)
param_log_file = "hyperparameter"
origin = (0,0)
reference_point = (0,1)
score = []
param_set = []
# Create the DataLoader object
createDirectories(param_log, [param_log_file])
log_file = open(os.path.join(param_log, param_log_file, 'log.txt'), 'w+')
dataloader_t = DataLoader(f_prefix, args.batch_size, args.seq_length, num_of_validation = args.num_validation, forcePreProcess = True, infer = True)
dataloader_v = DataLoader(f_prefix, 1, args.seq_length, num_of_validation = args.num_validation, forcePreProcess = True, infer = True)
for hyperparams in itertools.islice(sample_hyperparameters(), args.num_samples):
args = parameters(parser)
# randomly sample a parameter set
args.rnn_size = hyperparams.pop("rnn_size")
args.learning_schedule = hyperparams.pop("learning_schedule")
args.grad_clip = hyperparams.pop("grad_clip")
args.learning_rate = hyperparams.pop("learning_rate")
args.lambda_param = hyperparams.pop("lambda_param")
args.dropout = hyperparams.pop("dropout")
args.embedding_size = hyperparams.pop("embedding_size")
args.neighborhood_size = hyperparams.pop("neighborhood_size")
args.grid_size = hyperparams.pop("grid_size")
log_file.write("##########Parameters########"+'\n')
print("##########Parameters########")
write_to_file(log_file, args)
print_to_screen(args)
net = getModel(args.method, args)
if args.use_cuda:
net = net.cuda()
if(args.learning_schedule == "RMSprop"):
optimizer = torch.optim.RMSprop(net.parameters(), lr=args.learning_rate)
elif(args.learning_schedule == "adagrad"):
optimizer = torch.optim.Adagrad(net.parameters(), weight_decay=args.lambda_param)
else:
optimizer = torch.optim.Adam(net.parameters(), weight_decay=args.lambda_param)
learning_rate = args.learning_rate
total_process_start = time.time()
# Training
for epoch in range(args.num_epochs):
print('****************Training epoch beginning******************')
dataloader_t.reset_batch_pointer()
loss_epoch = 0
# For each batch
for batch in range(dataloader_t.num_batches):
start = time.time()
# Get batch data
x, y, d , numPedsList, PedsList ,target_ids = dataloader_t.next_batch()
loss_batch = 0
# For each sequence
for sequence in range(dataloader_t.batch_size):
# Get the data corresponding to the current sequence
x_seq ,_ , d_seq, numPedsList_seq, PedsList_seq = x[sequence], y[sequence], d[sequence], numPedsList[sequence], PedsList[sequence]
target_id = target_ids[sequence]
#get processing file name and then get dimensions of file
folder_name = dataloader_t.get_directory_name_with_pointer(d_seq)
dataset_data = dataloader_t.get_dataset_dimension(folder_name)
#dense vector creation
x_seq, lookup_seq = dataloader_t.convert_proper_array(x_seq, numPedsList_seq, PedsList_seq)
target_id_values = x_seq[0][lookup_seq[target_id], 0:2]
#grid mask calculation
if args.method == 2: #obstacle lstm
grid_seq = getSequenceGridMask(x_seq, dataset_data, PedsList_seq, args.neighborhood_size, args.grid_size, args.use_cuda, True)
elif args.method == 1: #social lstm
grid_seq = getSequenceGridMask(x_seq, dataset_data, PedsList_seq, args.neighborhood_size, args.grid_size, args.use_cuda)
# vectorize trajectories in sequence
x_seq, _ = vectorizeSeq(x_seq, PedsList_seq, lookup_seq)
if args.use_cuda:
x_seq = x_seq.cuda()
#number of peds in this sequence per frame
numNodes = len(lookup_seq)
hidden_states = Variable(torch.zeros(numNodes, args.rnn_size))
if args.use_cuda:
hidden_states = hidden_states.cuda()
cell_states = Variable(torch.zeros(numNodes, args.rnn_size))
if args.use_cuda:
cell_states = cell_states.cuda()
# Zero out gradients
net.zero_grad()
optimizer.zero_grad()
# Forward prop
if args.method == 3: #vanilla lstm
outputs, _, _ = net(x_seq, hidden_states, cell_states, PedsList_seq,numPedsList_seq ,dataloader_t, lookup_seq)
else:
outputs, _, _ = net(x_seq, grid_seq, hidden_states, cell_states, PedsList_seq,numPedsList_seq ,dataloader_t, lookup_seq)
# Compute loss
loss = Gaussian2DLikelihood(outputs, x_seq, PedsList_seq, lookup_seq)
loss_batch += loss.item()
# Compute gradients
loss.backward()
# Clip gradients
torch.nn.utils.clip_grad_norm_(net.parameters(), args.grad_clip)
# Update parameters
optimizer.step()
end = time.time()
loss_batch = loss_batch / dataloader_t.batch_size
loss_epoch += loss_batch
print('{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}'.format(epoch * dataloader_t.num_batches + batch,
args.num_epochs * dataloader_t.num_batches,
epoch,
loss_batch, end - start))
loss_epoch /= dataloader_t.num_batches
# Log loss values
log_file.write("Training epoch: "+str(epoch)+" loss: "+str(loss_epoch)+'\n')
net = getModel(args.method, args, True)
if args.use_cuda:
net = net.cuda()
if(args.learning_schedule == "RMSprop"):
optimizer = torch.optim.RMSprop(net.parameters(), lr=args.learning_rate)
elif(args.learning_schedule == "adagrad"):
optimizer = torch.optim.Adagrad(net.parameters(), weight_decay=args.lambda_param)
else:
optimizer = torch.optim.Adam(net.parameters(), weight_decay=args.lambda_param)
print('****************Validation dataset batch processing******************')
dataloader_v.reset_batch_pointer()
dataset_pointer_ins = dataloader_v.dataset_pointer
loss_epoch = 0
err_epoch = 0
f_err_epoch = 0
num_of_batch = 0
smallest_err = 100000
# For each batch
for batch in range(dataloader_v.num_batches):
start = time.time()
# Get batch data
x, y, d , numPedsList, PedsList ,target_ids = dataloader_v.next_batch()
if dataset_pointer_ins is not dataloader_v.dataset_pointer:
if dataloader_v.dataset_pointer is not 0:
print('Finished prosessed file : ', dataloader_v.get_file_name(-1),' Avarage error : ', err_epoch/num_of_batch)
num_of_batch = 0
dataset_pointer_ins = dataloader_v.dataset_pointer
# Loss for this batch
loss_batch = 0
err_batch = 0
f_err_batch = 0
# For each sequence
for sequence in range(dataloader_v.batch_size):
# Get data corresponding to the current sequence
x_seq ,_ , d_seq, numPedsList_seq, PedsList_seq = x[sequence], y[sequence], d[sequence], numPedsList[sequence], PedsList[sequence]
target_id = target_ids[sequence]
folder_name = dataloader_v.get_directory_name_with_pointer(d_seq)
dataset_data = dataloader_v.get_dataset_dimension(folder_name)
#dense vector creation
x_seq, lookup_seq = dataloader_v.convert_proper_array(x_seq, numPedsList_seq, PedsList_seq)
#will be used for error calculation
orig_x_seq = x_seq.clone()
target_id_values = x_seq[0][lookup_seq[target_id], 0:2]
#grid mask calculation
if args.method == 2: #obstacle lstm
grid_seq = getSequenceGridMask(x_seq, dataset_data, PedsList_seq, args.neighborhood_size, args.grid_size, args.use_cuda, True)
elif args.method == 1: #social lstm
grid_seq = getSequenceGridMask(x_seq, dataset_data, PedsList_seq, args.neighborhood_size, args.grid_size, args.use_cuda)
# vectorize trajectories in sequence
x_seq, first_values_dict = vectorizeSeq(x_seq, PedsList_seq, lookup_seq)
# <--------------Experimental block --------------->
# Construct variables
# x_seq, lookup_seq = dataloader_v.convert_proper_array(x_seq, numPedsList_seq, PedsList_seq)
# x_seq, target_id_values, first_values_dict = vectorize_seq_with_ped(x_seq, PedsList_seq, lookup_seq ,target_id)
# angle = angle_between(reference_point, (x_seq[1][lookup_seq[target_id], 0].data.numpy(), x_seq[1][lookup_seq[target_id], 1].data.numpy()))
# x_seq = rotate_traj_with_target_ped(x_seq, angle, PedsList_seq, lookup_seq)
if args.use_cuda:
x_seq = x_seq.cuda()
if args.method == 3: #vanilla lstm
ret_x_seq, loss = sampleValidationDataVanilla(x_seq, PedsList_seq, args, net, lookup_seq, numPedsList_seq, dataloader_v)
else:
ret_x_seq, loss = sample_validation_data(x_seq, PedsList_seq, grid_seq, args, net, lookup_seq, numPedsList_seq, dataloader_v)
#revert the points back to original space
ret_x_seq = revertSeq(ret_x_seq, PedsList_seq, lookup_seq, first_values_dict)
err = getMeanError(ret_x_seq.data, orig_x_seq.data, PedsList_seq, PedsList_seq, args.use_cuda, lookup_seq)
f_err = getFinalError(ret_x_seq.data, orig_x_seq.data, PedsList_seq, PedsList_seq, lookup_seq)
# ret_x_seq = rotate_traj_with_target_ped(ret_x_seq, -angle, PedsList_seq, lookup_seq)
# ret_x_seq = revert_seq(ret_x_seq, PedsList_seq, lookup_seq, target_id_values, first_values_dict)
loss_batch += loss.item()
err_batch += err
f_err_batch += f_err
end = time.time()
print('Current file : ', dataloader_v.get_file_name(0),' Batch : ', batch+1, ' Sequence: ', sequence+1, ' Sequence mean error: ', err,' Sequence final error: ',f_err,' time: ', end - start)
loss_batch = loss_batch / dataloader_v.batch_size
err_batch = err_batch / dataloader_v.batch_size
f_err_batch = f_err_batch / dataloader_v.batch_size
num_of_batch += 1
loss_epoch += loss_batch
err_epoch += err_batch
f_err_epoch += f_err_batch
total_process_end = time.time()
if dataloader_v.num_batches != 0:
loss_epoch = loss_epoch / dataloader_v.num_batches
err_epoch = err_epoch / dataloader_v.num_batches
f_err_epoch = f_err_epoch / dataloader_v.num_batches
# calculate avarage error and time
avg_err = (err_epoch+f_err_epoch)/2
elapsed_time = (total_process_end - total_process_start)
args.time = elapsed_time
args.avg_err = avg_err
score.append(avg_err)
param_set.append(args)
print('valid_loss = {:.3f}, valid_mean_err = {:.3f}, valid_final_err = {:.3f}, score = {:.3f}, time = {:.3f}'.format(loss_epoch, err_epoch, f_err_epoch, avg_err, elapsed_time))
log_file.write('valid_loss = {:.3f}, valid_mean_err = {:.3f}, valid_final_err = {:.3f}, score = {:.3f}, time = {:.3f}'.format(loss_epoch, err_epoch, f_err_epoch, avg_err, elapsed_time)+'\n')
print("--------------------------Best ", args.best_n," configuration------------------------")
log_file.write("-----------------------------Best "+str(args.best_n) +" configuration---------------------"+'\n')
biggest_indexes = np.array(score).argsort()[-args.best_n:]
print("biggest_index: ", biggest_indexes)
for arr_index, index in enumerate(biggest_indexes):
print("&&&&&&&&&&&&&&&&&&&& ", arr_index," &&&&&&&&&&&&&&&&&&&&&&")
log_file.write("&&&&&&&&&&&&&&&&&&&& "+ str(arr_index)+" &&&&&&&&&&&&&&&&&&&&&&"+'\n')
curr_arg = param_set[index]
write_to_file(log_file, curr_arg)
print_to_screen(curr_arg)
print("score: ",score)
print('error = {:.3f}, time = {:.3f}'.format(curr_arg.avg_err, curr_arg.time))
log_file.write('error = {:.3f}, time = {:.3f}'.format(curr_arg.avg_err, curr_arg.time)+'\n')
def train(args):
if args.visible:
os.environ["CUDA_VISIBLE_DEVICES"] = args.visible
save_path = make_save_path(args)
dataset_path = args.dataset_path
log_path = os.path.join(save_path, 'log')
if not os.path.isdir(log_path):
os.makedirs(log_path)
# Create the SocialDataLoader object
data_loader = SocialDataLoader(args.batch_size, args.seq_length,
args.maxNumPeds, dataset_path, forcePreProcess=True)
with open(os.path.join(save_path, 'social_config.pkl'), 'wb') as f:
pickle.dump(args, f)
# Create a SocialModel object with the arguments
model = SocialModel(args)
all_loss = []
# Initialize a TensorFlow session
with tf.Session() as sess:
# Initialize all variables in the graph
sess.run(tf.initialize_all_variables())
# Initialize a saver that saves all the variables in the graph
saver = tf.train.Saver(tf.all_variables())
summary_writer = tf.summary.FileWriter(log_path, sess.graph)
# For each epoch
for e in range(args.num_epochs):
# Assign the learning rate value for this epoch
sess.run(tf.assign(model.lr, args.learning_rate * (args.decay_rate ** e)))
# Reset the data pointers in the data_loader
data_loader.reset_batch_pointer()
# For each batch
for b in range(data_loader.num_batches):
# Tic
start = time.time()
# Get the source, target and dataset data for the next batch
# s_batch, t_batch are input and target data which are lists containing numpy arrays of size seq_length x maxNumPeds x 3
# d is the list of dataset indices from which each batch is generated (used to differentiate between datasets)
s_batch, t_batch, d = data_loader.next_batch()
# variable to store the loss for this batch
loss_batch = 0
# For each sequence in the batch
for seq_num in range(data_loader.batch_size):
# s_seq, t_seq and d_batch contains the source, target and dataset index data for
# seq_length long consecutive frames in the dataset
# s_seq, t_seq would be numpy arrays of size seq_length x maxNumPeds x 3
# d_batch would be a scalar identifying the dataset from which this sequence is extracted
s_seq, t_seq, d_seq = s_batch[seq_num], t_batch[seq_num], d[seq_num]
'''
if d_seq == 0 and datasets[0] == 0:
dataset_data = [640, 480]
else:
dataset_data = [720, 576]
'''
grid_batch = getSequenceGridMask(s_seq, [0, 0], args.neighborhood_size, args.grid_size)
# Feed the source, target data
feed = {model.input_data: s_seq, model.target_data: t_seq, model.grid_data: grid_batch}
train_loss, _ = sess.run([model.cost, model.train_op], feed)
loss_batch += train_loss
end = time.time()
loss_batch = loss_batch / data_loader.batch_size
all_loss.append(loss_batch)
print(
"{}/{} (epoch {}), train_loss = {:.3f}, time/seq_num = {:.3f}"
.format(
e * data_loader.num_batches + b,
args.num_epochs * data_loader.num_batches,
e,
loss_batch, end - start))
# Save the model if the current epoch and batch number match the frequency
if (e * data_loader.num_batches + b) % args.save_every == 0 and ((e * data_loader.num_batches + b) > 0):
checkpoint_path = os.path.join(save_path, 'social_model.ckpt')
saver.save(sess, checkpoint_path, global_step=e * data_loader.num_batches + b)
print("model saved to {}".format(checkpoint_path))
np.savetxt(os.path.join(log_path, 'loss.txt'), np.asarray(all_loss))
def train(args):
origin = (0, 0)
reference_point = (0, 1)
validation_dataset_executed = False
# 这一片是否可以去掉
prefix = ''
f_prefix = '.'
if args.drive is True:
prefix = 'drive/semester_project/social_lstm_final/'
f_prefix = 'drive/semester_project/social_lstm_final'
# 用于从云端读取数据
if not os.path.isdir("log/"):
print("Directory creation script is running...")
subprocess.call([f_prefix + '/make_directories.sh'])
args.freq_validation = np.clip(args.freq_validation, 0, args.num_epochs)
validation_epoch_list = list(
range(args.freq_validation, args.num_epochs + 1, args.freq_validation))
validation_epoch_list[-1] -= 1
# Create the data loader object. This object would preprocess the data in terms of
# batches each of size args.batch_size, of length args.seq_length
# 读取数据
dataloader = DataLoader(f_prefix,
args.batch_size,
args.seq_length,
args.num_validation,
forcePreProcess=True)
model_name = "LSTM"
method_name = "SOCIALLSTM"
save_tar_name = method_name + "_lstm_model_"
if args.gru:
model_name = "GRU"
save_tar_name = method_name + "_gru_model_"
# Log directory
log_directory = os.path.join(prefix, 'log/')
plot_directory = os.path.join(prefix, 'plot/', method_name, model_name)
# print(plot_directory)
plot_train_file_directory = 'validation'
# Logging files
log_file_curve = open(
os.path.join(log_directory, method_name, model_name, 'log_curve.txt'),
'w+')
log_file = open(
os.path.join(log_directory, method_name, model_name, 'val.txt'), 'w+')
# model directory
save_directory = os.path.join(prefix, 'model/')
# Save the arguments int the config file
# 将参数保存在配置文件中
with open(
os.path.join(save_directory, method_name, model_name,
'config.pkl'), 'wb') as f:
pickle.dump(args, f)
# Path to store the checkpoint file
# 存储检查点文件的路径
def checkpoint_path(x):
return os.path.join(save_directory, method_name, model_name,
save_tar_name + str(x) + '.tar')
# model creation
net = SocialModel(args)
if args.use_cuda:
net = net.cuda()
optimizer = torch.optim.Adagrad(net.parameters(),
weight_decay=args.lambda_param)
learning_rate = args.learning_rate
best_val_loss = 100
best_val_data_loss = 100
smallest_err_val = 100000
smallest_err_val_data = 100000
best_epoch_val = 0
best_epoch_val_data = 0
best_err_epoch_val = 0
best_err_epoch_val_data = 0
all_epoch_results = []
grids = []
num_batch = 0
dataset_pointer_ins_grid = -1
[grids.append([]) for dataset in range(dataloader.get_len_of_dataset())]
# Training
for epoch in range(args.num_epochs):
print('****************Training epoch beginning******************')
if dataloader.additional_validation and (epoch -
1) in validation_epoch_list:
dataloader.switch_to_dataset_type(True)
dataloader.reset_batch_pointer(valid=False)
loss_epoch = 0
# For each batch
for batch in range(dataloader.num_batches):
start = time.time()
# Get batch data
x, y, d, numPedsList, PedsList, target_ids = dataloader.next_batch(
)
loss_batch = 0
# 如果我们在新数据集中，则将批处理计数器清零
if dataset_pointer_ins_grid is not dataloader.dataset_pointer and epoch is not 0:
num_batch = 0
dataset_pointer_ins_grid = dataloader.dataset_pointer
# For each sequence
for sequence in range(dataloader.batch_size):
# 获取与当前序列相对应的数据
x_seq, _, d_seq, numPedsList_seq, PedsList_seq = x[
sequence], y[sequence], d[sequence], numPedsList[
sequence], PedsList[sequence]
target_id = target_ids[sequence]
# 获取处理文件名，然后获取文件尺寸
folder_name = dataloader.get_directory_name_with_pointer(d_seq)
dataset_data = dataloader.get_dataset_dimension(folder_name)
# dense vector creation
# 密集矢量创建
x_seq, lookup_seq = dataloader.convert_proper_array(
x_seq, numPedsList_seq, PedsList_seq)
target_id_values = x_seq[0][lookup_seq[target_id], 0:2]
# grid mask calculation and storage depending on grid parameter
# 网格掩码的计算和存储取决于网格参数
# 应该是用于判断是否有social性
if (args.grid):
if (epoch is 0):
grid_seq = getSequenceGridMask(x_seq, dataset_data,
PedsList_seq,
args.neighborhood_size,
args.grid_size,
args.use_cuda)
grids[dataloader.dataset_pointer].append(grid_seq)
else:
temp = (num_batch * dataloader.batch_size) + sequence
if temp > 128:
temp = 128
grid_seq = grids[dataloader.dataset_pointer][temp]
else:
grid_seq = getSequenceGridMask(x_seq, dataset_data,
PedsList_seq,
args.neighborhood_size,
args.grid_size,
args.use_cuda)
# 按顺序矢量化轨迹
x_seq, _ = vectorize_seq(x_seq, PedsList_seq, lookup_seq)
if args.use_cuda:
x_seq = x_seq.cuda()
# number of peds in this sequence per frame
# 每帧此序列中的ped数
numNodes = len(lookup_seq)
hidden_states = Variable(torch.zeros(numNodes, args.rnn_size))
if args.use_cuda:
hidden_states = hidden_states.cuda()
cell_states = Variable(torch.zeros(numNodes, args.rnn_size))
if args.use_cuda:
cell_states = cell_states.cuda()
# 零梯度
net.zero_grad()
optimizer.zero_grad()
# Forward prop
outputs, _, _ = net(x_seq, grid_seq, hidden_states,
cell_states, PedsList_seq, numPedsList_seq,
dataloader, lookup_seq)
# 计算损失loss
loss = Gaussian2DLikelihood(outputs, x_seq, PedsList_seq,
lookup_seq)
loss_batch += loss.item()
# 计算梯度
loss.backward()
# 裁剪梯度
torch.nn.utils.clip_grad_norm_(net.parameters(),
args.grad_clip)
# 更新梯度
optimizer.step()
end = time.time()
loss_batch = loss_batch / dataloader.batch_size
loss_epoch += loss_batch
num_batch += 1
print('{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}'.
format(epoch * dataloader.num_batches + batch,
args.num_epochs * dataloader.num_batches, epoch,
loss_batch, end - start))
loss_epoch /= dataloader.num_batches
# 记录loss
log_file_curve.write("Training epoch: " + str(epoch) + " loss: " +
str(loss_epoch) + '\n')
if dataloader.valid_num_batches > 0:
print(
'****************Validation epoch beginning******************')
# Validation
dataloader.reset_batch_pointer(valid=True)
loss_epoch = 0
err_epoch = 0
# 每一个batch
for batch in range(dataloader.valid_num_batches):
# 获取batch数据
x, y, d, numPedsList, PedsList, target_ids = dataloader.next_valid_batch(
)
# batch的损失loss
loss_batch = 0
err_batch = 0
# 对于每个序列
for sequence in range(dataloader.batch_size):
# 获取与当前序列相对应的数据
x_seq, _, d_seq, numPedsList_seq, PedsList_seq = x[
sequence], y[sequence], d[sequence], numPedsList[
sequence], PedsList[sequence]
target_id = target_ids[sequence]
# 获取处理文件名，然后获取文件尺寸
folder_name = dataloader.get_directory_name_with_pointer(
d_seq)
dataset_data = dataloader.get_dataset_dimension(
folder_name)
# 密集矢量创建
x_seq, lookup_seq = dataloader.convert_proper_array(
x_seq, numPedsList_seq, PedsList_seq)
target_id_values = x_seq[0][lookup_seq[target_id], 0:2]
# get grid mask
# 应该是用于判断是否有social
grid_seq = getSequenceGridMask(x_seq, dataset_data,
PedsList_seq,
args.neighborhood_size,
args.grid_size,
args.use_cuda)
x_seq, first_values_dict = vectorize_seq(
x_seq, PedsList_seq, lookup_seq)
if args.use_cuda:
x_seq = x_seq.cuda()
# number of peds in this sequence per frame
numNodes = len(lookup_seq)
hidden_states = Variable(
torch.zeros(numNodes, args.rnn_size))
if args.use_cuda:
hidden_states = hidden_states.cuda()
cell_states = Variable(torch.zeros(numNodes,
args.rnn_size))
if args.use_cuda:
cell_states = cell_states.cuda()
# Forward prop
outputs, _, _ = net(x_seq[:-1], grid_seq[:-1],
hidden_states, cell_states,
PedsList_seq[:-1], numPedsList_seq,
dataloader, lookup_seq)
# 计算损失loss
loss = Gaussian2DLikelihood(outputs, x_seq[1:],
PedsList_seq[1:], lookup_seq)
# 提取二元高斯的均值mean，std标准差和corr相关性
mux, muy, sx, sy, corr = getCoef(outputs)
# 来自二元高斯的样本
next_x, next_y = sample_gaussian_2d(
mux.data, muy.data, sx.data, sy.data, corr.data,
PedsList_seq[-1], lookup_seq)
next_vals = torch.FloatTensor(1, numNodes, 2)
next_vals[:, :, 0] = next_x
next_vals[:, :, 1] = next_y
err = get_mean_error(next_vals, x_seq[-1].data[None, :, :],
[PedsList_seq[-1]],
[PedsList_seq[-1]], args.use_cuda,
lookup_seq)
loss_batch += loss.item()
err_batch += err
loss_batch = loss_batch / dataloader.batch_size
err_batch = err_batch / dataloader.batch_size
loss_epoch += loss_batch
err_epoch += err_batch
if dataloader.valid_num_batches != 0:
loss_epoch = loss_epoch / dataloader.valid_num_batches
err_epoch = err_epoch / dataloader.num_batches
# 到目前为止更新最佳验证损失loss
if loss_epoch < best_val_loss:
best_val_loss = loss_epoch
best_epoch_val = epoch
if err_epoch < smallest_err_val:
smallest_err_val = err_epoch
best_err_epoch_val = epoch
print('(epoch {}), valid_loss = {:.3f}, valid_err = {:.3f}'.
format(epoch, loss_epoch, err_epoch))
print('Best epoch', best_epoch_val, 'Best validation loss',
best_val_loss, 'Best error epoch', best_err_epoch_val,
'Best error', smallest_err_val)
log_file_curve.write("Validation epoch: " + str(epoch) +
" loss: " + str(loss_epoch) + " err: " +
str(err_epoch) + '\n')
# Validation验证数据集
if dataloader.additional_validation and (
epoch) in validation_epoch_list:
dataloader.switch_to_dataset_type()
print(
'****************Validation with dataset epoch beginning******************'
)
dataloader.reset_batch_pointer(valid=False)
dataset_pointer_ins = dataloader.dataset_pointer
validation_dataset_executed = True
loss_epoch = 0
err_epoch = 0
f_err_epoch = 0
num_of_batch = 0
smallest_err = 100000
# results of one epoch for all validation datasets
# 所有验证数据集的一个时期的结果
epoch_result = []
# results of one validation dataset
# 一个验证数据集的结果
results = []
# For each batch
for batch in range(dataloader.num_batches):
# Get batch data
x, y, d, numPedsList, PedsList, target_ids = dataloader.next_batch(
)
if dataset_pointer_ins is not dataloader.dataset_pointer:
if dataloader.dataset_pointer is not 0:
print('Finished prosessed file : ',
dataloader.get_file_name(-1),
' Avarage error : ', err_epoch / num_of_batch)
num_of_batch = 0
epoch_result.append(results)
dataset_pointer_ins = dataloader.dataset_pointer
results = []
# Loss for this batch
loss_batch = 0
err_batch = 0
f_err_batch = 0
# For each sequence
for sequence in range(dataloader.batch_size):
# Get data corresponding to the current sequence
x_seq, _, d_seq, numPedsList_seq, PedsList_seq = x[
sequence], y[sequence], d[sequence], numPedsList[
sequence], PedsList[sequence]
target_id = target_ids[sequence]
# get processing file name and then get dimensions of file
folder_name = dataloader.get_directory_name_with_pointer(
d_seq)
dataset_data = dataloader.get_dataset_dimension(
folder_name)
# dense vector creation
x_seq, lookup_seq = dataloader.convert_proper_array(
x_seq, numPedsList_seq, PedsList_seq)
# will be used for error calculation
orig_x_seq = x_seq.clone()
target_id_values = orig_x_seq[0][lookup_seq[target_id],
0:2]
# grid mask calculation
grid_seq = getSequenceGridMask(x_seq, dataset_data,
PedsList_seq,
args.neighborhood_size,
args.grid_size,
args.use_cuda)
if args.use_cuda:
x_seq = x_seq.cuda()
orig_x_seq = orig_x_seq.cuda()
# 向量化数据点
x_seq, first_values_dict = vectorize_seq(
x_seq, PedsList_seq, lookup_seq)
# 从模型中抽取预测点
ret_x_seq, loss = sample_validation_data(
x_seq, PedsList_seq, grid_seq, args, net, lookup_seq,
numPedsList_seq, dataloader)
# 将点还原回原始空间
ret_x_seq = revert_seq(ret_x_seq, PedsList_seq, lookup_seq,
first_values_dict)
# get mean and final error
err = get_mean_error(ret_x_seq.data, orig_x_seq.data,
PedsList_seq, PedsList_seq,
args.use_cuda, lookup_seq)
f_err = get_final_error(ret_x_seq.data, orig_x_seq.data,
PedsList_seq, PedsList_seq,
lookup_seq)
loss_batch += loss.item()
err_batch += err
f_err_batch += f_err
print('Current file : ', dataloader.get_file_name(0),
' Batch : ', batch + 1, ' Sequence: ', sequence + 1,
' Sequence mean error: ', err,
' Sequence final error: ', f_err, ' time: ',
end - start)
results.append(
(orig_x_seq.data.cpu().numpy(),
ret_x_seq.data.cpu().numpy(), PedsList_seq,
lookup_seq,
dataloader.get_frame_sequence(args.seq_length),
target_id))
loss_batch = loss_batch / dataloader.batch_size
err_batch = err_batch / dataloader.batch_size
f_err_batch = f_err_batch / dataloader.batch_size
num_of_batch += 1
loss_epoch += loss_batch
err_epoch += err_batch
f_err_epoch += f_err_batch
epoch_result.append(results)
all_epoch_results.append(epoch_result)
if dataloader.num_batches != 0:
loss_epoch = loss_epoch / dataloader.num_batches
err_epoch = err_epoch / dataloader.num_batches
f_err_epoch = f_err_epoch / dataloader.num_batches
avarage_err = (err_epoch + f_err_epoch) / 2
# Update best validation loss until now
if loss_epoch < best_val_data_loss:
best_val_data_loss = loss_epoch
best_epoch_val_data = epoch
if avarage_err < smallest_err_val_data:
smallest_err_val_data = avarage_err
best_err_epoch_val_data = epoch
print('(epoch {}), valid_loss = {:.3f}, '
'valid_mean_err = {:.3f}, '
'valid_final_err = {:.3f}'.format(
epoch, loss_epoch, err_epoch, f_err_epoch))
print('Best epoch', best_epoch_val_data,
'Best validation loss', best_val_data_loss,
'Best error epoch', best_err_epoch_val_data,
'Best error', smallest_err_val_data)
log_file_curve.write("Validation dataset epoch: " +
str(epoch) + " loss: " + str(loss_epoch) +
" mean_err: " + str(err_epoch) +
'final_err: ' + str(f_err_epoch) + '\n')
optimizer = time_lr_scheduler(optimizer,
epoch,
lr_decay_epoch=args.freq_optimizer)
# Save the model after each epoch
print('Saving model')
torch.save(
{
'epoch': epoch,
'state_dict': net.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, checkpoint_path(epoch))
if dataloader.valid_num_batches != 0:
print('Best epoch', best_epoch_val, 'Best validation Loss',
best_val_loss, 'Best error epoch', best_err_epoch_val,
'Best error', smallest_err_val)
# Log the best epoch and best validation loss
log_file.write('Validation Best epoch:' + str(best_epoch_val) + ',' +
' Best validation Loss: ' + str(best_val_loss))
if dataloader.additional_validation:
print('Best epoch acording to validation dataset', best_epoch_val_data,
'Best validation Loss', best_val_data_loss, 'Best error epoch',
best_err_epoch_val_data, 'Best error', smallest_err_val_data)
log_file.write("Validation dataset Best epoch: " +
str(best_epoch_val_data) + ',' +
' Best validation Loss: ' + str(best_val_data_loss) +
'\n')
# FileNotFoundError: [Errno 2] No such file or directory: 'plot/SOCIALLSTM\\LSTM\\validation\\biwi\\biwi_hotel_4.pkl'
# validation_dataset_executed = True
if validation_dataset_executed:
# print("用于绘图的文件开始保存了")
dataloader.switch_to_dataset_type(load_data=False)
create_directories(plot_directory, [plot_train_file_directory])
# 找不到这个文件，是我手动添加的
# print(all_epoch_results)
# print(len(all_epoch_results) - 1)
dataloader.write_to_plot_file(
all_epoch_results[len(all_epoch_results) - 1],
os.path.join(plot_directory, plot_train_file_directory))
# Close logging files
log_file.close()
log_file_curve.close()
def main():
# Set random seed
np.random.seed(1)
parser = argparse.ArgumentParser()
# Observed length of the trajectory parameter
parser.add_argument('--obs_length',
type=int,
default=6,
help='Observed length of the trajectory')
# Predicted length of the trajectory parameter
parser.add_argument('--pred_length',
type=int,
default=6,
help='Predicted length of the trajectory')
# Test dataset
parser.add_argument('--test_dataset',
type=int,
default=3,
help='Dataset to be tested on')
# Model to be loaded
parser.add_argument('--epoch',
type=int,
default=0,
help='Epoch of model to be loaded')
# Parse the parameters
# sample_args = parser.parse_args()
args = parser.parse_args()
# Save directory
save_directory = 'save/' + str(args.test_dataset) + '/'
# Define the path for the config file for saved args
with open(os.path.join(save_directory, 'social_config.pkl'), 'rb') as f:
saved_args = pickle.load(f)
# Create a SocialModel object with the saved_args and infer set to true
model = SocialModel(saved_args, True)
# Initialize a TensorFlow session
sess = tf.InteractiveSession()
# Initialize a saver
saver = tf.train.Saver()
# Get the checkpoint state for the model
ckpt = tf.train.get_checkpoint_state(save_directory)
# print ('loading model: ', ckpt.model_checkpoint_path)
# print('hahah: ', len(ckpt.all_model_checkpoint_paths))
print('loading model: ', ckpt.all_model_checkpoint_paths[args.epoch])
# Restore the model at the checkpoint
saver.restore(sess, ckpt.all_model_checkpoint_paths[args.epoch])
# Dataset to get data from
dataset = [0]
# Create a SocialDataLoader object with batch_size 1 and seq_length equal to observed_length + pred_length
data_loader = SocialDataLoader(1,
args.pred_length + args.obs_length,
saved_args.maxNumPeds,
dataset,
True,
infer=True)
# Reset all pointers of the data_loader
data_loader.reset_batch_pointer()
results = []
# Variable to maintain total error
total_error = 0
# For each batch
for b in range(data_loader.num_batches):
# Get the source, target and dataset data for the next batch
x, y, d = data_loader.next_batch(randomUpdate=False)
# Batch size is 1
x_batch, y_batch, d_batch = x[0], y[0], d[0]
# if d_batch == 0 and dataset[0] == 0:
# dimensions = [640, 480]
# else:
# dimensions = [720, 576]
dimensions = [1640, 78]
grid_batch = getSequenceGridMask(x_batch, dimensions,
saved_args.neighborhood_size,
saved_args.grid_size)
obs_traj = x_batch[:args.obs_length]
obs_grid = grid_batch[:args.obs_length]
# obs_traj is an array of shape obs_length x maxNumPeds x 3
print "********************** SAMPLING A NEW TRAJECTORY", b, "******************************"
complete_traj = model.sample(sess, obs_traj, obs_grid, dimensions,
x_batch, args.pred_length)
# ipdb.set_trace()
# complete_traj is an array of shape (obs_length+pred_length) x maxNumPeds x 3
print('hahah', len(complete_traj))
total_error += get_mean_error(complete_traj, x[0], args.obs_length,
saved_args.maxNumPeds)
print "Processed trajectory number : ", b, "out of ", data_loader.num_batches, " trajectories"
# plot_trajectories(x[0], complete_traj, sample_args.obs_length)
# return
results.append((x[0], complete_traj, args.obs_length))
# Print the mean error across all the batches
print "Total mean error of the model is ", total_error / data_loader.num_batches
print "Saving results"
with open(os.path.join(save_directory, 'social_results.pkl'), 'wb') as f:
pickle.dump(results, f)
def main():
parser = argparse.ArgumentParser()
# Observed length of the trajectory parameter
parser.add_argument('--obs_length', type=int, default=8,
help='Observed length of the trajectory')
# Predicted length of the trajectory parameter
parser.add_argument('--pred_length', type=int, default=12,
help='Predicted length of the trajectory')
# Test dataset
parser.add_argument('--test_dataset', type=int, default=3,
help='Dataset to be tested on')
# Model to be loaded
parser.add_argument('--epoch', type=int, default=107,
help='Epoch of model to be loaded')
# Parse the parameters
sample_args = parser.parse_args()
# Save directory
save_directory = '/home/hesl/PycharmProjects/social-lstm-pytorch/save/FixedPixel_Normalized_150epoch/'+ str(sample_args.test_dataset) + '/'
save_directory='/home/hesl/PycharmProjects/social-lstm-pytorch/save/FixedPixel_Normalized_150epoch/1/'
ouput_directory='/home/hesl/PycharmProjects/social-lstm-pytorch/save/'
# Define the path for the config file for saved args
with open(os.path.join(save_directory, 'config.pkl'), 'rb') as f:
saved_args = pickle.load(f)
# Initialize net
net = SocialLSTM(saved_args, True)
net.cuda()
# Get the checkpoint path
checkpoint_path = os.path.join(save_directory, 'social_lstm_model_'+str(sample_args.epoch)+'.tar')
# checkpoint_path = os.path.join(save_directory, 'srnn_model.tar')
if os.path.isfile(checkpoint_path):
print('Loading checkpoint')
checkpoint = torch.load(checkpoint_path)
# model_iteration = checkpoint['iteration']
model_epoch = checkpoint['epoch']
net.load_state_dict(checkpoint['state_dict'])
print('Loaded checkpoint at epoch', model_epoch)
#homography
H = np.loadtxt(H_path[sample_args.test_dataset])
# Test dataset
dataset = [sample_args.test_dataset]
# Create the DataLoader object
dataloader = DataLoader(1, sample_args.pred_length + sample_args.obs_length, dataset, True, infer=True)
dataloader.reset_batch_pointer()
# Construct the ST-graph object
stgraph = ST_GRAPH(1, sample_args.pred_length + sample_args.obs_length)
results = []
# Variable to maintain total error
total_error = 0
final_error = 0
# For each batch
for batch in range(dataloader.num_batches):
start = time.time()
# Get data
x, _, d = dataloader.next_batch(randomUpdate=False)
# Get the sequence
x_seq, d_seq = x[0], d[0]
# Dimensions of the dataset
if d_seq == 0 and dataset[0] == 0:
dimensions = [640, 480]
else:
dimensions = [720, 576]
dimensions=[1224,370]
# Get the grid masks for the sequence
grid_seq = getSequenceGridMask(x_seq, dimensions, saved_args.neighborhood_size, saved_args.grid_size)
# Construct ST graph
stgraph.readGraph(x)
# Get nodes and nodesPresent
nodes, _, nodesPresent, _ = stgraph.getSequence(0)
nodes = Variable(torch.from_numpy(nodes).float(), volatile=True).cuda()
# Extract the observed part of the trajectories
obs_nodes, obs_nodesPresent, obs_grid = nodes[:sample_args.obs_length], nodesPresent[:sample_args.obs_length], grid_seq[:sample_args.obs_length]
# The sample function
ret_nodes = sample(obs_nodes, obs_nodesPresent, obs_grid, sample_args, net, nodes, nodesPresent, grid_seq, saved_args, dimensions)
#print(nodes[sample_args.obs_length:].data)
# Record the mean and final displacement error
total_error += get_mean_error(ret_nodes[sample_args.obs_length:].data, nodes[sample_args.obs_length:].data, nodesPresent[sample_args.obs_length-1], nodesPresent[sample_args.obs_length:],H,sample_args.test_dataset)
final_error += get_final_error(ret_nodes[sample_args.obs_length:].data, nodes[sample_args.obs_length:].data, nodesPresent[sample_args.obs_length-1], nodesPresent[sample_args.obs_length:],H,sample_args.test_dataset)
end = time.time()
print('Processed trajectory number : ', batch, 'out of', dataloader.num_batches, 'trajectories in time', end - start)
results.append((nodes.data.cpu().numpy(), ret_nodes.data.cpu().numpy(), nodesPresent, sample_args.obs_length))
# Reset the ST graph
stgraph.reset()
print('Total mean error of the model is ', total_error / dataloader.num_batches)
print('Total final error of the model is ', final_error / dataloader.num_batches)
print('Saving results')
with open(os.path.join(ouput_directory, 'results.pkl'), 'wb') as f:
pickle.dump(results, f)
def train(args):
origin = (0, 0)
reference_point = (0, 1)
validation_dataset_executed = False
prefix = ''
f_prefix = '.'
if args.drive is True:
prefix = 'drive/semester_project/social_lstm_final/'
f_prefix = 'drive/semester_project/social_lstm_final'
if not os.path.isdir("log/"):
print("Directory creation script is running...")
subprocess.call([f_prefix + '/make_directories.sh'])
args.freq_validation = np.clip(args.freq_validation, 0, args.num_epochs)
validation_epoch_list = list(
range(args.freq_validation, args.num_epochs + 1, args.freq_validation))
validation_epoch_list[-1] -= 1
# Create the data loader object. This object would preprocess the data in terms of
# batches each of size args.batch_size, of length args.seq_length
dataloader = DataLoader(f_prefix,
args.batch_size,
args.seq_length,
args.num_validation,
forcePreProcess=True)
method_name = "OBSTACLELSTM"
model_name = "LSTM"
save_tar_name = method_name + "_lstm_model_"
if args.gru:
model_name = "GRU"
save_tar_name = method_name + "_gru_model_"
# Log directory
log_directory = os.path.join(prefix, 'log/')
plot_directory = os.path.join(prefix, 'plot/', method_name, model_name)
plot_train_file_directory = 'validation'
# Logging files
log_file_curve = open(
os.path.join(log_directory, method_name, model_name, 'log_curve.txt'),
'w+')
log_file = open(
os.path.join(log_directory, method_name, model_name, 'val.txt'), 'w+')
# model directory
save_directory = os.path.join(prefix, 'model/')
# save_directory += str(args.leaveDataset) + '/'
# Save the arguments in the config file
with open(
os.path.join(save_directory, method_name, model_name,
'config.pkl'), 'wb') as f:
pickle.dump(args, f)
# Path to store the checkpoint file
def checkpoint_path(x):
return os.path.join(save_directory, method_name, model_name,
save_tar_name + str(x) + '.tar')
# model creation
net = OLSTMModel(args)
if args.use_cuda:
net = net.cuda()
# optimizer = torch.optim.RMSprop(net.parameters(), lr=args.learning_rate)
optimizer = torch.optim.Adagrad(net.parameters(),
weight_decay=args.lambda_param)
# optimizer = torch.optim.Adam(net.parameters(), weight_decay=args.lambda_param)
learning_rate = args.learning_rate
best_val_loss = 100
best_val_data_loss = 100
smallest_err_val = 100000
smallest_err_val_data = 100000
best_epoch_val = 0
best_epoch_val_data = 0
best_err_epoch_val = 0
best_err_epoch_val_data = 0
all_epoch_results = []
grids = []
num_batch = 0
dataset_pointer_ins_grid = -1
[grids.append([]) for dataset in range(dataloader.get_len_of_dataset())]
# Training
for epoch in range(args.num_epochs):
print('****************Training epoch beginning******************')
if dataloader.additional_validation and (epoch -
1) in validation_epoch_list:
dataloader.switch_to_dataset_type(True)
dataloader.reset_batch_pointer(valid=False)
loss_epoch = 0
# For each batch
for batch in range(dataloader.num_batches):
start = time.time()
# Get batch data
x, y, d, numPedsList, PedsList, target_ids = dataloader.next_batch(
)
loss_batch = 0
# if we are in a new dataset, zero the counter of batch
if dataset_pointer_ins_grid is not dataloader.dataset_pointer and epoch is not 0:
num_batch = 0
dataset_pointer_ins_grid = dataloader.dataset_pointer
# For each sequence
for sequence in range(dataloader.batch_size):
# Get the data corresponding to the current sequence
x_seq, _, d_seq, numPedsList_seq, PedsList_seq = x[
sequence], y[sequence], d[sequence], numPedsList[
sequence], PedsList[sequence]
target_id = target_ids[sequence]
# get processing file name and then get dimensions of file
folder_name = dataloader.get_directory_name_with_pointer(d_seq)
dataset_data = dataloader.get_dataset_dimension(
folder_name
) #dataloader.get_dataset_dimension(folder_name)
# dense vector creation
x_seq, lookup_seq = dataloader.convert_proper_array(
x_seq, numPedsList_seq, PedsList_seq)
# print("LOOKUP SEQ = " ,lookup_seq)
#print("TARGET_ID = " , target_id)
target_id_values = x_seq[0][lookup_seq[target_id], 0:2] #:2
# Compute grid masks
if (args.grid):
if (epoch is 0):
grid_seq = getSequenceGridMask(x_seq, dataset_data,
PedsList_seq,
args.neighborhood_size,
args.grid_size,
args.use_cuda, True)
grids[dataloader.dataset_pointer].append(grid_seq)
else:
grid_seq = grids[dataloader.dataset_pointer][
(num_batch * dataloader.batch_size) + sequence]
else:
grid_seq = getSequenceGridMask(x_seq, dataset_data,
PedsList_seq,
args.neighborhood_size,
args.grid_size,
args.use_cuda, True)
# vectorize trajectories in sequence
x_seq, _ = vectorize_seq(x_seq, PedsList_seq, lookup_seq)
if args.use_cuda:
x_seq = x_seq.cuda()
# number of peds in this sequence per frame
numNodes = len(lookup_seq)
hidden_states = Variable(torch.zeros(numNodes, args.rnn_size))
if args.use_cuda:
hidden_states = hidden_states.cuda()
cell_states = Variable(torch.zeros(numNodes, args.rnn_size))
if args.use_cuda:
cell_states = cell_states.cuda()
# Zero out gradients
net.zero_grad()
optimizer.zero_grad()
# Forward prop
outputs, _, _ = net(x_seq, grid_seq, hidden_states,
cell_states, PedsList_seq, numPedsList_seq,
dataloader, lookup_seq)
# Compute loss
loss = Gaussian2DLikelihood(outputs, x_seq, PedsList_seq,
lookup_seq)
loss_batch += loss.item()
# Compute gradients
loss.backward()
# Clip gradients
torch.nn.utils.clip_grad_norm_(net.parameters(),
args.grad_clip)
# Update parameters
optimizer.step()
end = time.time()
loss_batch = loss_batch / dataloader.batch_size
loss_epoch += loss_batch
num_batch += 1
print('{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}'.
format(epoch * dataloader.num_batches + batch,
args.num_epochs * dataloader.num_batches, epoch,
loss_batch, end - start))
loss_epoch /= dataloader.num_batches
# Log loss values
log_file_curve.write("Training epoch: " + str(epoch) + " loss: " +
str(loss_epoch) + '\n')
if dataloader.valid_num_batches > 0:
print(
'****************Validation epoch beginning******************')
# Validation
dataloader.reset_batch_pointer(valid=True)
loss_epoch = 0
err_epoch = 0
# For each batch
for batch in range(dataloader.valid_num_batches):
# Get batch data
x, y, d, numPedsList, PedsList, target_ids = dataloader.next_valid_batch(
)
# Loss for this batch
loss_batch = 0
err_batch = 0
# For each sequence
for sequence in range(dataloader.batch_size):
# Get data corresponding to the current sequence
x_seq, _, d_seq, numPedsList_seq, PedsList_seq = x[
sequence], y[sequence], d[sequence], numPedsList[
sequence], PedsList[sequence]
target_id = target_ids[sequence]
# get processing file name and then get dimensions of file
folder_name = dataloader.get_directory_name_with_pointer(
d_seq)
dataset_data = dataloader.get_dataset_dimension(
folder_name)
# dense vector creation
x_seq, lookup_seq = dataloader.convert_proper_array(
x_seq, numPedsList_seq, PedsList_seq)
target_id_values = x_seq[0][lookup_seq[target_id], 0:2]
# Compute grid masks
if (args.grid):
if (epoch is 0):
grid_seq = getSequenceGridMask(
x_seq, dataset_data, PedsList_seq,
args.neighborhood_size, args.grid_size,
args.use_cuda, True)
grids[dataloader.dataset_pointer].append(grid_seq)
else:
grid_seq = grids[dataloader.dataset_pointer][
(num_batch * dataloader.batch_size) + sequence]
else:
grid_seq = getSequenceGridMask(x_seq, dataset_data,
PedsList_seq,
args.neighborhood_size,
args.grid_size,
args.use_cuda, True)
# vectorize trajectories in sequence
x_seq, _ = vectorize_seq(x_seq, PedsList_seq, lookup_seq)
if args.use_cuda:
x_seq = x_seq.cuda()
# number of peds in this sequence per frame
numNodes = len(lookup_seq)
hidden_states = Variable(
torch.zeros(numNodes, args.rnn_size))
if args.use_cuda:
hidden_states = hidden_states.cuda()
cell_states = Variable(torch.zeros(numNodes,
args.rnn_size))
if args.use_cuda:
cell_states = cell_states.cuda()
# Forward prop
outputs, _, _ = net(x_seq[:-1], grid_seq[:-1],
hidden_states, cell_states,
PedsList_seq[:-1], numPedsList_seq,
dataloader, lookup_seq)
# Compute loss
loss = Gaussian2DLikelihood(outputs, x_seq[1:],
PedsList_seq[1:], lookup_seq)
# Extract the mean, std and corr of the bivariate Gaussian
mux, muy, sx, sy, corr = getCoef(outputs)
# Sample from the bivariate Gaussian
next_x, next_y = sample_gaussian_2d(
mux.data, muy.data, sx.data, sy.data, corr.data,
PedsList_seq[-1], lookup_seq)
next_vals = torch.FloatTensor(1, numNodes, 2)
next_vals[:, :, 0] = next_x
next_vals[:, :, 1] = next_y
err = get_mean_error(next_vals, x_seq[-1].data[None, :, :],
[PedsList_seq[-1]],
[PedsList_seq[-1]], args.use_cuda,
lookup_seq)
loss_batch += loss.item()
err_batch += err
loss_batch = loss_batch / dataloader.batch_size
err_batch = err_batch / dataloader.batch_size
loss_epoch += loss_batch
err_epoch += err_batch
if dataloader.valid_num_batches != 0:
loss_epoch = loss_epoch / dataloader.valid_num_batches
err_epoch = err_epoch / dataloader.num_batches
# Update best validation loss until now
if loss_epoch < best_val_loss:
best_val_loss = loss_epoch
best_epoch_val = epoch
if err_epoch < smallest_err_val:
smallest_err_val = err_epoch
best_err_epoch_val = epoch
print('(epoch {}), valid_loss = {:.3f}, valid_err = {:.3f}'.
format(epoch, loss_epoch, err_epoch))
print('Best epoch', best_epoch_val, 'Best validation loss',
best_val_loss, 'Best error epoch', best_err_epoch_val,
'Best error', smallest_err_val)
log_file_curve.write("Validation epoch: " + str(epoch) +
" loss: " + str(loss_epoch) + " err: " +
str(err_epoch) + '\n')
# Validation dataset
if dataloader.additional_validation and (
epoch) in validation_epoch_list:
dataloader.switch_to_dataset_type()
print(
'****************Validation with dataset epoch beginning******************'
)
dataloader.reset_batch_pointer(valid=False)
dataset_pointer_ins = dataloader.dataset_pointer
validation_dataset_executed = True
loss_epoch = 0
err_epoch = 0
f_err_epoch = 0
num_of_batch = 0
smallest_err = 100000
# results of one epoch for all validation datasets
epoch_result = []
# results of one validation dataset
results = []
# For each batch
for batch in range(dataloader.num_batches):
# Get batch data
x, y, d, numPedsList, PedsList, target_ids = dataloader.next_batch(
)
if dataset_pointer_ins is not dataloader.dataset_pointer:
if dataloader.dataset_pointer is not 0:
print('Finished prosessed file : ',
dataloader.get_file_name(-1),
' Avarage error : ', err_epoch / num_of_batch)
num_of_batch = 0
epoch_result.append(results)
dataset_pointer_ins = dataloader.dataset_pointer
results = []
# Loss for this batch
loss_batch = 0
err_batch = 0
f_err_batch = 0
# For each sequence
for sequence in range(dataloader.batch_size):
# Get data corresponding to the current sequence
x_seq, _, d_seq, numPedsList_seq, PedsList_seq = x[
sequence], y[sequence], d[sequence], numPedsList[
sequence], PedsList[sequence]
target_id = target_ids[sequence]
# get processing file name and then get dimensions of file
folder_name = dataloader.get_directory_name_with_pointer(
d_seq)
dataset_data = dataloader.get_dataset_dimension(
folder_name)
# dense vector creation
x_seq, lookup_seq = dataloader.convert_proper_array(
x_seq, numPedsList_seq, PedsList_seq)
# will be used for error calculation
orig_x_seq = x_seq.clone()
target_id_values = orig_x_seq[0][lookup_seq[target_id],
0:2]
# grid mask calculation
grid_seq = getSequenceGridMask(x_seq, dataset_data,
PedsList_seq,
args.neighborhood_size,
args.grid_size,
args.use_cuda, True)
# vectorize datapoints
x_seq, first_values_dict = vectorize_seq(
x_seq, PedsList_seq, lookup_seq)
if args.use_cuda:
x_seq = x_seq.cuda()
# sample predicted points from model
ret_x_seq, loss = sample_validation_data(
x_seq, PedsList_seq, grid_seq, args, net, lookup_seq,
numPedsList_seq, dataloader)
# revert the points back to original space
ret_x_seq = revert_seq(ret_x_seq, PedsList_seq, lookup_seq,
first_values_dict)
# get mean and final error
err = get_mean_error(ret_x_seq.data, orig_x_seq.data,
PedsList_seq, PedsList_seq,
args.use_cuda, lookup_seq)
f_err = get_final_error(ret_x_seq.data, orig_x_seq.data,
PedsList_seq, PedsList_seq,
lookup_seq)
loss_batch += loss.item()
err_batch += err
f_err_batch += f_err
print('Current file : ', dataloader.get_file_name(0),
' Batch : ', batch + 1, ' Sequence: ', sequence + 1,
' Sequence mean error: ', err,
' Sequence final error: ', f_err, ' time: ',
end - start)
results.append(
(orig_x_seq.data.cpu().numpy(),
ret_x_seq.data.cpu().numpy(), PedsList_seq,
lookup_seq,
dataloader.get_frame_sequence(args.seq_length),
target_id))
loss_batch = loss_batch / dataloader.batch_size
err_batch = err_batch / dataloader.batch_size
f_err_batch = f_err_batch / dataloader.batch_size
num_of_batch += 1
loss_epoch += loss_batch
err_epoch += err_batch
f_err_epoch += f_err_batch
epoch_result.append(results)
all_epoch_results.append(epoch_result)
if dataloader.num_batches != 0:
loss_epoch = loss_epoch / dataloader.num_batches
err_epoch = err_epoch / dataloader.num_batches
f_err_epoch = f_err_epoch / dataloader.num_batches
# Update best validation loss until now
if loss_epoch < best_val_data_loss:
best_val_data_loss = loss_epoch
best_epoch_val_data = epoch
if err_epoch < smallest_err_val_data:
smallest_err_val_data = err_epoch
best_err_epoch_val_data = epoch
print(
'(epoch {}), valid_loss = {:.3f}, valid_mean_err = {:.3f}, valid_final_err = {:.3f}'
.format(epoch, loss_epoch, err_epoch, f_err_epoch))
print('Best epoch', best_epoch_val_data,
'Best validation loss', best_val_data_loss,
'Best error epoch', best_err_epoch_val_data,
'Best error', smallest_err_val_data)
log_file_curve.write("Validation dataset epoch: " +
str(epoch) + " loss: " + str(loss_epoch) +
" mean_err: " + str(err_epoch) +
'final_err: ' + str(f_err_epoch) + '\n')
optimizer = time_lr_scheduler(optimizer,
epoch,
lr_decay_epoch=args.freq_optimizer)
# Save the model after each epoch
print('Saving model')
torch.save(
{
'epoch': epoch,
'state_dict': net.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, checkpoint_path(epoch))
if dataloader.valid_num_batches != 0:
print('Best epoch', best_epoch_val, 'Best validation Loss',
best_val_loss, 'Best error epoch', best_err_epoch_val,
'Best error', smallest_err_val)
# Log the best epoch and best validation loss
log_file.write('Validation Best epoch:' + str(best_epoch_val) + ',' +
' Best validation Loss: ' + str(best_val_loss))
if dataloader.additional_validation:
print('Best epoch acording to validation dataset', best_epoch_val_data,
'Best validation Loss', best_val_data_loss, 'Best error epoch',
best_err_epoch_val_data, 'Best error', smallest_err_val_data)
log_file.write("Validation dataset Best epoch: " +
str(best_epoch_val_data) + ',' +
' Best validation Loss: ' + str(best_val_data_loss) +
'\n')
# dataloader.write_to_plot_file(all_epoch_results[best_epoch_val_data], plot_directory)
# elif dataloader.valid_num_batches != 0:
# dataloader.write_to_plot_file(all_epoch_results[best_epoch_val], plot_directory)
# else:
if validation_dataset_executed: # if we executed a validation epoch, write last epoch file
dataloader.switch_to_dataset_type(load_data=False)
create_directories(plot_directory, [plot_train_file_directory])
dataloader.write_to_plot_file(
all_epoch_results[len(all_epoch_results) - 1],
os.path.join(plot_directory, plot_train_file_directory))
# Close logging files
log_file.close()
log_file_curve.close()
def train(args):
origin = (0,0)
reference_point = (0,1)
validation_dataset_executed = False
prefix = '' # prefix = ''
f_prefix = args.data_dir
# if args.drive is True:
# prefix='drive/semester_project/social_lstm_final/'
# f_prefix = 'drive/semester_project/social_lstm_final'
print('data_dir:', args.data_dir)
# if not os.path.isdir("log/"):
# print("Directory creation script is running...")
# subprocess.call(['make_directories.sh'])
args.freq_validation = np.clip(args.freq_validation, 0, args.num_epochs)
validation_epoch_list = list(range(args.freq_validation, args.num_epochs+1, args.freq_validation))
validation_epoch_list[-1]-=1
# Create the data loader object. This object would preprocess the data in terms of
# batches each of size args.batch_size, of length args.seq_length
dataloader = DataLoader(f_prefix, args.batch_size, args.seq_length, args.num_validation, forcePreProcess=True)
model_name = "LSTM"
method_name = "SOCIALLSTM"
save_tar_name = method_name+"_lstm_model_"
if args.gru:
model_name = "GRU"
save_tar_name = method_name+"_gru_model_"
# Log directory
log_directory = os.path.join(prefix, 'log/')
plot_directory = os.path.join(prefix, 'plot/', method_name, model_name)
plot_train_file_directory = 'validation'
# Logging files
log_file_curve = open(os.path.join(log_directory, method_name, model_name,'log_curve.txt'), 'w+')
log_file = open(os.path.join(log_directory, method_name, model_name, 'val.txt'), 'w+')
# model directory
save_directory = os.path.join(prefix, 'model/')
# Save the arguments int the config file
import json
with open(os.path.join(save_directory, method_name, model_name,'config.pkl'), 'wb') as f:
args_dict = vars(args)
pickle.dump(args, f)
# Path to store the checkpoint file
def checkpoint_path(x):
return os.path.join(save_directory, method_name, model_name, save_tar_name+str(x)+'.tar')
# model creation
net = SocialModel(args)
if args.use_cuda:
net = net.cuda()
#optimizer = torch.optim.RMSprop(net.parameters(), lr=args.learning_rate)
optimizer = torch.optim.Adagrad(net.parameters(), weight_decay=args.lambda_param)
#optimizer = torch.optim.Adam(net.parameters(), weight_decay=args.lambda_param)
learning_rate = args.learning_rate
best_val_loss = 100
best_val_data_loss = 100
smallest_err_val = 100000
smallest_err_val_data = 100000
best_epoch_val = 0
best_epoch_val_data = 0
best_err_epoch_val = 0
best_err_epoch_val_data = 0
all_epoch_results = []
grids = []
num_batch = 0
dataset_pointer_ins_grid = -1
[grids.append([]) for dataset in range(dataloader.get_len_of_dataset())]
# Training
for epoch in range(args.num_epochs):
print('****************Training epoch beginning******************')
if dataloader.additional_validation and (epoch-1) in validation_epoch_list:
dataloader.switch_to_dataset_type(True)
dataloader.reset_batch_pointer(valid=False)
loss_epoch = 0
# For each batch
for batch in range(dataloader.num_batches):
start = time.time()
# Get batch data
x, y, d , numPedsList, PedsList ,target_ids= dataloader.next_batch()
loss_batch = 0
#if we are in a new dataset, zero the counter of batch
if dataset_pointer_ins_grid is not dataloader.dataset_pointer and epoch is not 0:
num_batch = 0
dataset_pointer_ins_grid = dataloader.dataset_pointer
# For each sequence
for sequence in range(dataloader.batch_size):
# Get the data corresponding to the current sequence
x_seq ,_ , d_seq, numPedsList_seq, PedsList_seq = x[sequence], y[sequence], d[sequence], numPedsList[sequence], PedsList[sequence]
target_id = target_ids[sequence]
#get processing file name and then get dimensions of file
folder_name = dataloader.get_directory_name_with_pointer(d_seq)
dataset_data = dataloader.get_dataset_dimension(folder_name)
#dense vector creation
x_seq, lookup_seq = dataloader.convert_proper_array(x_seq, numPedsList_seq, PedsList_seq)
target_id_values = x_seq[0][lookup_seq[target_id], 0:2]
#grid mask calculation and storage depending on grid parameter
if(args.grid):
if(epoch is 0):
grid_seq = getSequenceGridMask(x_seq, dataset_data, PedsList_seq,args.neighborhood_size, args.grid_size, args.use_cuda)
grids[dataloader.dataset_pointer].append(grid_seq)
else:
grid_seq = grids[dataloader.dataset_pointer][(num_batch*dataloader.batch_size)+sequence]
else:
grid_seq = getSequenceGridMask(x_seq, dataset_data, PedsList_seq,args.neighborhood_size, args.grid_size, args.use_cuda)
# vectorize trajectories in sequence
if args.use_cuda:
x_seq = x_seq.cuda()
x_seq, _ = vectorize_seq(x_seq, PedsList_seq, lookup_seq)
# <---------------------- Experimental block ----------------------->
# Main approach:
# 1) Translate all trajectories using first frame value of target trajectory so that target trajectory will start (0,0).
# 2) Get angle between first trajectory point of target ped and (0, 1) for turning.
# 3) Rotate all trajectories in the sequence using this angle.
# 4) Calculate grid mask for hidden layer pooling.
# 5) Vectorize all trajectories (substract first frame values of each trajectories from subsequent points in the trajectory).
#
# Problem:
# Low accuracy
#
# Possible causes:
# *Each function has been already checked -> low possibility.
# *Logic errors or algorithm errors -> high possibility.
# *Wrong order of execution each step -> high possibility.
# <------------------------------------------------------------------------>
# x_seq = translate(x_seq, PedsList_seq, lookup_seq ,target_id_values)
# angle = angle_between(reference_point, (x_seq[1][lookup_seq[target_id], 0].data.numpy(), x_seq[1][lookup_seq[target_id], 1].data.numpy()))
# x_seq = rotate_traj_with_target_ped(x_seq, angle, PedsList_seq, lookup_seq)
# if(args.grid):
# if(epoch is 0):
# grid_seq = getSequenceGridMask(x_seq, dataset_data, PedsList_seq,args.neighborhood_size, args.grid_size, args.use_cuda)
# grids[dataloader.dataset_pointer].append(grid_seq)
# else:
# #grid_seq1 = getSequenceGridMask(x_seq, dataset_data, PedsList_seq,args.neighborhood_size, args.grid_size, args.use_cuda)
# grid_seq = grids[dataloader.dataset_pointer][(num_batch*dataloader.batch_size)+sequence]
# #print([ torch.equal(x.data, y.data) for (x,y) in zip(grid_seq1, grid_seq)])
# #if not (all([ torch.equal(x.data, y.data) for (x,y) in zip(grid_seq1, grid_seq)])):
# # print("not equal")
# # quit()
# else:
# grid_seq = getSequenceGridMask(x_seq, dataset_data, PedsList_seq,args.neighborhood_size, args.grid_size, args.use_cuda)
# x_seq, first_values_dict = vectorize_seq(x_seq, PedsList_seq, lookup_seq)
#print(grid_seq)
# Construct variables
#print("target id : ", target_id)
#print("look up : ", lookup_seq)
#print("pedlist_seq: ", PedsList_seq)
#print("before_xseq: ", x_seq)
#x_seq, target_id_values, first_values_dict = vectorize_seq_with_ped(x_seq, PedsList_seq, lookup_seq ,target_id)
#print("after_vectorize_seq: ", x_seq)
#print("angle: ", np.rad2deg(angle))
#print("after_xseq: ", x_seq)
#x_seq = rotate_traj_with_target_ped(x_seq, -angle, PedsList_seq, lookup_seq)
#x_seq = revert_seq(x_seq, PedsList_seq, lookup_seq, first_values_dict)
#number of peds in this sequence per frame
numNodes = len(lookup_seq)
hidden_states = Variable(torch.zeros(numNodes, args.rnn_size))
if args.use_cuda:
hidden_states = hidden_states.cuda()
cell_states = Variable(torch.zeros(numNodes, args.rnn_size))
if args.use_cuda:
cell_states = cell_states.cuda()
# Zero out gradients
net.zero_grad()
optimizer.zero_grad()
# Forward prop
outputs, _, _ = net(x_seq, grid_seq, hidden_states, cell_states, PedsList_seq,numPedsList_seq ,dataloader, lookup_seq)
# Compute loss
loss = Gaussian2DLikelihood(outputs, x_seq, PedsList_seq, lookup_seq)
loss_batch += loss.item()
# Compute gradients
loss.backward()
# Clip gradients
torch.nn.utils.clip_grad_norm_(net.parameters(), args.grad_clip)
# Update parameters
optimizer.step()
end = time.time()
loss_batch = loss_batch / dataloader.batch_size
loss_epoch += loss_batch
num_batch+=1
print('{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}'.format(epoch * dataloader.num_batches + batch,
args.num_epochs * dataloader.num_batches,
epoch,
loss_batch, end - start))
loss_epoch /= dataloader.num_batches
# Log loss values
log_file_curve.write("Training epoch: "+str(epoch)+" loss: "+str(loss_epoch)+'\n')
if dataloader.valid_num_batches > 0:
print('****************Validation epoch beginning******************')
# Validation
dataloader.reset_batch_pointer(valid=True)
loss_epoch = 0
err_epoch = 0
# For each batch
for batch in range(dataloader.valid_num_batches):
# Get batch data
x, y, d , numPedsList, PedsList ,target_ids= dataloader.next_valid_batch()
# Loss for this batch
loss_batch = 0
err_batch = 0
# For each sequence
for sequence in range(dataloader.batch_size):
# Get data corresponding to the current sequence
x_seq ,_ , d_seq, numPedsList_seq, PedsList_seq = x[sequence], y[sequence], d[sequence], numPedsList[sequence], PedsList[sequence]
target_id = target_ids[sequence]
#get processing file name and then get dimensions of file
folder_name = dataloader.get_directory_name_with_pointer(d_seq)
dataset_data = dataloader.get_dataset_dimension(folder_name)
#dense vector creation
x_seq, lookup_seq = dataloader.convert_proper_array(x_seq, numPedsList_seq, PedsList_seq)
target_id_values = x_seq[0][lookup_seq[target_id], 0:2]
#get grid mask
grid_seq = getSequenceGridMask(x_seq, dataset_data, PedsList_seq, args.neighborhood_size, args.grid_size, args.use_cuda)
if args.use_cuda:
x_seq = x_seq.cuda()
x_seq, first_values_dict = vectorize_seq(x_seq, PedsList_seq, lookup_seq)
# <---------------------- Experimental block ----------------------->
# x_seq = translate(x_seq, PedsList_seq, lookup_seq ,target_id_values)
# angle = angle_between(reference_point, (x_seq[1][lookup_seq[target_id], 0].data.numpy(), x_seq[1][lookup_seq[target_id], 1].data.numpy()))
# x_seq = rotate_traj_with_target_ped(x_seq, angle, PedsList_seq, lookup_seq)
# grid_seq = getSequenceGridMask(x_seq, dataset_data, PedsList_seq, args.neighborhood_size, args.grid_size, args.use_cuda)
# x_seq, first_values_dict = vectorize_seq(x_seq, PedsList_seq, lookup_seq)
#number of peds in this sequence per frame
numNodes = len(lookup_seq)
hidden_states = Variable(torch.zeros(numNodes, args.rnn_size))
if args.use_cuda:
hidden_states = hidden_states.cuda()
cell_states = Variable(torch.zeros(numNodes, args.rnn_size))
if args.use_cuda:
cell_states = cell_states.cuda()
# Forward prop
outputs, _, _ = net(x_seq[:-1], grid_seq[:-1], hidden_states, cell_states, PedsList_seq[:-1], numPedsList_seq , dataloader, lookup_seq)
# Compute loss
loss = Gaussian2DLikelihood(outputs, x_seq[1:], PedsList_seq[1:], lookup_seq)
# Extract the mean, std and corr of the bivariate Gaussian
mux, muy, sx, sy, corr = getCoef(outputs)
# Sample from the bivariate Gaussian
next_x, next_y = sample_gaussian_2d(mux.data, muy.data, sx.data, sy.data, corr.data, PedsList_seq[-1], lookup_seq)
next_vals = torch.FloatTensor(1,numNodes,2)
next_vals[:,:,0] = next_x
next_vals[:,:,1] = next_y
err = get_mean_error(next_vals, x_seq[-1].data[None, : ,:], [PedsList_seq[-1]], [PedsList_seq[-1]], args.use_cuda, lookup_seq)
loss_batch += loss.item()
err_batch += err
loss_batch = loss_batch / dataloader.batch_size
err_batch = err_batch / dataloader.batch_size
loss_epoch += loss_batch
err_epoch += err_batch
if dataloader.valid_num_batches != 0:
loss_epoch = loss_epoch / dataloader.valid_num_batches
err_epoch = err_epoch / dataloader.num_batches
# Update best validation loss until now
if loss_epoch < best_val_loss:
best_val_loss = loss_epoch
best_epoch_val = epoch
if err_epoch<smallest_err_val:
smallest_err_val = err_epoch
best_err_epoch_val = epoch
print('(epoch {}), valid_loss = {:.3f}, valid_err = {:.3f}'.format(epoch, loss_epoch, err_epoch))
print('Best epoch', best_epoch_val, 'Best validation loss', best_val_loss, 'Best error epoch',best_err_epoch_val, 'Best error', smallest_err_val)
log_file_curve.write("Validation epoch: "+str(epoch)+" loss: "+str(loss_epoch)+" err: "+str(err_epoch)+'\n')
# Validation dataset
if dataloader.additional_validation and (epoch) in validation_epoch_list:
dataloader.switch_to_dataset_type()
print('****************Validation with dataset epoch beginning******************')
dataloader.reset_batch_pointer(valid=False)
dataset_pointer_ins = dataloader.dataset_pointer
validation_dataset_executed = True
loss_epoch = 0
err_epoch = 0
f_err_epoch = 0
num_of_batch = 0
smallest_err = 100000
#results of one epoch for all validation datasets
epoch_result = []
#results of one validation dataset
results = []
# For each batch
for batch in range(dataloader.num_batches):
# Get batch data
x, y, d , numPedsList, PedsList ,target_ids = dataloader.next_batch()
if dataset_pointer_ins is not dataloader.dataset_pointer:
if dataloader.dataset_pointer is not 0:
print('Finished prosessed file : ', dataloader.get_file_name(-1),' Avarage error : ', err_epoch/num_of_batch)
num_of_batch = 0
epoch_result.append(results)
dataset_pointer_ins = dataloader.dataset_pointer
results = []
# Loss for this batch
loss_batch = 0
err_batch = 0
f_err_batch = 0
# For each sequence
for sequence in range(dataloader.batch_size):
# Get data corresponding to the current sequence
x_seq ,_ , d_seq, numPedsList_seq, PedsList_seq = x[sequence], y[sequence], d[sequence], numPedsList[sequence], PedsList[sequence]
target_id = target_ids[sequence]
#get processing file name and then get dimensions of file
folder_name = dataloader.get_directory_name_with_pointer(d_seq)
dataset_data = dataloader.get_dataset_dimension(folder_name)
#dense vector creation
x_seq, lookup_seq = dataloader.convert_proper_array(x_seq, numPedsList_seq, PedsList_seq)
#will be used for error calculation
orig_x_seq = x_seq.clone()
target_id_values = orig_x_seq[0][lookup_seq[target_id], 0:2]
#grid mask calculation
grid_seq = getSequenceGridMask(x_seq, dataset_data, PedsList_seq, args.neighborhood_size, args.grid_size, args.use_cuda)
#vectorize datapoints
if args.use_cuda:
x_seq = x_seq.cuda()
x_seq, first_values_dict = vectorize_seq(x_seq, PedsList_seq, lookup_seq)
# <---------------------- Experimental block ----------------------->
# x_seq = translate(x_seq, PedsList_seq, lookup_seq ,target_id_values)
# angle = angle_between(reference_point, (x_seq[1][lookup_seq[target_id], 0].data.numpy(), x_seq[1][lookup_seq[target_id], 1].data.numpy()))
# x_seq = rotate_traj_with_target_ped(x_seq, angle, PedsList_seq, lookup_seq)
# grid_seq = getSequenceGridMask(x_seq, dataset_data, PedsList_seq, args.neighborhood_size, args.grid_size, args.use_cuda)
# x_seq, first_values_dict = vectorize_seq(x_seq, PedsList_seq, lookup_seq)
if args.use_cuda:
x_seq = x_seq.cuda()
#sample predicted points from model
ret_x_seq, loss = sample_validation_data(x_seq, PedsList_seq, grid_seq, args, net, lookup_seq, numPedsList_seq, dataloader)
#revert the points back to original space
ret_x_seq = revert_seq(ret_x_seq, PedsList_seq, lookup_seq, first_values_dict)
# <---------------------- Experimental block revert----------------------->
# Revert the calculated coordinates back to original space:
# 1) Convert point from vectors to absolute coordinates
# 2) Rotate all trajectories in reverse angle
# 3) Translate all trajectories back to original space by adding the first frame value of target ped trajectory
# *It works without problems which mean that it reverts a trajectory back completely
# Possible problems:
# *Algoritmical errors caused by first experimental block -> High possiblity
# <------------------------------------------------------------------------>
# ret_x_seq = revert_seq(ret_x_seq, PedsList_seq, lookup_seq, first_values_dict)
# ret_x_seq = rotate_traj_with_target_ped(ret_x_seq, -angle, PedsList_seq, lookup_seq)
# ret_x_seq = translate(ret_x_seq, PedsList_seq, lookup_seq ,-target_id_values)
#get mean and final error
err = get_mean_error(ret_x_seq.data, orig_x_seq.data, PedsList_seq, PedsList_seq, args.use_cuda, lookup_seq)
f_err = get_final_error(ret_x_seq.data, orig_x_seq.data, PedsList_seq, PedsList_seq, args.use_cuda, lookup_seq)
loss_batch += loss.item()
err_batch += err
f_err_batch += f_err
print('Current file : ', dataloader.get_file_name(0),' Batch : ', batch+1, ' Sequence: ', sequence+1, ' Sequence mean error: ', err,' Sequence final error: ',f_err,' time: ', end - start)
results.append((orig_x_seq.data.cpu().numpy(), ret_x_seq.data.cpu().numpy(), PedsList_seq, lookup_seq, dataloader.get_frame_sequence(args.seq_length), target_id))
loss_batch = loss_batch / dataloader.batch_size
err_batch = err_batch / dataloader.batch_size
f_err_batch = f_err_batch / dataloader.batch_size
num_of_batch += 1
loss_epoch += loss_batch
err_epoch += err_batch
f_err_epoch += f_err_batch
epoch_result.append(results)
all_epoch_results.append(epoch_result)
if dataloader.num_batches != 0:
loss_epoch = loss_epoch / dataloader.num_batches
err_epoch = err_epoch / dataloader.num_batches
f_err_epoch = f_err_epoch / dataloader.num_batches
avarage_err = (err_epoch + f_err_epoch)/2
# Update best validation loss until now
if loss_epoch < best_val_data_loss:
best_val_data_loss = loss_epoch
best_epoch_val_data = epoch
if avarage_err<smallest_err_val_data:
smallest_err_val_data = avarage_err
best_err_epoch_val_data = epoch
print('(epoch {}), valid_loss = {:.3f}, valid_mean_err = {:.3f}, valid_final_err = {:.3f}'.format(epoch, loss_epoch, err_epoch, f_err_epoch))
print('Best epoch', best_epoch_val_data, 'Best validation loss', best_val_data_loss, 'Best error epoch',best_err_epoch_val_data, 'Best error', smallest_err_val_data)
log_file_curve.write("Validation dataset epoch: "+str(epoch)+" loss: "+str(loss_epoch)+" mean_err: "+str(err_epoch)+'final_err: '+str(f_err_epoch)+'\n')
optimizer = time_lr_scheduler(optimizer, epoch, lr_decay_epoch = args.freq_optimizer)
# Save the model after each epoch
print('Saving model')
torch.save({
'epoch': epoch,
'state_dict': net.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, checkpoint_path(epoch))
if dataloader.valid_num_batches != 0:
print('Best epoch', best_epoch_val, 'Best validation Loss', best_val_loss, 'Best error epoch',best_err_epoch_val, 'Best error', smallest_err_val)
# Log the best epoch and best validation loss
log_file.write('Validation Best epoch:'+str(best_epoch_val)+','+' Best validation Loss: '+str(best_val_loss))
if dataloader.additional_validation:
print('Best epoch acording to validation dataset', best_epoch_val_data, 'Best validation Loss', best_val_data_loss, 'Best error epoch',best_err_epoch_val_data, 'Best error', smallest_err_val_data)
log_file.write("Validation dataset Best epoch: "+str(best_epoch_val_data)+','+' Best validation Loss: '+str(best_val_data_loss)+'\n')
#dataloader.write_to_plot_file(all_epoch_results[best_epoch_val_data], plot_directory)
#elif dataloader.valid_num_batches != 0:
# dataloader.write_to_plot_file(all_epoch_results[best_epoch_val], plot_directory)
#else:
if validation_dataset_executed:
dataloader.switch_to_dataset_type(load_data=False)
create_directories(plot_directory, [plot_train_file_directory])
dataloader.write_to_plot_file(all_epoch_results[len(all_epoch_results)-1], os.path.join(plot_directory, plot_train_file_directory))
# Close logging files
log_file.close()
log_file_curve.close()
def main():
parser = argparse.ArgumentParser()
# Model to be loaded
parser.add_argument('--epoch', type=int, default=15,
help='Epoch of model to be loaded')
parser.add_argument('--seq_length', type=int, default=20,
help='RNN sequence length')
parser.add_argument('--use_cuda', action="store_true", default=False,
help='Use GPU or not')
parser.add_argument('--drive', action="store_true", default=False,
help='Use Google drive or not')
# Size of neighborhood to be considered parameter
parser.add_argument('--neighborhood_size', type=int, default=32,
help='Neighborhood size to be considered for social grid')
# Size of the social grid parameter
parser.add_argument('--grid_size', type=int, default=4,
help='Grid size of the social grid')
# number of validation will be used
parser.add_argument('--num_validation', type=int, default=5,
help='Total number of validation dataset will be visualized')
# gru support
parser.add_argument('--gru', action="store_true", default=False,
help='True : GRU cell, False: LSTM cell')
# method selection
parser.add_argument('--method', type=int, default=1,
help='Method of lstm will be used (1 = social lstm, 2 = obstacle lstm, 3 = vanilla lstm)')
# Parse the parameters
sample_args = parser.parse_args()
#for drive run
prefix = ''
f_prefix = '.'
if sample_args.drive is True:
prefix='drive/semester_project/social_lstm_final/'
f_prefix = 'drive/semester_project/social_lstm_final'
method_name = getMethodName(sample_args.method)
model_name = "LSTM"
save_tar_name = method_name+"_lstm_model_"
if sample_args.gru:
model_name = "GRU"
save_tar_name = method_name+"_gru_model_"
# Save directory
save_directory = os.path.join(f_prefix, 'model/', method_name, model_name)
#plot directory for plotting in the future
plot_directory = os.path.join(f_prefix, 'plot/', method_name, model_name)
plot_validation_file_directory = 'validation'
# Define the path for the config file for saved args
with open(os.path.join(save_directory,'config.pkl'), 'rb') as f:
saved_args = pickle.load(f)
origin = (0,0)
reference_point = (0,1)
net = getModel(sample_args.method, saved_args, True)
if sample_args.use_cuda:
net = net.cuda()
# Get the checkpoint path
checkpoint_path = os.path.join(save_directory, save_tar_name+str(sample_args.epoch)+'.tar')
if os.path.isfile(checkpoint_path):
print('Loading checkpoint')
checkpoint = torch.load(checkpoint_path)
model_epoch = checkpoint['epoch']
net.load_state_dict(checkpoint['state_dict'])
print('Loaded checkpoint at epoch', model_epoch)
# Create the DataLoader object
dataloader = DataLoader(f_prefix, 1, sample_args.seq_length, num_of_validation = sample_args.num_validation, forcePreProcess = True, infer = True)
createDirectories(plot_directory, [plot_validation_file_directory])
dataloader.reset_batch_pointer()
print('****************Validation dataset batch processing******************')
dataloader.reset_batch_pointer(valid=False)
dataset_pointer_ins = dataloader.dataset_pointer
loss_epoch = 0
err_epoch = 0
f_err_epoch = 0
num_of_batch = 0
smallest_err = 100000
#results of one epoch for all validation datasets
epoch_result = []
#results of one validation dataset
results = []
# For each batch
for batch in range(dataloader.num_batches):
start = time.time()
# Get batch data
x, y, d , numPedsList, PedsList ,target_ids = dataloader.next_batch()
if dataset_pointer_ins is not dataloader.dataset_pointer:
if dataloader.dataset_pointer is not 0:
print('Finished prosessed file : ', dataloader.get_file_name(-1),' Avarage error : ', err_epoch/num_of_batch)
num_of_batch = 0
epoch_result.append(results)
dataset_pointer_ins = dataloader.dataset_pointer
results = []
# Loss for this batch
loss_batch = 0
err_batch = 0
f_err_batch = 0
# For each sequence
for sequence in range(dataloader.batch_size):
# Get data corresponding to the current sequence
x_seq ,_ , d_seq, numPedsList_seq, PedsList_seq = x[sequence], y[sequence], d[sequence], numPedsList[sequence], PedsList[sequence]
target_id = target_ids[sequence]
folder_name = dataloader.get_directory_name_with_pointer(d_seq)
dataset_data = dataloader.get_dataset_dimension(folder_name)
#dense vector creation
x_seq, lookup_seq = dataloader.convert_proper_array(x_seq, numPedsList_seq, PedsList_seq)
#will be used for error calculation
orig_x_seq = x_seq.clone()
target_id_values = x_seq[0][lookup_seq[target_id], 0:2]
#grid mask calculation
if sample_args.method == 2: #obstacle lstm
grid_seq = getSequenceGridMask(x_seq, dataset_data, PedsList_seq, saved_args.neighborhood_size, saved_args.grid_size, saved_args.use_cuda, True)
elif sample_args.method == 1: #social lstm
grid_seq = getSequenceGridMask(x_seq, dataset_data, PedsList_seq, saved_args.neighborhood_size, saved_args.grid_size, saved_args.use_cuda)
#vectorize datapoints
x_seq, first_values_dict = vectorizeSeq(x_seq, PedsList_seq, lookup_seq)
# <---------------- Experimental block (may need update in methods)----------------------->
# x_seq = translate(x_seq, PedsList_seq, lookup_seq ,target_id_values)
# angle = angle_between(reference_point, (x_seq[1][lookup_seq[target_id], 0].data.numpy(), x_seq[1][lookup_seq[target_id], 1].data.numpy()))
# x_seq = rotate_traj_with_target_ped(x_seq, angle, PedsList_seq, lookup_seq)
# # Compute grid masks
# grid_seq = getSequenceGridMask(x_seq, dataset_data, PedsList_seq, sample_args.neighborhood_size, sample_args.grid_size, sample_args.use_cuda)
# x_seq, first_values_dict = vectorize_seq(x_seq, PedsList_seq, lookup_seq)
if sample_args.use_cuda:
x_seq = x_seq.cuda()
if sample_args.method == 3: #vanilla lstm
ret_x_seq, loss = sampleValidationDataVanilla(x_seq, PedsList_seq, sample_args, net, lookup_seq, numPedsList_seq, dataloader)
else:
ret_x_seq, loss = sample_validation_data(x_seq, PedsList_seq, grid_seq, sample_args, net, lookup_seq, numPedsList_seq, dataloader)
#<---------------------Experimental inverse block -------------->
# ret_x_seq = revert_seq(ret_x_seq, PedsList_seq, lookup_seq, target_id_values, first_values_dict)
# ret_x_seq = rotate_traj_with_target_ped(ret_x_seq, -angle, PedsList_seq, lookup_seq)
# ret_x_seq = translate(ret_x_seq, PedsList_seq, lookup_seq ,-target_id_values)
#revert the points back to original space
ret_x_seq = revertSeq(ret_x_seq, PedsList_seq, lookup_seq, first_values_dict)
err = getMeanError(ret_x_seq.data, orig_x_seq.data, PedsList_seq, PedsList_seq, sample_args.use_cuda, lookup_seq)
f_err = getFinalError(ret_x_seq.data, orig_x_seq.data, PedsList_seq, PedsList_seq, lookup_seq)
loss_batch += loss
err_batch += err
f_err_batch += f_err
results.append((orig_x_seq.data.cpu().numpy(), ret_x_seq.data.cpu().numpy(), PedsList_seq, lookup_seq, dataloader.get_frame_sequence(sample_args.seq_length), target_id))
end = time.time()
print('Current file : ', dataloader.get_file_name(0),' Batch : ', batch+1, ' Sequence: ', sequence+1, ' Sequence mean error: ', err,' Sequence final error: ',f_err,' time: ', end - start)
loss_batch = loss_batch / dataloader.batch_size
err_batch = err_batch / dataloader.batch_size
f_err_batch = f_err_batch / dataloader.batch_size
num_of_batch += 1
loss_epoch += loss_batch.item()
err_epoch += err_batch
f_err_epoch += f_err_batch
epoch_result.append(results)
if dataloader.num_batches != 0:
loss_epoch = loss_epoch / dataloader.num_batches
err_epoch = err_epoch / dataloader.num_batches
f_err_epoch = f_err_epoch / dataloader.num_batches
print('valid_loss = {:.3f}, valid_mean_err = {:.3f}, valid_final_err = {:.3f}'.format(loss_epoch, err_epoch, f_err_epoch))
dataloader.write_to_plot_file(epoch_result, os.path.join(plot_directory, plot_validation_file_directory))
def train(args):
datasets = range(4)
# Remove the leaveDataset from datasets
datasets.remove(args.leaveDataset)
# Create the SocialDataLoader object
data_loader = SocialDataLoader(args.batch_size,
args.seq_length,
args.maxNumPeds,
datasets,
forcePreProcess=True)
with open(os.path.join('save', 'social_config.pkl'), 'wb') as f:
pickle.dump(args, f)
# Create a SocialModel object with the arguments
model = SocialModel(args)
# Initialize a TensorFlow session
with tf.Session() as sess:
# Initialize all variables in the graph
sess.run(tf.initialize_all_variables())
# Initialize a saver that saves all the variables in the graph
saver = tf.train.Saver(tf.all_variables())
# summary_writer = tf.train.SummaryWriter('/tmp/lstm/logs', graph_def=sess.graph_def)
# For each epoch
for e in range(args.num_epochs):
# Assign the learning rate value for this epoch
sess.run(
tf.assign(model.lr, args.learning_rate * (args.decay_rate**e)))
# Reset the data pointers in the data_loader
data_loader.reset_batch_pointer()
# For each batch
for b in range(data_loader.num_batches):
# Tic
start = time.time()
# Get the source, target and dataset data for the next batch
# x, y are input and target data which are lists containing numpy arrays of size seq_length x maxNumPeds x 3
# d is the list of dataset indices from which each batch is generated (used to differentiate between datasets)
x, y, d = data_loader.next_batch()
# variable to store the loss for this batch
loss_batch = 0
# For each sequence in the batch
for batch in range(data_loader.batch_size):
# x_batch, y_batch and d_batch contains the source, target and dataset index data for
# seq_length long consecutive frames in the dataset
# x_batch, y_batch would be numpy arrays of size seq_length x maxNumPeds x 3
# d_batch would be a scalar identifying the dataset from which this sequence is extracted
x_batch, y_batch, d_batch = x[batch], y[batch], d[batch]
if d_batch == 0 and datasets[0] == 0:
dataset_data = [640, 480]
else:
dataset_data = [720, 576]
grid_batch = getSequenceGridMask(x_batch, dataset_data,
args.neighborhood_size,
args.grid_size)
# Feed the source, target data
feed = {
model.input_data: x_batch,
model.target_data: y_batch,
model.grid_data: grid_batch
}
train_loss, _ = sess.run([model.cost, model.train_op],
feed)
loss_batch += train_loss
end = time.time()
loss_batch = loss_batch / data_loader.batch_size
print(
"{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}"
.format(e * data_loader.num_batches + b,
args.num_epochs * data_loader.num_batches, e,
loss_batch, end - start))
# Save the model if the current epoch and batch number match the frequency
if (e * data_loader.num_batches +
b) % args.save_every == 0 and (
(e * data_loader.num_batches + b) > 0):
checkpoint_path = os.path.join('save', 'social_model.ckpt')
saver.save(sess,
checkpoint_path,
global_step=e * data_loader.num_batches + b)
print("model saved to {}".format(checkpoint_path))
def testHelper(net, test_loader, sample_args, saved_args):
num_batches = math.floor(len(test_loader.dataset) / sample_args.batch_size)
# For each batch
iteration_submission = []
iteration_result = []
results = []
submission = []
# Variable to maintain total error
total_error = 0
final_error = 0
# *Kevin Murphy*
norm_l2_dists = torch.zeros(sample_args.obs_length)
if sample_args.use_cuda:
norm_l2_dists = norm_l2_dists.cuda()
for batch_idx, batch in enumerate(test_loader):
start = time.time()
# Get the sequence
x_seq, num_peds_list_seq, peds_list_seq, folder_path = batch[
0] # because source code assumes batch_size=0 and doesn't iterate over sequences of a batch
# Get processing file name and then get dimensions of file
folder_name = getFolderName(folder_path, sample_args.dataset)
dataset_data = dataset_dimensions[folder_name]
# Dense vector creation
x_seq, lookup_seq = convertToTensor(x_seq, peds_list_seq)
# Will be used for error calculation
orig_x_seq = x_seq.clone()
# Grid mask calculation
if sample_args.method == 2: # obstacle lstm
grid_seq = getSequenceGridMask(x_seq, dataset_data, peds_list_seq,
saved_args.neighborhood_size,
saved_args.grid_size,
saved_args.use_cuda, True)
elif sample_args.method == 1: # social lstm
grid_seq = getSequenceGridMask(x_seq, dataset_data, peds_list_seq,
saved_args.neighborhood_size,
saved_args.grid_size,
saved_args.use_cuda)
# Replace relative positions with true positions in x_seq
x_seq, first_values_dict = vectorizeSeq(x_seq, peds_list_seq,
lookup_seq)
# *CUDA*
if sample_args.use_cuda:
x_seq = x_seq.cuda()
first_values_dict = {
k: v.cuda()
for k, v in first_values_dict.items()
}
orig_x_seq = orig_x_seq.cuda()
# The sample function
if sample_args.method == 3: # vanilla lstm
# Extract the observed part of the trajectories
obs_traj, obs_PedsList_seq = x_seq[:sample_args.
obs_length], peds_list_seq[:
sample_args
.
obs_length]
ret_x_seq = sample(obs_traj, obs_PedsList_seq, sample_args, net,
x_seq, peds_list_seq, saved_args, dataset_data,
test_loader, lookup_seq, num_peds_list_seq,
sample_args.gru)
else:
# Extract the observed part of the trajectories
obs_traj, obs_PedsList_seq, obs_grid = x_seq[:sample_args.
obs_length], peds_list_seq[:
sample_args
.
obs_length], grid_seq[:
sample_args
.
obs_length]
ret_x_seq = sample(obs_traj, obs_PedsList_seq, sample_args, net,
x_seq, peds_list_seq, saved_args, dataset_data,
test_loader, lookup_seq, num_peds_list_seq,
sample_args.gru, obs_grid)
# revert the points back to original space
ret_x_seq = revertSeq(ret_x_seq, peds_list_seq, lookup_seq,
first_values_dict)
# *CUDA*
if sample_args.use_cuda:
ret_x_seq = ret_x_seq.cuda()
# *ORIGINAL TEST*
total_error += getMeanError(ret_x_seq[sample_args.obs_length:].data,
orig_x_seq[sample_args.obs_length:].data,
peds_list_seq[sample_args.obs_length:],
peds_list_seq[sample_args.obs_length:],
sample_args.use_cuda, lookup_seq)
final_error += getFinalError(ret_x_seq[sample_args.obs_length:].data,
orig_x_seq[sample_args.obs_length:].data,
peds_list_seq[sample_args.obs_length:],
peds_list_seq[sample_args.obs_length:],
lookup_seq)
# *Kevin Murphy*
norm_l2_dists += getNormalizedL2Distance(
ret_x_seq[:sample_args.obs_length].data,
orig_x_seq[:sample_args.obs_length].data,
peds_list_seq[:sample_args.obs_length],
peds_list_seq[:sample_args.obs_length], sample_args.use_cuda,
lookup_seq)
end = time.time()
print('Current file : ', folder_name,
' Processed trajectory number : ', batch_idx + 1, 'out of',
num_batches, 'trajectories in time', end - start)
return total_error, final_error, norm_l2_dists