def get_predictions(args, dataloader, model):
all_predictions = []
if args.key == 'covernet':
trajectories = np.array(get_fixed_trajectory_set(args))
for j, (img, agent_state_vector, _, instance_tokens, sample_tokens) in enumerate(dataloader):
if (j % 20 == 0) and (j > 0):
print(f"Completed batch {j}", datetime.datetime.now())
img = img.to(device).float()
agent_state_vector = agent_state_vector.to(device).float()
with torch.no_grad():
model_pred = model(img, agent_state_vector)
model_pred = model_pred.cpu().detach().numpy()
# model outputs are x, y positions
if args.key == 'mtp':
for i in range(model_pred.shape[0]):
pred = model_pred[i]
instance_token = instance_tokens[i]
sample_token = sample_tokens[i]
logits = pred[-args.num_modes:]
trajectories = pred[:-args.num_modes].reshape(args.num_modes, args.n_steps, 2)
# only write the top K to the predictions file
topk_idx = np.argsort(logits)[:args.top_k]
all_predictions.append(Prediction(
instance_token,
sample_token,
trajectories[topk_idx],
F.softmax(torch.Tensor(logits[topk_idx]), dim=0).numpy()).serialize())
# model outputs are logits corresponding to a pre-built fixed trajectory set
elif args.key == 'covernet':
for i in range(model_pred.shape[0]):
trajectories_cpy = copy.deepcopy(trajectories)
logits = model_pred[i]
instance_token = instance_tokens[i]
sample_token = sample_tokens[i]
trajectories_cpy = trajectories[np.argsort(logits)[::-1]]
trajectories_cpy = trajectories_cpy[:args.top_k]
logits = logits[np.argsort(logits)[::-1]]
logits = logits[:args.top_k]
all_predictions.append(Prediction(
instance_token,
sample_token,
trajectories_cpy,
F.softmax(torch.Tensor(logits), dim=0).numpy()).serialize())
return all_predictions
def test(self):
prediction = Prediction('instance', 'sample', np.ones((2, 2, 2)),
np.zeros(2))
self.assertEqual(prediction.number_of_modes, 2)
self.assertDictEqual(
prediction.serialize(), {
'instance': 'instance',
'sample': 'sample',
'prediction': [[[1, 1], [1, 1]], [[1, 1], [1, 1]]],
'probabilities': [0, 0]
})
def compute_metrics(predictions: List[Dict[str, Any]], helper: PredictHelper,
config: PredictionConfig) -> Dict[str, Any]:
"""
Computes metrics from a set of predictions.
:param predictions: List of prediction JSON objects.
:param helper: Instance of PredictHelper that wraps the nuScenes val set.
:param config: Config file.
:return: Metrics. Nested dictionary where keys are metric names and value is a dictionary
mapping the Aggregator name to the results.
"""
n_preds = len(predictions)
containers = {
metric.name: np.zeros((n_preds, metric.shape))
for metric in config.metrics
}
for i, prediction_str in enumerate(predictions):
prediction = Prediction.deserialize(prediction_str)
ground_truth = helper.get_future_for_agent(prediction.instance,
prediction.sample,
config.seconds,
in_agent_frame=False)
for metric in config.metrics:
containers[metric.name][i] = metric(ground_truth, prediction)
aggregations: Dict[str, Dict[str, List[float]]] = defaultdict(dict)
for metric in config.metrics:
for agg in metric.aggregators:
aggregations[metric.name][agg.name] = agg(containers[metric.name])
return aggregations
def compute_metrics(self,
config_file='predict_2020_icra.json',
result_file='metrics.json',
agent_frame=True):
config = load_prediction_config(self.helper, config_file)
predictions = json.load(
open(os.path.join(self.output_dir, 'mtp_preds.json'), "r"))
n_preds = len(predictions)
result_output_path = os.path.join(self.output_dir, result_file)
containers = {
metric.name: np.zeros((n_preds, metric.shape))
for metric in config.metrics
}
for i, prediction_str in enumerate(predictions):
prediction = Prediction.deserialize(prediction_str)
ground_truth = exp.helper.get_future_for_agent(
prediction.instance,
prediction.sample,
config.seconds,
in_agent_frame=agent_frame)
for metric in config.metrics:
containers[metric.name][i] = metric(ground_truth, prediction)
aggregations: Dict[str, Dict[str, List[float]]] = defaultdict(dict)
for metric in config.metrics:
for agg in metric.aggregators:
aggregations[metric.name][agg.name] = agg(
containers[metric.name])
print(aggregations)
json.dump(aggregations, open(result_output_path, "w"), indent=2)
def __call__(self, token: str) -> Prediction:
"""
Makes prediction.
:param token: string of format {instance_token}_{sample_token}.
"""
instance_token_img, sample_token_img = token.split("_")
# kinematics = _kinematics_from_tokens(self.helper, instance, sample)
# cv_heading = _constant_velocity_heading_from_kinematics(kinematics, self.sec_from_now, self.sampled_at)
# anns = [ann for ann in nuscenes.sample_annotation if ann['instance_token'] == instance_token_img]
img = self.mtp_input_representation.make_input_representation(instance_token_img, sample_token_img)
image_tensor = torch.Tensor(img).permute(2, 0, 1).unsqueeze(0)
# plt.imshow(img)
agent_state_vector = torch.Tensor([[self.helper.get_velocity_for_agent(instance_token_img, sample_token_img),
self.helper.get_acceleration_for_agent(instance_token_img, sample_token_img),
self.helper.get_heading_change_rate_for_agent(instance_token_img,
sample_token_img)]])
mtp_out = self.mtp(image_tensor, agent_state_vector)
covernet_logits = self.covernet(image_tensor, agent_state_vector)
covernet_probabilities = covernet_logits.argsort(descending=True).squeeze()
covernet_probabilities = covernet_probabilities[:5] # Print 5 most likely output
covernet_trajectories = self.trajectories[covernet_probabilities]
covernet_trajectories = covernet_trajectories.detach().cpu().numpy()
covernet_probabilities = covernet_probabilities.detach().cpu().numpy()
# Need the prediction to have 2d.
return Prediction(instance_token_img, sample_token_img, covernet_trajectories, covernet_probabilities)
def __call__(self, token) -> Prediction:
"""
Makes prediction.
:param token: string of format {instance_token}_{sample_token}.
"""
instance, sample = token.split("_")
kinematics = _kinematics_from_tokens(self.helper, instance, sample)
ground_truth = self.helper.get_future_for_agent(instance, sample, self.sec_from_now, in_agent_frame=False)
assert ground_truth.shape[0] == int(self.sec_from_now * self.sampled_at), ("Ground truth does not correspond "
f"to {self.sec_from_now} seconds.")
path_funs = [
_constant_acceleration_and_heading,
_constant_magnitude_accel_and_yaw_rate,
_constant_speed_and_yaw_rate,
_constant_velocity_heading_from_kinematics
]
paths = [path_fun(kinematics, self.sec_from_now, self.sampled_at) for path_fun in path_funs]
# Select the one with the least l2 error, averaged (or equivalently, summed) over all
# points of the path. This is (proportional to) the Frobenius norm of the difference
# between the path (as an n x 2 matrix) and the ground truth.
# oracle = sorted(paths,
# key=lambda path: np.linalg.norm(np.array(path) - ground_truth, ord="fro"))[0]
paths = np.array(paths)
probs = np.ones(paths.shape[0]) / paths.shape[0]
return Prediction(instance, sample, paths, probs)
def __call__(self, token: str) -> Prediction:
"""
Makes prediction.
:param token: string of format {instance_token}_{sample_token}.
"""
instance, sample = token.split("_")
kinematics = _kinematics_from_tokens(self.helper, instance, sample)
cv_heading = _constant_velocity_heading_from_kinematics(kinematics, self.sec_from_now, self.sampled_at)
# Need the prediction to have 2d.
return Prediction(instance, sample, np.expand_dims(cv_heading, 0), np.array([1]))
def _do_test(self, map_name, predictions, answer):
with patch.object(PredictHelper,
'get_map_name_from_sample_token') as get_map_name:
get_map_name.return_value = map_name
nusc = NuScenes('v1.0-mini', dataroot=os.environ['NUSCENES'])
helper = PredictHelper(nusc)
off_road_rate = metrics.OffRoadRate(helper, [metrics.RowMean()])
probabilities = np.array([1 / 3] * predictions.shape[0])
prediction = Prediction('foo-instance', 'foo-sample', predictions,
probabilities)
# Two violations out of three trajectories
np.testing.assert_allclose(off_road_rate(np.array([]), prediction),
np.array([answer]))
def generate_predictions_from_validation(self,
json_file_name='mtp_preds.json'):
mtp_output = []
val_loader = DataLoader(self.validation_set,
batch_size=1,
shuffle=True,
pin_memory=True,
num_workers=0)
print('starting validation predictions')
self.model.eval()
prediction_output_path = os.path.join(self.output_dir, json_file_name)
with torch.no_grad():
count = 0
for image_tensor, agent_state_vec, ground_truth, token in val_loader:
config = self.validation_set.config
instance_token_img, sample_token_img = token[0].split('_')
output = self.model(image_tensor.to(self.device),
agent_state_vec.to(self.device))
prediction = output[:, :-self.num_modes].cpu().numpy()
probabilites = output[:, -self.num_modes:].squeeze(
0).cpu().numpy()
prediction = prediction.reshape(self.num_modes,
config.seconds * 2, 2)
serialized_pred = Prediction(instance_token_img,
sample_token_img, prediction,
probabilites).serialize()
mtp_output.append(serialized_pred)
print("{0}/{1} predictions saved".format(
count, len(val_loader)))
count += 1
print("saving predictions")
json.dump(mtp_output, open(prediction_output_path, "w"))
def compute_metrics(predictions: List[Dict[str, Any]], helper: PredictHelper,
config: PredictionConfig) -> Dict[str, Any]:
"""
Computes metrics from a set of output.
:param predictions: List of prediction JSON objects.
:param helper: Instance of PredictHelper that wraps the nuScenes val set.
:param config: Config file.
:return: Metrics. Nested dictionary where keys are metric names and value is a dictionary
mapping the Aggregator name to the results.
"""
n_preds = len(predictions)
containers = {
metric.name: np.zeros((n_preds, metric.shape))
for metric in config.metrics
}
containers['imginfo'] = []
containers['tokens'] = []
BACK_SAMPLES = 5
for i, prediction_str in enumerate(tqdm(predictions)):
prediction = Prediction.deserialize(prediction_str)
ground_truth = helper.get_future_for_agent(prediction.instance,
prediction.sample,
config.seconds,
in_agent_frame=False)
agent_past = helper.get_past_for_agent(prediction.instance,
prediction.sample,
BACK_SAMPLES * 0.5,
in_agent_frame=False,
just_xy=False)
if len(agent_past) < BACK_SAMPLES:
print('Sample {} didnt have enough history {}'.format(
i, len(agent_past)))
continue
cam_names = [
sensor['channel'] for sensor in nusc.sensor
if 'CAM' in sensor['channel']
]
containers['tokens'].append(prediction_str)
cameras = []
containers['imginfo'].append(cameras)
# Append current timestep
d = {}
cameras.append(d)
token = prediction.instance + '_' + prediction.sample
for cam_name in cam_names:
d[cam_name] = get_im_and_box(
nusc, token, cam_name=cam_name,
imgAsName=True) # impath, box, camera_intrinsics
# Append earlier timesteps
for t in range(BACK_SAMPLES):
d = {}
cameras.append(d)
token = agent_past[t]['instance_token'] + '_' + agent_past[t][
'sample_token']
for cam_name in cam_names:
d[cam_name] = get_im_and_box(
nusc, token, cam_name=cam_name,
imgAsName=True) # impath, box, camera_intrinsics
for metric in config.metrics:
containers[metric.name][i] = metric(ground_truth, prediction)
aggregations: Dict[str, Dict[str, List[float]]] = defaultdict(dict)
for metric in config.metrics:
for agg in metric.aggregators:
aggregations[metric.name][agg.name] = agg(containers[metric.name])
return aggregations, containers
pred_local = pred_local[counts != 0]
counts = counts[counts != 0]
counts = counts.numpy()
pred_local = pred_local.numpy()
starting_annotation = helper.get_sample_annotation(
instance_token[n], sample_token[n])
pred_global = np.zeros_like(pred_local)
for m in range(pred_local.shape[0]):
pred_global[m] = convert_local_coords_to_global(
pred_local[m], starting_annotation['translation'],
starting_annotation['rotation'])
preds5.append(
Prediction(instance=instance_token[n],
sample=sample_token[n],
prediction=pred_global,
probabilities=counts).serialize())
# Cluster (K=10)
params = [(traj_vec[i], 10) for i in range(len(traj_vec))]
labels = process_pool.starmap(u.km_cluster, params)
traj_clustered = torch.zeros(traj.shape[0], 10, traj.shape[2],
traj.shape[3])
counts_clustered = torch.zeros(traj.shape[0], 10)
for m in range(traj.shape[0]):
clusters = set(labels[m])
tmp1 = torch.zeros(len(clusters), traj.shape[2], traj.shape[3])
tmp2 = torch.zeros(len(clusters))
for idx, c in enumerate(clusters):
tmp = np.where(labels[m] == c)
tmp1[idx] = torch.mean(traj[m, tmp[0]], dim=0)