def batches_of_patches_with_position(self,
full_example,
window_step_size=32):
"""
A generator for extracting patches from an image in batches.
:param full_example: The full example to be patched.
:type full_example: CrowdExample
:param window_step_size: The sliding window size.
:type window_step_size: int
:return: A batch of patches.
:rtype: list[list[CrowdExample]]
"""
extract_patch_transform = ExtractPatchForPosition()
test_transform = torchvision.transforms.Compose([
data.NegativeOneToOneNormalizeImage(),
data.NumpyArraysToTorchTensors()
])
batch = []
for y in range(0, full_example.label.shape[0], window_step_size):
for x in range(0, full_example.label.shape[1], window_step_size):
patch = extract_patch_transform(full_example, y, x)
example = test_transform(patch)
example_with_position = CrowdExample(image=example.image,
label=example.label,
patch_center_x=x,
patch_center_y=y)
batch.append(example_with_position)
if len(batch) == self.settings.batch_size:
yield batch
batch = []
yield batch
def __getitem__(self, index):
"""
:param index: The index within the entire dataset.
:type index: int
:return: An example and label from the crowd dataset.
:rtype: torch.Tensor, torch.Tensor
"""
index_ = random.randrange(self.length)
file_name_index = np.searchsorted(self.start_indexes, index_, side='right') - 1
start_index = self.start_indexes[file_name_index]
file_name = self.file_names[file_name_index]
position_index = index_ - start_index
extract_patch_transform = ExtractPatchForPosition(self.image_patch_size, self.label_patch_size,
allow_padded=True) # In case image is smaller than patch.
preprocess_transform = torchvision.transforms.Compose([NegativeOneToOneNormalizeImage(),
NumpyArraysToTorchTensors()])
image = np.load(os.path.join(self.dataset_directory, 'images', file_name), mmap_mode='r')
label = np.load(os.path.join(self.dataset_directory, 'labels', file_name), mmap_mode='r')
map_ = np.load(os.path.join(self.dataset_directory, self.map_directory_name, file_name), mmap_mode='r')
half_patch_size = int(self.image_patch_size // 2)
y_positions = range(half_patch_size, image.shape[0] - half_patch_size + 1)
x_positions = range(half_patch_size, image.shape[1] - half_patch_size + 1)
positions_shape = [len(y_positions), len(x_positions)]
y_index, x_index = np.unravel_index(position_index, positions_shape)
y = y_positions[y_index]
x = x_positions[x_index]
example = CrowdExample(image=image, label=label, map_=map_)
example = extract_patch_transform(example, y, x)
if self.middle_transform:
example = self.middle_transform(example)
example = preprocess_transform(example)
return example.image, example.label, example.map
def __getitem__(self, index):
"""
:param index: The index within the entire dataset.
:type index: int
:return: An example and label from the crowd dataset.
:rtype: torch.Tensor, torch.Tensor
"""
if self.unlabeled:
image = self.images[index]
label = np.zeros(self.images.shape[:2], dtype=np.float32)
example = CrowdExample(image=image, label=label)
else:
example = CrowdExample(image=self.images[index],
label=self.labels[index])
if self.transform:
example = self.transform(example)
return example.image, example.label
def test_summaries(self):
"""Evaluates the model on test data during training."""
test_dataset = self.dataset_class(
dataset='test',
map_directory_name=self.settings.map_directory_name)
if self.settings.test_summary_size is not None:
indexes = random.sample(range(test_dataset.length),
self.settings.test_summary_size)
else:
indexes = range(test_dataset.length)
dnn_mae_count = None
dnn_rmse_count = None
for network in [self.DNN, self.D]:
totals = defaultdict(lambda: 0)
for index in indexes:
full_image, full_label, full_maps = test_dataset[index]
full_example = CrowdExample(image=full_image, label=full_label)
full_predicted_count, full_predicted_label = self.predict_full_example(
full_example, network)
totals['Count error'] += np.abs(full_predicted_count -
full_example.label.sum())
totals['NAE'] += np.abs(
full_predicted_count -
full_example.label.sum()) / full_example.label.sum()
totals['Density sum error'] += np.abs(
full_predicted_label.sum() - full_example.label.sum())
totals['SE count'] += (full_predicted_count -
full_example.label.sum())**2
totals['SE density'] += (full_predicted_label.sum() -
full_example.label.sum())**2
if network is self.DNN:
summary_writer = self.dnn_summary_writer
else:
summary_writer = self.gan_summary_writer
nae_count = totals['NAE'] / len(indexes)
summary_writer.add_scalar('0 Test Error/NAE count', nae_count)
mae_count = totals['Count error'] / len(indexes)
summary_writer.add_scalar('0 Test Error/MAE count', mae_count)
mae_density = totals['Density sum error'] / len(indexes)
summary_writer.add_scalar('0 Test Error/MAE density', mae_density)
rmse_count = (totals['SE count'] / len(indexes))**0.5
summary_writer.add_scalar('0 Test Error/RMSE count', rmse_count)
rmse_density = (totals['SE density'] / len(indexes))**0.5
summary_writer.add_scalar('0 Test Error/RMSE density',
rmse_density)
if network is self.DNN:
dnn_mae_count = mae_count
dnn_rmse_count = rmse_count
else:
summary_writer.add_scalar('0 Test Error/Ratio MAE GAN DNN',
mae_count / dnn_mae_count)
summary_writer.add_scalar('0 Test Error/Ratio RMSE GAN DNN',
rmse_count / dnn_rmse_count)
def inference(self, input_):
"""
Run the inference for crowd detection.
"""
label = np.zeros(input_.shape[:2], dtype=np.float32)
example = CrowdExample(image=input_, label=label)
import datetime
start = datetime.datetime.now()
with torch.no_grad():
predicted_count, predicted_label = self.predict_full_example(
full_example=example, network=self.inference_network)
print(datetime.datetime.now() - start)
return predicted_count, predicted_label
def __getitem__(self, index):
"""
:param index: The index within the entire dataset.
:type index: int
:return: An example and label from the crowd dataset.
:rtype: torch.Tensor, torch.Tensor
"""
example = CrowdExample(image=self.images[index],
label=self.labels[index],
roi=self.rois[index],
perspective=self.perspectives[index])
if self.transform:
example = self.transform(example)
return example.image, example.label
def evaluate(self,
during_training=False,
step=None,
number_of_examples=None):
"""Evaluates the model on test data."""
super().evaluate()
for network in [self.DNN, self.D]:
test_dataset = self.settings.dataset_class(dataset='test')
totals = defaultdict(lambda: 0)
for full_example_index, (full_image,
full_label) in enumerate(test_dataset):
print(
'Processing full example {}...'.format(full_example_index),
end='\r')
full_example = CrowdExample(image=full_image, label=full_label)
full_predicted_count, full_predicted_label = self.predict_full_example(
full_example, network)
totals['Count'] += full_example.label.sum()
totals['Density error'] += np.abs(full_predicted_label -
full_example.label).sum()
totals['Count error'] += np.abs(full_predicted_count -
full_example.label.sum())
totals['Density sum error'] += np.abs(
full_predicted_label.sum() - full_example.label.sum())
totals['Predicted count'] += full_predicted_count
totals['Predicted density sum'] += full_predicted_label.sum()
totals['SE count'] += (full_predicted_count -
full_example.label.sum())**2
totals['SE density'] += (full_predicted_label.sum() -
full_example.label.sum())**2
else:
if network is self.DNN:
print('=== DNN ===')
else:
print('=== GAN ===')
print('MAE count: {}'.format(totals['Count error'] /
len(test_dataset)))
print('MAE density: {}'.format(totals['Density sum error'] /
len(test_dataset)))
print('MSE count: {}'.format(totals['SE count'] /
len(test_dataset)))
print('MSE density: {}'.format(totals['SE density'] /
len(test_dataset)))
for key, value in totals.items():
print('Total {}: {}'.format(key, value))
def evaluate(self,
during_training=False,
step=None,
number_of_examples=None):
"""Evaluates the model on test data."""
self.model_setup()
self.load_models()
self.gpu_mode()
self.eval_mode()
self.settings.dataset_class = UcfQnrfFullImageDataset
test_dataset = self.settings.dataset_class(dataset='test')
if self.settings.test_summary_size is not None:
indexes = random.sample(range(test_dataset.length),
self.settings.test_summary_size)
else:
indexes = range(test_dataset.length)
network = self.DNN
totals = defaultdict(lambda: 0)
for index in indexes:
full_image, full_label = test_dataset[index]
full_example = CrowdExample(image=full_image, label=full_label)
full_predicted_count, full_predicted_label = self.predict_full_example(
full_example, network)
totals['Count error'] += np.abs(full_predicted_count -
full_example.label.sum())
totals['Density sum error'] += np.abs(full_predicted_label.sum() -
full_example.label.sum())
totals['SE count'] += (full_predicted_count -
full_example.label.sum())**2
totals['SE density'] += (full_predicted_label.sum() -
full_example.label.sum())**2
print('Count MAE: {}'.format(totals['Count error'] / len(indexes)))
print('Count RMSE: {}'.format(
(totals['SE count'] / len(indexes))**0.5))
print('Density MAE: {}'.format(totals['Density sum error'] /
len(indexes)))
print('Density RMSE: {}'.format(
(totals['SE density'] / len(indexes))**0.5))
def __getitem__(self, index):
"""
:param index: The index within the entire dataset.
:type index: int
:return: An example and label from the crowd dataset.
:rtype: torch.Tensor, torch.Tensor
"""
index_ = random.randrange(self.length)
camera_data_index = np.searchsorted(self.start_indexes, index_, side='right') - 1
start_index = self.start_indexes[camera_data_index]
camera_data = self.camera_data_list[camera_data_index]
camera_images = camera_data.images
array_index = index_ - start_index
half_patch_size = int(self.image_patch_size // 2)
y_positions = range(half_patch_size, camera_images.shape[1] - half_patch_size + 1)
x_positions = range(half_patch_size, camera_images.shape[2] - half_patch_size + 1)
image_indexes_length = len(y_positions) * len(x_positions)
image_index = math.floor(array_index / image_indexes_length)
position_index = array_index % image_indexes_length
image = camera_data.images[image_index]
label = camera_data.labels[image_index]
map_ = label
extract_patch_transform = ExtractPatchForPosition(self.image_patch_size, self.label_patch_size,
allow_padded=True) # In case image is smaller than patch.
preprocess_transform = torchvision.transforms.Compose([NegativeOneToOneNormalizeImage(),
NumpyArraysToTorchTensors()])
y_positions = range(half_patch_size, image.shape[0] - half_patch_size + 1)
x_positions = range(half_patch_size, image.shape[1] - half_patch_size + 1)
positions_shape = [len(y_positions), len(x_positions)]
y_index, x_index = np.unravel_index(position_index, positions_shape)
y = y_positions[y_index]
x = x_positions[x_index]
example = CrowdExample(image=image, label=label, map_=map_)
example = extract_patch_transform(example, y, x)
if self.middle_transform:
example = self.middle_transform(example)
example = preprocess_transform(example)
return example.image, example.label, example.map
def evaluate(self, sliding_window_step=10):
super().evaluate()
settings = self.settings
if settings.crowd_dataset == 'ShanghaiTech':
test_dataset = ShanghaiTechDataset(dataset='test')
else:
raise ValueError('{} is not an understood crowd dataset.'.format(settings.crowd_dataset))
totals = {'count': 0, 'count_error': 0, 'density_error': 0, 'density_sum_error': 0, 'predicted_count': 0,
'predicted_density_sum': 0}
for full_example_index, (full_image, full_label) in enumerate(test_dataset):
print('Processing full example {}...'.format(full_example_index), end='\r')
full_example = CrowdExample(full_image, full_label)
sum_density_label = np.zeros_like(full_example.label, dtype=np.float32)
sum_count_label = np.zeros_like(full_example.label, dtype=np.float32)
hit_predicted_label = np.zeros_like(full_example.label, dtype=np.int32)
for batch in self.batches_of_patches_with_position(full_example):
images = torch.stack([example_with_position.image for example_with_position in batch])
network = self.DNN
predicted_labels, predicted_counts = network(images)
for example_index, example_with_position in enumerate(batch):
predicted_label = predicted_labels[example_index].detach().numpy()
predicted_count = predicted_counts[example_index].detach().numpy()
x, y = example_with_position.x, example_with_position.y
predicted_label_sum = np.sum(predicted_label)
predicted_label = scipy.misc.imresize(predicted_label, (patch_size, patch_size), mode='F')
unnormalized_predicted_label_sum = np.sum(predicted_label)
if unnormalized_predicted_label_sum != 0:
predicted_label = predicted_label * predicted_label_sum / unnormalized_predicted_label_sum
predicted_count_array = predicted_label * predicted_count / unnormalized_predicted_label_sum
else:
predicted_label = predicted_label
predicted_count_array = np.full(predicted_label.shape, predicted_count / predicted_label.size)
half_patch_size = patch_size // 2
y_start_offset = 0
if y - half_patch_size < 0:
y_start_offset = half_patch_size - y
y_end_offset = 0
if y + half_patch_size >= full_example.label.shape[0]:
y_end_offset = y + half_patch_size - full_example.label.shape[0]
x_start_offset = 0
if x - half_patch_size < 0:
x_start_offset = half_patch_size - x
x_end_offset = 0
if x + half_patch_size >= full_example.label.shape[1]:
x_end_offset = x + half_patch_size - full_example.label.shape[1]
sum_density_label[y - half_patch_size + y_start_offset:y + half_patch_size - y_end_offset,
x - half_patch_size + x_start_offset:x + half_patch_size - x_end_offset
] += predicted_label[y_start_offset:predicted_label.shape[0] - y_end_offset,
x_start_offset:predicted_label.shape[1] - x_end_offset]
sum_count_label[y - half_patch_size + y_start_offset:y + half_patch_size - y_end_offset,
x - half_patch_size + x_start_offset:x + half_patch_size - x_end_offset
] += predicted_count_array[y_start_offset:predicted_count_array.shape[0] - y_end_offset,
x_start_offset:predicted_count_array.shape[1] - x_end_offset]
hit_predicted_label[y - half_patch_size + y_start_offset:y + half_patch_size - y_end_offset,
x - half_patch_size + x_start_offset:x + half_patch_size - x_end_offset
] += 1
hit_predicted_label[hit_predicted_label == 0] = 1
full_predicted_label = sum_density_label / hit_predicted_label.astype(np.float32)
full_predicted_count = np.sum(sum_count_label / hit_predicted_label.astype(np.float32))
totals['count'] += full_example.label.sum()
totals['count_error'] += np.abs(full_predicted_count - full_example.label.sum())
totals['density_error'] += np.abs(full_predicted_label - full_example.label).sum()
totals['density_sum_error'] += np.abs(full_predicted_label.sum() - full_example.label.sum())
totals['predicted_count'] += full_predicted_count
totals['predicted_density_sum'] += full_predicted_label.sum()
for key, value in totals.items():
print('Total {}: {}'.format(key, value))