def predict(self, image_path):
image, image_height, image_width = self.get_image(image_path)
image = image.unsqueeze(0)
sem_seg_prediction, ins_seg_prediction, n_objects_prediction = \
self.model.predict(image)
sem_seg_prediction = sem_seg_prediction.squeeze(0)
ins_seg_prediction = ins_seg_prediction.squeeze(0)
n_objects_prediction = n_objects_prediction.squeeze(0)
sem_seg_prediction, ins_seg_prediction, \
n_objects_prediction = self.cluster(sem_seg_prediction,
ins_seg_prediction,
n_objects_prediction)
sem_seg_prediction = self.upsample_prediction(
sem_seg_prediction, image_height, image_width)
ins_seg_prediction = self.upsample_prediction(
ins_seg_prediction, image_height, image_width)
raw_image_pil = ImageUtilities.read_image(image_path)
raw_image = np.array(raw_image_pil)
return raw_image, sem_seg_prediction, ins_seg_prediction, \
n_objects_prediction
def upsample_prediction(self, prediction, image_height, image_width):
#assert len(prediction.size()) == 4 # n, c, h, w
#return nn.UpsamplingNearest2d((image_height, image_width))(prediction)
resizer = ImageUtilities.image_resizer(image_height, image_width, interpolation=Image.NEAREST)
return resizer(prediction)
def predict(self, image_path):
image, image_height, image_width = self.get_image(image_path)
image = image.unsqueeze(0)
sem_seg_prediction, ins_seg_prediction, n_objects_prediction = \
self.model.predict(image)
sem_seg_prediction = sem_seg_prediction.squeeze(0)
ins_seg_prediction = ins_seg_prediction.squeeze(0)
n_objects_prediction = n_objects_prediction.squeeze(0)
sem_seg_prediction, ins_seg_prediction, \
n_objects_prediction = self.cluster(sem_seg_prediction,
ins_seg_prediction,
n_objects_prediction)
sem_seg_prediction = self.upsample_prediction(sem_seg_prediction,
image_height,
image_width)
ins_seg_prediction = self.upsample_prediction(ins_seg_prediction,
image_height,
image_width)
raw_image_pil = ImageUtilities.read_image(image_path)
raw_image = np.array(raw_image_pil)
return raw_image, sem_seg_prediction, ins_seg_prediction, \
n_objects_prediction
def __init__(self, resize_height, resize_width, mean, std, use_coordinates,
model, n_workers):
self.normalizer = ImageUtilities.image_normalizer(mean, std)
self.use_coordinates = use_coordinates
self.resize_height = resize_height
self.resize_width = resize_width
self.model = model
self.n_workers = n_workers
self.img_resizer = ImageUtilities.image_resizer(
self.resize_height, self.resize_width)
if self.use_coordinates:
self.coordinate_adder = AddCoordinates(with_r=True, usegpu=False)
def __init__(self, resize_height, resize_width, mean,
std, use_coordinates, model, n_workers):
self.normalizer = ImageUtilities.image_normalizer(mean, std)
self.use_coordinates = use_coordinates
self.resize_height = resize_height
self.resize_width = resize_width
self.model = model
self.n_workers = n_workers
self.img_resizer = ImageUtilities.image_resizer(
self.resize_height, self.resize_width)
if self.use_coordinates:
self.coordinate_adder = ImageUtilities.coordinate_adder(
self.resize_height, self.resize_width)
def get_image(self, image_path):
img = ImageUtilities.read_image(image_path)
image_width, image_height = img.size
img = self.img_resizer(img)
img = self.normalizer(img)
return img, image_height, image_width
def get_image(self, image_path):
img = ImageUtilities.read_image(image_path)
image_width, image_height = img.size
img = self.img_resizer(img)
img = self.normalizer(img)
return img, image_height, image_width
def get_image(self, image_path):
img = ImageUtilities.read_image(image_path)
image_width, image_height = img.size
#assert image_height >= image_width
#image_ar = float(image_height) / image_width
#new_height = int(round(self.resize_width * image_ar))
#new_width = int(round(self.resize_width))
new_height = self.resize_height
new_width = self.resize_width
resizer = ImageUtilities.image_resizer(new_height, new_width)
img = resizer(img)
img = self.normalizer(img)
return img, image_height, image_width
def cluster(self, sem_seg_prediction, ins_seg_prediction,
n_objects_prediction):
seg_height, seg_width = ins_seg_prediction.shape[1:]
sem_seg_prediction = sem_seg_prediction.cpu().numpy()
sem_seg_prediction = sem_seg_prediction.argmax(0).astype(np.uint8)
embeddings = ImageUtilities.coordinate_adder(
seg_height, seg_width)(ins_seg_prediction)
embeddings = embeddings.cpu().numpy()
embeddings = embeddings.transpose(1, 2, 0) # h, w, c
n_objects_prediction = n_objects_prediction.cpu().numpy()[0]
embeddings = np.stack([
embeddings[:, :, i][sem_seg_prediction != 0]
for i in range(embeddings.shape[2])
],
axis=1)
clustering = SpectralClustering(n_clusters=n_objects_prediction,
eigen_solver=None,
random_state=None,
n_init=10,
gamma=1.0,
affinity='rbf',
n_neighbors=10,
eigen_tol=0.0,
assign_labels='discretize',
degree=3,
coef0=1,
kernel_params=None,
n_jobs=self.n_workers).fit(embeddings)
labels = clustering.labels_
instance_mask = np.zeros((seg_height, seg_width), dtype=np.uint8)
fg_coords = np.where(sem_seg_prediction != 0)
for si in range(len(fg_coords[0])):
y_coord = fg_coords[0][si]
x_coord = fg_coords[1][si]
_label = labels[si] + 1
instance_mask[y_coord, x_coord] = _label
return sem_seg_prediction, instance_mask, n_objects_prediction
def predict(self, image_path):
image, image_height, image_width = self.get_image(image_path)
image = image.unsqueeze(0)
prediction = self.model.predict(image)
#prediction = self.upsample_prediction(prediction, image_height, image_width)
prediction = prediction.squeeze(0)
prediction = prediction.data.cpu().numpy()
prediction = prediction.argmax(0).astype(np.uint8) #TODO: max 255 classes
prediction_pil = Image.fromarray(prediction)
prediction_pil = self.upsample_prediction(prediction_pil, image_height, image_width)
image_pil = ImageUtilities.read_image(image_path)
#prediction_np = prediction.data.cpu().numpy()
return image_pil, prediction_pil
def predict(self, image_path, n_samples=1):
image, image_height, image_width = self.get_image(image_path)
image = image.unsqueeze(0)
softmax = nn.Softmax2d()
for n in range(n_samples):
prediction = softmax(self.model.predict(image))
print(np.size(prediction))
#prediction = self.upsample_prediction(prediction, image_height, image_width)
prediction = prediction.squeeze(0)
prediction = prediction.data.cpu().numpy()
if n == 0:
mean = prediction
variance = prediction * 0.0
else:
delta = prediction - mean
mean += delta / float(n + 1)
delta2 = prediction - mean
variance += delta * delta2
prediction = mean
variance = variance / float(n_samples)
prediction = prediction.argmax(0).astype(
np.uint8) #TODO: max 255 classes
variance_scaled = variance.sum(0) / variance.max() * 255.0
variance_scaled = variance_scaled.astype(np.uint8)
if np.sum(prediction) == 0:
print('HELP!!!')
prediction_pil = Image.fromarray(prediction)
prediction_pil = self.upsample_prediction(prediction_pil, image_height,
image_width)
variance_pil = Image.fromarray(variance_scaled)
variance_pil = self.upsample_prediction(variance_pil, image_height,
image_width)
image_pil = ImageUtilities.read_image(image_path)
#prediction_np = prediction.data.cpu().numpy()
return image_pil, prediction_pil, variance_pil, variance
def __init__(self, mode, labels, mean, std, image_size_height, image_size_width, annotation_size_height, annotation_size_width,
crop_scale, crop_ar, random_cropping=True, horizontal_flipping=True, random_jitter=True):
self._mode = mode
assert self._mode in ['training', 'test']
self.n_classes = len(labels)
self.mean = mean
self.std = std
self.image_size_height = image_size_height
self.image_size_width = image_size_width
self.random_cropping = random_cropping
self.crop_scale = crop_scale
self.crop_ar = crop_ar
self.annotation_size_height = annotation_size_height
self.annotation_size_width = annotation_size_width
self.horizontal_flipping = horizontal_flipping
self.random_jitter = random_jitter
if self._mode == 'training':
if self.random_cropping:
self.image_random_cropper = ImageUtilities.image_random_cropper_and_resizer(self.image_size_height, self.image_size_width)
self.annotation_random_cropper = ImageUtilities.image_random_cropper_and_resizer(self.annotation_size_height,
self.annotation_size_width,
interpolation=Image.NEAREST)
else:
self.image_resizer = ImageUtilities.image_resizer(self.image_size_height, self.image_size_width)
self.annotation_resizer = ImageUtilities.image_resizer(self.annotation_size_height, self.annotation_size_width,
interpolation=Image.NEAREST)
if self.horizontal_flipping:
self.horizontal_flipper = ImageUtilities.image_random_horizontal_flipper()
if self.random_jitter:
self.random_jitterer = transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.1)
else:
self.image_resizer = ImageUtilities.image_resizer(self.image_size_height, self.image_size_width)
self.annotation_resizer = ImageUtilities.image_resizer(self.annotation_size_height, self.annotation_size_width,
interpolation=Image.NEAREST)
self.image_normalizer = ImageUtilities.image_normalizer(self.mean, self.std)
def cluster(self, sem_seg_prediction, ins_seg_prediction,
n_objects_prediction):
seg_height, seg_width = ins_seg_prediction.shape[1:]
sem_seg_prediction = sem_seg_prediction.cpu().numpy()
sem_seg_prediction = sem_seg_prediction.argmax(0).astype(np.uint8)
embeddings = ImageUtilities.coordinate_adder(
seg_height, seg_width)(ins_seg_prediction)
embeddings = embeddings.cpu().numpy()
embeddings = embeddings.transpose(1, 2, 0) # h, w, c
n_objects_prediction = n_objects_prediction.cpu().numpy()[0]
embeddings = np.stack([embeddings[:, :, i][sem_seg_prediction != 0]
for i in range(embeddings.shape[2])], axis=1)
clustering = SpectralClustering(n_clusters=n_objects_prediction,
eigen_solver=None, random_state=None,
n_init=10, gamma=1.0, affinity='rbf',
n_neighbors=10, eigen_tol=0.0,
assign_labels='discretize', degree=3,
coef0=1,
kernel_params=None,
n_jobs=self.n_workers).fit(embeddings)
labels = clustering.labels_
instance_mask = np.zeros((seg_height, seg_width), dtype=np.uint8)
fg_coords = np.where(sem_seg_prediction != 0)
for si in range(len(fg_coords[0])):
y_coord = fg_coords[0][si]
x_coord = fg_coords[1][si]
_label = labels[si] + 1
instance_mask[y_coord, x_coord] = _label
return sem_seg_prediction, instance_mask, n_objects_prediction
def __init__(self,
mode,
n_classes,
max_n_objects,
mean,
std,
image_height,
image_width,
random_hor_flipping=True,
random_ver_flipping=True,
random_transposing=True,
random_90x_rotation=True,
random_rotation=True,
random_color_jittering=True,
random_grayscaling=True,
random_channel_swapping=True,
random_gamma=True,
random_resolution=True):
self._mode = mode
self.n_classes = n_classes
self.max_n_objects = max_n_objects
assert self._mode in ['training', 'test']
self.mean = mean
self.std = std
self.image_height = image_height
self.image_width = image_width
self.random_horizontal_flipping = random_hor_flipping
self.random_vertical_flipping = random_ver_flipping
self.random_transposing = random_transposing
self.random_90x_rotation = random_90x_rotation
self.random_rotation = random_rotation
self.random_color_jittering = random_color_jittering
self.random_grayscaling = random_grayscaling
self.random_channel_swapping = random_channel_swapping
self.random_gamma = random_gamma
self.random_resolution = random_resolution
if self._mode == 'training':
if self.random_horizontal_flipping:
self.horizontal_flipper = IU.image_random_horizontal_flipper()
if self.random_vertical_flipping:
self.vertical_flipper = IU.image_random_vertical_flipper()
if self.random_transposing:
self.transposer = IU.image_random_transposer()
if self.random_rotation:
self.image_rotator = IU.image_random_rotator(random_bg=True)
self.annotation_rotator = IU.image_random_rotator(
Image.NEAREST, random_bg=False)
if self.random_90x_rotation:
self.image_rotator_90x = IU.image_random_90x_rotator()
self.annotation_rotator_90x = IU.image_random_90x_rotator(
Image.NEAREST)
if self.random_color_jittering:
self.color_jitter = IU.image_random_color_jitter(
brightness=0.4, contrast=0.4, saturation=0.4, hue=0.2)
if self.random_grayscaling:
self.grayscaler = IU.image_random_grayscaler(p=0.3)
if self.random_channel_swapping:
self.channel_swapper = IU.image_random_channel_swapper(p=0.5)
if self.random_gamma:
self.gamma_adjuster = IU.image_random_gamma([0.7, 1.3], gain=1)
if self.random_resolution:
self.resolution_degrader = IU.image_random_resolution(
[0.7, 1.3])
self.img_resizer = IU.image_resizer(self.image_height,
self.image_width)
self.ann_resizer = IU.image_resizer(self.image_height,
self.image_width,
interpolation=Image.NEAREST)
else:
self.img_resizer = IU.image_resizer(self.image_height,
self.image_width)
self.ann_resizer = IU.image_resizer(self.image_height,
self.image_width,
interpolation=Image.NEAREST)
self.image_normalizer = IU.image_normalizer(self.mean, self.std)
def get_annotation(self, annotation_path):
img = ImageUtilities.read_image(annotation_path)
return img
def __init__(self, resize_height, resize_width, mean, std, model):
self.normalizer = ImageUtilities.image_normalizer(mean, std)
self.resize_height = resize_height
self.resize_width = resize_width
self.model = model
def get_annotation(self, annotation_path):
img = ImageUtilities.read_image(annotation_path)
return img
def __init__(self,
mode,
n_classes,
max_n_objects,
mean,
std,
image_height,
image_width,
random_hor_flipping=True,
random_ver_flipping=True,
random_90x_rotation=True,
random_rotation=True,
random_color_jittering=True,
use_coordinates=False):
self._mode = mode
self.n_classes = n_classes
self.max_n_objects = max_n_objects
assert self._mode in ['training', 'test']
self.mean = mean
self.std = std
self.image_height = image_height
self.image_width = image_width
self.random_horizontal_flipping = random_hor_flipping
self.random_vertical_flipping = random_ver_flipping
self.random_90x_rotation = random_90x_rotation
self.random_rotation = random_rotation
self.random_color_jittering = random_color_jittering
self.use_coordinates = use_coordinates
if self._mode == 'training':
if self.random_horizontal_flipping:
self.horizontal_flipper = IU.image_random_horizontal_flipper()
if self.random_vertical_flipping:
self.vertical_flipper = IU.image_random_vertical_flipper()
if self.random_90x_rotation:
self.rotator_90x = IU.image_random_90x_rotator()
if self.random_rotation:
self.rotator = IU.image_random_rotator(expand=True)
if self.random_color_jittering:
self.color_jitter = IU.image_random_color_jitter(
brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1)
self.img_resizer = IU.image_resizer(self.image_height,
self.image_width)
self.ann_resizer = IU.image_resizer(self.image_height,
self.image_width,
interpolation=Image.NEAREST)
else:
self.img_resizer = IU.image_resizer(self.image_height,
self.image_width)
self.ann_resizer = IU.image_resizer(self.image_height,
self.image_width,
interpolation=Image.NEAREST)
self.image_normalizer = IU.image_normalizer(self.mean, self.std)
if self.use_coordinates:
self.coordinate_adder = IU.coordinate_adder(
self.image_height, self.image_width)