def __init__(self, path):
mndata = MNIST(path)
images_train, self.labels_train = mndata.load_training()
images_test, self.labels_test = mndata.load_testing()
self.images_train_norm = normalize_images(images_train)
self.images_test_norm = normalize_images(images_test)
self.img_size = len(images_train[0])
self.outputs = 10
def test(self):
# Import test images
test_input_function = _get_input_fn(filenames=str(self.test_data_path),
epochs=1,
batchsize=32,
shuffle=1,
shape=self.input_shape)
test_images = next(iter(test_input_function))
test_images = test_images[0].numpy()
test_images = normalize_images(test_images)
# Results of Network
holograms = self.get_hologram(test_images)
multi_planes = self._prop_to_planes(holograms).numpy()
multi_planes = normalize_images(multi_planes)
mse_cnn = mean_sq_errs(multi_planes, test_images)
ssim_cnn = tf.image.ssim(tf.cast(multi_planes, tf.float32),
tf.cast(test_images, tf.float32),
max_val=1)
acc = self._acc(test_images, multi_planes)
# Save results
result_save = tqdm(range(test_images.shape[0]))
result_save.set_description("Saving test results...")
test_result_dir = self.path / "test_results" / self.model_name
if not os.path.isdir(test_result_dir):
os.mkdir(test_result_dir)
for image_index in result_save:
path = self.path / "test_results" / self.model_name / f"image_{image_index}.mat"
test = {
f"test_z{image_plane}": test_images[image_index, :, :,
image_plane]
for image_plane in range(test_images.shape[-1])
}
predict = {
f"pred_z{image_plane}": multi_planes[image_index, :, :,
image_plane]
for image_plane in range(test_images.shape[-1])
}
test_result = {
"test": test,
"predict": predict,
"mse": mse_cnn[image_index],
"ssim": ssim_cnn.numpy()[image_index],
"acc": acc.numpy()[image_index]
}
scipy.io.savemat(path, test_result)
scipy.io.savemat(test_result_dir / "errors.mat", {
"mse": mse_cnn,
"ssim": ssim_cnn.numpy(),
"acc": acc.numpy()
})
def load_images_from_numpy_records(self, start_idx, end_idx):
"""Loads images from numpy records
Args:
start_idx: First index in the list
end_idx : End index in the list
Returns:
Images as pairs
"""
images_A = utils.normalize_images(self.t_images_data[start_idx:end_idx, 0].astype(np.float32))
images_B = utils.normalize_images(self.t_images_data[start_idx:end_idx, 1].astype(np.float32))
return images_A, images_B
def load_images_from_files(self, start_idx, end_idx):
"""Loads images by reading images from files
Args:
start_idx: First index in the list
end_idx : End index in the list
Returns:
Images as pairs
"""
img_dim = self.opts.h
images_A = np.empty([self.opts.batch_size, img_dim, img_dim, 3], dtype=np.float32)
images_B = np.empty([self.opts.batch_size, img_dim, img_dim, 3], dtype=np.float32)
for idx, path in enumerate(self.t_image_paths[start_idx:end_idx]):
# TODO: Generalize this method to load test/val images
path = os.path.join(self.opts.dataset_dir, self.opts.dataset, 'train', path)
try:
image = utils.imread(path)
image = utils.normalize_images(images=image)
except IOError:
raise IOError("Cannot read the image {}" % path)
split_len = 600 if self.opts.dataset == 'maps' else 256
images_A[idx] = image[:, :split_len, :]
images_B[idx] = image[:, split_len:, :]
return images_A, images_B
def _preprocess_input_data(self, data_splits):
train_data, validation_data, test_data = data_splits
train_images, train_classes = train_data
validation_images, validation_classes = validation_data
test_images, test_classes = test_data
# encode outputs into one hot vectors
class_arg_splits = (train_classes, validation_classes, test_classes)
categorical_data = [
to_categorical(class_arg_split)
for class_arg_split in class_arg_splits
]
train_classes, validation_classes, test_classes = categorical_data
# flatten, add salt/pepper noise and normalize images
image_splits = (train_images, validation_images, test_images)
image_splits = [flatten(image_split) for image_split in image_splits]
image_splits = [
spice_up_images(image_split) for image_split in image_splits
]
image_splits = [
normalize_images(image_split) for image_split in image_splits
]
train_images, validation_images, test_images = image_splits
train_data = (train_images, train_classes)
validation_data = (validation_images, validation_classes)
test_data = (test_images, test_classes)
preprocessed_data_split = (train_data, validation_data, test_data)
return preprocessed_data_split
def make_grid(self, *args):
samples = self.model.generate_samples(self.latents, training=False)
if self.show_blurred_samples:
samples = self.model.blur(samples)
samples = utils.normalize_images(samples)
figure = utils.samples_grid(samples) # TODO: write figure to a file?
figure.savefig(self.log_dir + f"/samples_grid_{self.samples_seen:06}.png")
image = utils.plot_to_image(figure)
with self.model.summary_writer.as_default():
tf.summary.image("samples_grid", image)
def test_model(model):
cap = cv2.VideoCapture(0)
while (True):
ret, image = cap.read()
cv2.imshow("capture", image)
ret = model.predict(normalize_images([image]))[0]
ret_index = np.argmax(ret)
print(ret_index, GESTURE_TYPE[ret_index], ret[ret_index], ret)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
def predict(images):
labels = []
solver = Solver()
# process image & predict if it's an image or doc
images = utils.process_images(images)
is_image = np.round(
solver.doc_image_classifier.predict(images)) == utils.IMAGE
images = utils.normalize_images(images, is_img=is_image)
# predict labels
labels = solver.predict(images, is_image)
return labels
def load_val_batch(self):
"""Loads validation images"""
img_dim = self.opts.h
images_A = np.empty([self.opts.batch_size, img_dim, img_dim, 3], dtype=np.float32)
images_B = np.empty([self.opts.batch_size, img_dim, img_dim, 3], dtype=np.float32)
np.random.shuffle(self.v_image_paths)
for idx, path in enumerate(self.v_image_paths[:self.opts.sample_num]):
path = os.path.join(self.opts.dataset_dir, self.opts.dataset, 'val', path)
try:
image = utils.imread(path)
image = utils.normalize_images(images=image)
except IOError:
raise IOError("Cannot read the image {}" % path)
split_len = 600 if self.opts.dataset == 'maps' else 256
images_A[idx] = image[:, :split_len, :]
images_B[idx] = image[:, split_len:, :]
return images_A, images_B
def evaluate(self, paths):
total = 0
score = 0.0
for path in paths:
images = utils.get_images_from_path(path)
images = utils.process_images(images)
images = utils.normalize_images(images, not self.is_docs)
images = np.asarray(images)
tiles = np.sqrt(images.shape[0])
tile_pairs = utils.tile_neighbors_lr[tiles] if not self.is_docs \
else utils.tile_neighbors_ud[tiles]
non_neighbor_pairs = utils.non_neighbor_tiles_lr[tiles] if not self.is_docs \
else utils.non_neighbor_tiles_ud[tiles]
total += len(tile_pairs) + len(non_neighbor_pairs)
for i1, i2 in tile_pairs:
pred = self.predict(images[i1], images[i2])
score += (pred > 0.5)
for i1, i2 in non_neighbor_pairs:
pred = self.predict(images[i1], images[i2])
score += (pred < 0.5)
return score / total
def augment_images(images,
masks=None,
num_aug=1,
do_composite=True,
output_dir=None,
AugSettings=None):
"""Augment images and/or masks.
:param images: Images to augment;
NumPy array of shape (H,W), (H,W,3), (N,H,W), or (N,H,W,3);
values can be in range [0,1], [0,2^8-1], or [0,2^16-1]
:param masks:
(Optional) Masks to correspondingly augment;
NumPy array of shape (H,W) or (N,H,W);
values are 0 and positive integers
:param num_aug:
(Optional) Number of augmentations to perform;
number of output images will be N * num_aug
:type num_aug: int.
:param do_composite:
(Optional) Whether to do composite augmentations (multiple augmentations at once; True) or individual augmentations (for observing the effect of each augmentation; False)
:type do_composite: bool.
:param output_dir:
(Optional) If not set to None, location where .tif images should be saved for observation purposes;
if set to None, no saving will be done
:type output_dir: str.
:param AugSettings:
(Optional) Object of the class that inherists from AugmentationSettings specifying the augmentation parameters and sequences to apply; whether composite or individual augmentation is determined by the do_composite parameter
if set to None, the default base class TestAugment, defined in this module, is used;
to customize the augmentation parameters only, instantiate from TestAugment and modify the instance variables;
to customize the sequences as well, inherit a custom derived class from AugmentationSettings and override the composite_sequence and/or individual_seqs_and_outnames methods
:type AugSettingsClass: cls.
:returns:
* If do_composite=True: augmented images ((N,H,W,C)), and, if masks were input, augmented masks ((N,H,W)); these are both NumPy arrays of dtype='uint8'\n
* If do_composite=False: list of augmented images ((N,H,W,C)), one for each individual augmentation; these are all NumPy arrays of dtype='uint8'\n
"""
# Import relevant modules
import imgaug as ia
from imgaug import augmenters as iaa
import numpy as np
from skimage import io
import utils
#from . import utils
print(type(TestAugment))
# Instantiate from the input augmentation settings class, whether a base class or derived
if AugSettings == None:
aug_settings = TestAugment()
else:
aug_settings = AugSettings
import os
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
# Preprocess the images and masks (note that masks in imgaug are assumed to be integer-based, bool, or float-based; we're settling on int16)
images = utils.normalize_images(
images, 1
) # ensure the images are uint8; inputs can therefore be [0,1], [0,2^8-1], or [0,2^16-1]
images, masks = utils.stack_and_color_images(
images,
masks=masks) # allow for flexible dimensions of input images and masks
# If multiple augmentations are desired, set up imgaug to do the multiple augmentations
# images is (N,H,W,C) and masks is (N,H,W) due to the stack_and_color_images() function called above
images = np.tile(images, (num_aug, 1, 1, 1))
if masks is not None:
masks = np.tile(masks, (num_aug, 1, 1))
# Get the shapes of the images and masks
N, H, W, C = images.shape
# Optionally save .tif files to put the input in the same format as the output in order to do visual sanity checks
if output_dir is not None:
io.imsave(output_dir + '/' + 'input.tif', images)
if masks is not None:
utils.quick_overlay_output(images, masks,
output_dir + '/' + 'input-overlay.tif')
# If we want to do each augmentation individually in order to manually inspect exactly what each augmentation does...
if not do_composite:
# For each augmentation...
image_aug_list = []
for aug_num, seq_and_outname in enumerate(
aug_settings.individual_seqs_and_outnames(), start=1):
seq = seq_and_outname[0]
outname = '{:02d}'.format(aug_num) + '-' + seq_and_outname[1]
# For each image in the stack...
image_aug = []
for i in range(N):
seq_det = seq.to_deterministic()
img_channels = []
for ic in range(C):
img_channels.append(
seq_det.augment_image(images[i, :, :, ic].squeeze()))
image_aug.append(img_channels)
# Convert the lists to numpy arrays
image_aug = np.array(image_aug, dtype=np.uint8)
# Transpose the indices since they come out like (N,C,H,W)
image_aug = np.transpose(image_aug,
[0, 2, 3, 1]) # comes out as (N,H,W,C)
# Save the augmented images to a list
image_aug_list.append(image_aug)
# Optionally save the outputs as .tif files so you can check the individual augmentations
if output_dir is not None:
io.imsave(output_dir + '/' + outname + '.tif', image_aug)
# Return the list of numpy arrays
return (image_aug_list)
# If we want to do a composite augmentation...
else:
# Determine if there are masks present
if masks is not None:
do_masks = True
else:
do_masks = False
# Define the arrays to contain the stack for the current single augmentation
image_aug = []
if do_masks:
mask_aug = []
# For each image in the stack...
for i in range(N):
seq_det = aug_settings.composite_sequence().to_deterministic()
img_channels = []
for ic in range(C):
img_channels.append(
seq_det.augment_image(images[i, :, :, ic].squeeze()))
image_aug.append(img_channels)
if do_masks:
#mask_segmap = ia.SegmentationMapOnImage(masks[i,:,:].squeeze(),shape=(H,W),nb_classes=2)
mask_segmap = ia.SegmentationMapOnImage(
masks[i, :, :].squeeze(),
shape=(H, W),
nb_classes=np.max(masks[i, :, :]) + 1)
mask_aug.append(
seq_det.augment_segmentation_maps([mask_segmap
])[0].get_arr_int())
# Convert the lists to numpy arrays
image_aug = np.array(image_aug, dtype=np.uint8)
if do_masks:
mask_aug = np.array(mask_aug,
dtype=np.int16) # comes out as (N,H,W)
# Transpose the indices since they come out like (N,C,H,W)
image_aug = np.transpose(image_aug,
[0, 2, 3, 1]) # comes out as (N,H,W,C)
# Optionally save the output as a .tif file and overlay the masks on the images and save as .tif
if output_dir is not None:
io.imsave(output_dir + '/' + 'output.tif', image_aug)
if do_masks:
utils.quick_overlay_output(
image_aug, mask_aug,
output_dir + '/' + 'output-overlay.tif')
# Return the augmented images (and masks if they were input)
if not do_masks:
return (image_aug)
else:
return (image_aug, mask_aug)
error_4_count = 0
sum_4 = utils.AverageMeter()
sum_16 = utils.AverageMeter()
sum_25 = utils.AverageMeter()
for idx, path in enumerate(X_img_test):
images = list(utils.get_images_from_path(path))
tiles = len(images)
count += tiles
labels = list(range(0, tiles))
images, labels = shuffle(images, labels)
new_labels = list(range(0, tiles))
# process image & predict if it's an image or doc
images = utils.process_images(images)
is_image = True #np.round(solver.doc_image_classifier.predict(images)) == utils.IMAGE
images = utils.normalize_images(images, is_img=is_image)
for i, label in enumerate(labels, 0):
new_labels[label] = i
labels = new_labels
acc, predictions = solver.evaluate(images, labels, is_image)
if acc < 1:
error_counter += 1
sum += acc * tiles
if tiles == 4:
sum_4.update(acc)
if tiles == 16:
sum_16.update(acc)
if tiles == 25:
sum_25.update(acc)
# print('correct probability [{:.3f}], error probability [{:.3f}]'.format(corre_prob.avg, error_prob.avg))
print(
