def test_is_from_grayscale():
"""TGS salt dataset is from grayscale"""
path = './data/TGS_salt/test/images'
images, names, ids = collect_images(path)
for i in range(len(images)):
print(i)
image = images[i]
c1 = image[:, :, 0]
c2 = image[:, :, 1]
c3 = image[:, :, 2]
if not np.array_equal(c1, c2) or not np.array_equal(
c2, c3) or not np.array_equal(c3, c1):
print('not equal')
break
path = './data/TGS_salt/train/images'
images, names, ids = collect_images(path)
for i in range(len(images)):
print(i)
image = images[i]
c1 = image[:, :, 0]
c2 = image[:, :, 1]
c3 = image[:, :, 2]
if not np.array_equal(c1, c2) or not np.array_equal(
c2, c3) or not np.array_equal(c3, c1):
print('not equal')
break
def filter_white_image(self):
limit = None
print(f'collect train images')
images, _, ids = collect_images(TRAIN_IMAGE_PATH, limit=limit)
images = np_img_gray_to_rgb(images)
print(f'collect train masks images')
masks, _, _ = collect_images(TRAIN_MASK_PATH, limit=limit)
masks = masks.reshape([-1, 101, 101])
masked = []
mean = []
for image, mask, id_ in zip(images, masks, ids):
a = np.sum(image) / (101 * 101 * 3 * 255)
print(id_, a)
if a > 0.85:
masked += [data_helper.masking_images(image, mask)]
mean += [a]
print(len(masked))
print(np.mean(mean), np.std(mean))
pprint(mean)
masked = np.array(masked)
plot.plot_image_tile(masked, title='almost_white')
def pca_cluster_image(self):
# not good...
from sklearn.preprocessing import MinMaxScaler
limit = 500
print(f'collect train images')
images, _, ids = collect_images(TRAIN_IMAGE_PATH, limit=limit)
pca = PCA(n_components=2)
pca.fit(images.reshape([-1, 101 * 101]))
vector = pca.transform(images.reshape([-1, 101 * 101]))
scaler = MinMaxScaler()
scaler.fit(vector)
vector = scaler.transform(vector)
print(vector, vector.shape)
x = vector[:, 0]
y = vector[:, 1]
plot.scatter_2d(vector, title='pca_cluster')
# images = np_img_gray_to_rgb(x, y)
cluster_image = np_img_to_img_scatter(images, vector, 5000, 5000)
cluster_image = np_img_gray_to_rgb(cluster_image)
plot.plot_image(cluster_image, title='pca_cluster_images')
# TODO
pass
def test_TGS_salt_metric():
print(f'collect train mask images')
train_mask_images, _, _ = collect_images(TRAIN_MASK_PATH)
train_mask_images = train_mask_images.reshape([-1, 101, 101])
idx = 10
size = 10
images = train_mask_images[idx:idx + size]
metric_images = []
for a in images:
for b in images:
a = a.reshape([101, 101, 1])
b = b.reshape([101, 101, 1])
zero_channel = np.zeros([101, 101, 1])
rgb_image = np.concatenate([a, b, zero_channel], axis=2)
metric_images += [rgb_image]
metric_images = np.array(metric_images)
plot.plot_image_tile(metric_images, title='test_metric', column=size)
metric_score = []
for a in images:
for b in images:
metric_score += [Metrics.TGS_salt_score(a, b)]
metric_score = np.array(metric_score).reshape([size, size])
print(metric_score)
def plot_test_image(self):
print(f'collect test images')
test_images, _, _ = collect_images(TEST_IMAGE_PATH)
test_images = np_img_gray_to_rgb(test_images)
for i in range(0, len(test_images), 500):
print(i)
plot.plot_image_tile(test_images[i: i + 500], title=f'test_{i}', column=20)
def test_train_dataset_callback():
sample_IMAGE_PATH = path_join(HEAD_PATH, 'sample/images')
sample_MASK_PATH = path_join(HEAD_PATH, 'sample/masks')
sample_size = 7
limit = None
print(f'collect sample images')
train_images, _, _ = collect_images(sample_IMAGE_PATH, limit=limit)
train_images = train_images.reshape([-1, 101, 101])
print(f'collect sample images')
train_mask_images, _, _ = collect_images(sample_MASK_PATH, limit=limit)
train_mask_images = train_mask_images.reshape([-1, 101, 101])
x = train_images
y = train_mask_images
import cv2
x = np.array([cv2.resize(a, (128, 128)) for a in x]).reshape([-1, 128, 128, 1])
y = np.array([cv2.resize(a, (128, 128)) for a in y]).reshape([-1, 128, 128, 1])
y_gt = y
y_encode = mask_label_encoder.to_label(y)
print(x.shape)
print(y_encode.shape)
Unet = SemanticSegmentation(stage=4, batch_size=7)
# Unet.train(x, y_encode, epoch=100)
Unet.train(x, y_encode, epoch=1000, dataset_callback=dataset_callback)
Unet.train(x, y_encode, epoch=1000, dataset_callback=dataset_callback)
score = Unet.score(x, y_encode)
pprint(score)
predict = Unet.predict(x)
pprint(predict[0])
pprint(predict.shape)
proba = Unet.predict_proba(x)
pprint(proba[0])
pprint(proba.shape)
metric = Unet.metric(x, y_encode)
print(metric)
predict = mask_label_encoder.from_label(predict)
plot.plot_image_tile(np.concatenate([x, predict, y_gt], axis=0), title='predict', column=sample_size)
def test_RL_encoding():
images, _, _ = collect_images(TRAIN_MASK_PATH)
images = np.transpose(images, (0, 2, 1, 3))
images = np.reshape(images, [-1, 101, 101])
@deco_timeit
def encoding_timeit(images):
return RLE_mask_encoding(images)
encoded = encoding_timeit(images)
def plot_train_image_with_mask(self):
limit = None
print(f'collect train images')
train_images, _, _ = collect_images(TRAIN_IMAGE_PATH, limit=limit)
train_images = train_images.reshape([-1, 101, 101])
print(f'collect train mask images')
train_mask_images, _, _ = collect_images(TRAIN_MASK_PATH, limit=limit)
train_mask_images = train_mask_images.reshape([-1, 101, 101])
train_images = np_img_gray_to_rgb(train_images)
masked_images = []
for image, mask in zip(train_images, train_mask_images):
masked_images += [data_helper.masking_images(image, mask)]
masked_images = np.array(masked_images)
plot.plot_image_tile(masked_images[:500], title='masked_0', column=20)
plot.plot_image_tile(masked_images[500:1000], title='masked_1', column=20)
plot.plot_image_tile(masked_images[1000:1500], title='masked_2', column=20)
plot.plot_image_tile(masked_images[1500:2000], title='masked_3', column=20)
def chopped_mask_image(self):
limit = None
print(f'collect train images')
images, _, ids = collect_images(TRAIN_IMAGE_PATH, limit=limit)
images = np_img_gray_to_rgb(images)
print(f'collect train masks images')
masks, _, _ = collect_images(TRAIN_MASK_PATH, limit=limit)
masks = masks.reshape([-1, 101, 101])
masked = []
for image, mask, id_ in zip(images, masks, ids):
rle_mask = RLE_mask_encoding(mask.reshape([101, 101]).transpose())
n_rle_mask = len(rle_mask)
a = n_rle_mask
mask_area = np.sum(mask) / (101 * 101 * 255)
if 0 < a / 2 < 8 and 0.1 < mask_area < 0.99:
print(id_, a, rle_mask)
masked += [data_helper.masking_images(image, mask)]
masked = np.array(masked)
print(len(masked))
plot.plot_image_tile(masked, title='chopped')
def tsne_cluster_image(self):
# not good...
from sklearn.preprocessing import MinMaxScaler
print(f'collect train images')
limit = 500
images, _, ids = collect_images(TRAIN_IMAGE_PATH, limit=limit)
print('fit transform')
tsne = TSNE()
tsne.fit(images.reshape([-1, 101 * 101]))
vector = tsne.fit_transform(images.reshape([-1, 101 * 101]))
scaler = MinMaxScaler()
scaler.fit(vector)
vector = scaler.transform(vector)
print(vector, vector.shape)
plot.scatter_2d(vector, title='tsne_cluster')
cluster_image = np_img_to_img_scatter(images, vector, 5000, 5000)
cluster_image = np_img_gray_to_rgb(cluster_image)
plot.plot_image(cluster_image, title='tsne_cluster_images')