def plot_some_data():
memmap_name = "patchClassification_ws_resampled"
with open("../data/%s_properties.pkl" % memmap_name, 'r') as f:
memmap_properties = cPickle.load(f)
n_pos_train = memmap_properties["train_pos"]
n_neg_train = memmap_properties["train_neg"]
n_pos_val = memmap_properties["val_pos"]
n_neg_val = memmap_properties["val_neg"]
train_pos_memmap = memmap("../data/%s_train_pos.memmap" % memmap_name,
dtype=np.float32,
mode="r+",
shape=memmap_properties["train_pos_shape"])
train_neg_memmap = memmap("../data/%s_train_neg.memmap" % memmap_name,
dtype=np.float32,
mode="r+",
shape=memmap_properties["train_neg_shape"])
val_pos_memmap = memmap("../data/%s_val_pos.memmap" % memmap_name,
dtype=np.float32,
mode="r+",
shape=memmap_properties["val_pos_shape"])
val_neg_memmap = memmap("../data/%s_val_neg.memmap" % memmap_name,
dtype=np.float32,
mode="r+",
shape=memmap_properties["val_neg_shape"])
i = 0
ctr = 0
for data, seg, labels in memmapGenerator(val_neg_memmap, val_pos_memmap,
128, n_pos_val, n_neg_val):
if i == 2:
break
data2 = np.array(data)
for img, segm, lab in zip(data2, seg, labels):
img -= img.min()
img /= img.max()
plt.figure(figsize=(12, 12))
img = np.array(img[0]) # dont write into memmap
img = np.repeat(img[np.newaxis, :, :], 3, 0)
img = img.transpose((1, 2, 0))
img[:, :, 0][segm[0] > 1] *= 1.0
plt.imshow(img, interpolation='nearest')
if lab == 0:
color = 'green'
else:
color = 'red'
plt.text(0,
0,
lab,
color=color,
bbox=dict(facecolor='white', alpha=1))
plt.savefig("../some_images/img_%04.0f.png" % ctr)
plt.close()
ctr += 1
i += 1
def plot_some_data():
memmap_name = "patchClassification_ws_resampled"
with open("../data/%s_properties.pkl" % memmap_name, 'r') as f:
memmap_properties = cPickle.load(f)
n_pos_train = memmap_properties["train_pos"]
n_neg_train = memmap_properties["train_neg"]
n_pos_val = memmap_properties["val_pos"]
n_neg_val = memmap_properties["val_neg"]
train_pos_memmap = memmap("../data/%s_train_pos.memmap" % memmap_name, dtype=np.float32, mode="r+", shape=memmap_properties["train_pos_shape"])
train_neg_memmap = memmap("../data/%s_train_neg.memmap" % memmap_name, dtype=np.float32, mode="r+", shape=memmap_properties["train_neg_shape"])
val_pos_memmap = memmap("../data/%s_val_pos.memmap" % memmap_name, dtype=np.float32, mode="r+", shape=memmap_properties["val_pos_shape"])
val_neg_memmap = memmap("../data/%s_val_neg.memmap" % memmap_name, dtype=np.float32, mode="r+", shape=memmap_properties["val_neg_shape"])
i = 0
ctr = 0
for data, seg, labels in memmapGenerator(val_neg_memmap, val_pos_memmap, 128, n_pos_val, n_neg_val):
if i == 2:
break
data2 = np.array(data)
for img, segm, lab in zip(data2, seg, labels):
img -= img.min()
img /= img.max()
plt.figure(figsize=(12,12))
img = np.array(img[0]) # dont write into memmap
img = np.repeat(img[np.newaxis, :, :], 3, 0)
img = img.transpose((1, 2, 0))
img[:, :, 0][segm[0] > 1] *= 1.0
plt.imshow(img, interpolation='nearest')
if lab == 0:
color = 'green'
else:
color = 'red'
plt.text(0, 0, lab, color=color, bbox=dict(facecolor='white', alpha=1))
plt.savefig("../some_images/img_%04.0f.png"%ctr)
plt.close()
ctr += 1
i += 1
printLosses(all_training_losses, all_training_accs, all_validation_losses, all_validation_accuracies, "../results/%s.png" % EXPERIMENT_NAME, 10)
loss, acc = train_fn(data, labels)
train_loss += loss
train_loss_tmp += loss
train_acc_tmp += acc
batch_ctr += 1
if batch_ctr > n_batches_per_epoch:
break
train_loss /= n_batches_per_epoch
print "training loss average on epoch: ", train_loss
test_loss = 0
accuracies = []
valid_batch_ctr = 0
for data, seg, labels in threaded_generator(memmapGenerator(val_neg_memmap, val_pos_memmap, BATCH_SIZE, n_pos_val, n_neg_val)):
loss, acc = val_fn(data, labels)
test_loss += loss
accuracies.append(acc)
valid_batch_ctr += 1
if valid_batch_ctr > n_test_batches:
break
test_loss /= n_test_batches
print "test loss: ", test_loss
print "test acc: ", np.mean(accuracies), "\n"
all_validation_losses.append(test_loss)
all_validation_accuracies.append(np.mean(accuracies))
printLosses(all_training_losses, all_training_accs, all_validation_losses, all_validation_accuracies, "../results/%s.png" % EXPERIMENT_NAME, 10)
learning_rate *= 0.5
with open("../results/%s_Params_ep%d.pkl" % (EXPERIMENT_NAME, epoch), 'w') as f:
cPickle.dump(lasagne.layers.get_all_param_values(net['prob']), f)
printLosses(all_training_losses, all_training_accs, all_validation_losses, all_validation_accuracies, "../results/%s.png" % EXPERIMENT_NAME, 10)
loss, acc = train_fn(data, labels)
train_loss += loss
train_loss_tmp += loss
train_acc_tmp += acc
batch_ctr += 1
if batch_ctr > n_batches_per_epoch:
break
train_loss /= n_batches_per_epoch
print "training loss average on epoch: ", train_loss
test_loss = 0
accuracies = []
valid_batch_ctr = 0
for data, seg, labels in threaded_generator(memmapGenerator(val_neg_memmap, val_pos_memmap, BATCH_SIZE, n_pos_val, n_neg_val)):
loss, acc = val_fn(data, labels)
test_loss += loss
accuracies.append(acc)
valid_batch_ctr += 1
if valid_batch_ctr > n_test_batches:
break
test_loss /= n_test_batches
print "test loss: ", test_loss
print "test acc: ", np.mean(accuracies), "\n"
all_validation_losses.append(test_loss)
all_validation_accuracies.append(np.mean(accuracies))
printLosses(all_training_losses, all_training_accs, all_validation_losses, all_validation_accuracies, "../results/%s.png" % EXPERIMENT_NAME, 10)
learning_rate *= 0.3
with open("../results/%s_Params_ep%d.pkl" % (EXPERIMENT_NAME, epoch), 'w') as f:
cPickle.dump(lasagne.layers.get_all_param_values(net), f)
loss, acc = train_fn(data, labels)
train_loss += loss
train_loss_tmp += loss
train_acc_tmp += acc
batch_ctr += 1
if batch_ctr > n_batches_per_epoch:
break
train_loss /= n_batches_per_epoch
print "training loss average on epoch: ", train_loss
test_loss = 0
accuracies = []
valid_batch_ctr = 0
for data, seg, labels in threaded_generator(
memmapGenerator(val_neg_memmap, val_pos_memmap, BATCH_SIZE,
n_pos_val, n_neg_val)):
loss, acc = val_fn(data, labels)
test_loss += loss
accuracies.append(acc)
valid_batch_ctr += 1
if valid_batch_ctr > n_test_batches:
break
test_loss /= n_test_batches
print "test loss: ", test_loss
print "test acc: ", np.mean(accuracies), "\n"
all_validation_losses.append(test_loss)
all_validation_accuracies.append(np.mean(accuracies))
printLosses(all_training_losses, all_training_accs, all_validation_losses,
all_validation_accuracies,
"../results/%s.png" % EXPERIMENT_NAME, 10)
learning_rate *= 0.3
def imgSaveFalsePositiveFalseNegativeCorrectPositiveCorrectNegative(
pred_fn, n_images=16, BATCH_SIZE=50):
with open("../data/patchClassification_memmap_properties.pkl", 'r') as f:
memmap_properties = cPickle.load(f)
n_pos_val = memmap_properties["val_pos"]
n_neg_val = memmap_properties["val_neg"]
val_pos_memmap = memmap("../data/patchClassification_val_pos.memmap",
dtype=np.float32,
mode="r+",
shape=memmap_properties["val_pos_shape"])
val_neg_memmap = memmap("../data/patchClassification_val_neg.memmap",
dtype=np.float32,
mode="r+",
shape=memmap_properties["val_neg_shape"])
n_fpos = 0
n_fneg = 0
n_tpos = 0
n_tneg = 0
# it is simpler to just extract the fpos, fneg, tpos and tneg images one cathegory after the other. speed doesnt matter here
plt.figure(figsize=(16, 16))
for data, seg, labels in memmapGenerator(val_neg_memmap, val_pos_memmap,
BATCH_SIZE, n_pos_val, n_neg_val):
if n_fpos < n_images:
pred = pred_fn(data).argmax(-1)
idx = np.where((pred == 1) & (labels == 0))[0]
for id in idx:
plt.subplot(int(np.ceil(np.sqrt(n_images))),
int(np.ceil(np.sqrt(n_images))), n_fpos)
plt.imshow(data[id, 0, :, :],
cmap="gray",
interpolation="nearest")
plt.text(0,
0,
labels[id],
color='red',
bbox=dict(facecolor='white', alpha=1))
plt.text(0,
12,
pred[id],
color='green',
bbox=dict(facecolor='white', alpha=1))
plt.text(8, 0, 'true', color='blue')
plt.text(8, 12, 'predicted', color='blue')
n_fpos += 1
if n_fpos >= n_images:
break
else:
break
plt.savefig("../results/falsePositives.png")
plt.close()
plt.figure(figsize=(16, 16))
for data, seg, labels in memmapGenerator(val_neg_memmap, val_pos_memmap,
BATCH_SIZE, n_pos_val, n_neg_val):
if n_fneg < n_images:
pred = pred_fn(data).argmax(-1)
idx = np.where((pred == 0) & (labels == 1))[0]
for id in idx:
plt.subplot(int(np.ceil(np.sqrt(n_images))),
int(np.ceil(np.sqrt(n_images))), n_fneg)
plt.imshow(data[id, 0, :, :],
cmap="gray",
interpolation="nearest")
plt.text(0,
0,
labels[id],
color='green',
bbox=dict(facecolor='white', alpha=1))
plt.text(0,
12,
pred[id],
color='red',
bbox=dict(facecolor='white', alpha=1))
plt.text(8, 0, 'true', color='blue')
plt.text(8, 12, 'predicted', color='blue')
n_fneg += 1
if n_fneg >= n_images:
break
else:
break
plt.savefig("../results/falseNegatives.png")
plt.close()
plt.figure(figsize=(16, 16))
for data, seg, labels in memmapGenerator(val_neg_memmap, val_pos_memmap,
BATCH_SIZE, n_pos_val, n_neg_val):
if n_tpos < n_images:
pred = pred_fn(data).argmax(-1)
idx = np.where((pred == 1) & (labels == 1))[0]
for id in idx:
plt.subplot(int(np.ceil(np.sqrt(n_images))),
int(np.ceil(np.sqrt(n_images))), n_tpos)
plt.imshow(data[id, 0, :, :],
cmap="gray",
interpolation="nearest")
plt.text(0,
0,
labels[id],
color='green',
bbox=dict(facecolor='white', alpha=1))
plt.text(0,
12,
pred[id],
color='green',
bbox=dict(facecolor='white', alpha=1))
plt.text(8, 0, 'true', color='blue')
plt.text(8, 12, 'predicted', color='blue')
n_tpos += 1
if n_tpos >= n_images:
break
else:
break
plt.savefig("../results/truePositives.png")
plt.close()
plt.figure(figsize=(16, 16))
for data, seg, labels in memmapGenerator(val_neg_memmap, val_pos_memmap,
BATCH_SIZE, n_pos_val, n_neg_val):
if n_tneg < n_images:
pred = pred_fn(data).argmax(-1)
idx = np.where((pred == 0) & (labels == 0))[0]
for id in idx:
plt.subplot(int(np.ceil(np.sqrt(n_images))),
int(np.ceil(np.sqrt(n_images))), n_tneg)
plt.imshow(data[id, 0, :, :],
cmap="gray",
interpolation="nearest")
plt.text(0,
0,
labels[id],
color='red',
bbox=dict(facecolor='white', alpha=1))
plt.text(0,
12,
pred[id],
color='red',
bbox=dict(facecolor='white', alpha=1))
plt.text(8, 0, 'true', color='blue')
plt.text(8, 12, 'predicted', color='blue')
n_tneg += 1
if n_tneg >= n_images:
break
else:
break
plt.savefig("../results/trueNegatives.png")
plt.close()
def imgSaveFalsePositiveFalseNegativeCorrectPositiveCorrectNegative(pred_fn, n_images=16, BATCH_SIZE = 50):
with open("../data/patchClassification_memmap_properties.pkl", 'r') as f:
memmap_properties = cPickle.load(f)
n_pos_val = memmap_properties["val_pos"]
n_neg_val = memmap_properties["val_neg"]
val_pos_memmap = memmap("../data/patchClassification_val_pos.memmap", dtype=np.float32, mode="r+", shape=memmap_properties["val_pos_shape"])
val_neg_memmap = memmap("../data/patchClassification_val_neg.memmap", dtype=np.float32, mode="r+", shape=memmap_properties["val_neg_shape"])
n_fpos = 0
n_fneg = 0
n_tpos = 0
n_tneg = 0
# it is simpler to just extract the fpos, fneg, tpos and tneg images one cathegory after the other. speed doesnt matter here
plt.figure(figsize=(16,16))
for data, seg, labels in memmapGenerator(val_neg_memmap, val_pos_memmap, BATCH_SIZE, n_pos_val, n_neg_val):
if n_fpos < n_images:
pred = pred_fn(data).argmax(-1)
idx = np.where((pred == 1) & (labels == 0))[0]
for id in idx:
plt.subplot(int(np.ceil(np.sqrt(n_images))), int(np.ceil(np.sqrt(n_images))), n_fpos)
plt.imshow(data[id, 0, :, :], cmap="gray", interpolation="nearest")
plt.text(0, 0, labels[id], color='red', bbox=dict(facecolor='white', alpha=1))
plt.text(0, 12, pred[id], color='green', bbox=dict(facecolor='white', alpha=1))
plt.text(8, 0, 'true', color='blue')
plt.text(8, 12, 'predicted', color='blue')
n_fpos += 1
if n_fpos >= n_images:
break
else:
break
plt.savefig("../results/falsePositives.png")
plt.close()
plt.figure(figsize=(16,16))
for data, seg, labels in memmapGenerator(val_neg_memmap, val_pos_memmap, BATCH_SIZE, n_pos_val, n_neg_val):
if n_fneg < n_images:
pred = pred_fn(data).argmax(-1)
idx = np.where((pred == 0) & (labels == 1))[0]
for id in idx:
plt.subplot(int(np.ceil(np.sqrt(n_images))), int(np.ceil(np.sqrt(n_images))), n_fneg)
plt.imshow(data[id, 0, :, :], cmap="gray", interpolation="nearest")
plt.text(0, 0, labels[id], color='green', bbox=dict(facecolor='white', alpha=1))
plt.text(0, 12, pred[id], color='red', bbox=dict(facecolor='white', alpha=1))
plt.text(8, 0, 'true', color='blue')
plt.text(8, 12, 'predicted', color='blue')
n_fneg += 1
if n_fneg >= n_images:
break
else:
break
plt.savefig("../results/falseNegatives.png")
plt.close()
plt.figure(figsize=(16,16))
for data, seg, labels in memmapGenerator(val_neg_memmap, val_pos_memmap, BATCH_SIZE, n_pos_val, n_neg_val):
if n_tpos < n_images:
pred = pred_fn(data).argmax(-1)
idx = np.where((pred == 1) & (labels == 1))[0]
for id in idx:
plt.subplot(int(np.ceil(np.sqrt(n_images))), int(np.ceil(np.sqrt(n_images))), n_tpos)
plt.imshow(data[id, 0, :, :], cmap="gray", interpolation="nearest")
plt.text(0, 0, labels[id], color='green', bbox=dict(facecolor='white', alpha=1))
plt.text(0, 12, pred[id], color='green', bbox=dict(facecolor='white', alpha=1))
plt.text(8, 0, 'true', color='blue')
plt.text(8, 12, 'predicted', color='blue')
n_tpos += 1
if n_tpos >= n_images:
break
else:
break
plt.savefig("../results/truePositives.png")
plt.close()
plt.figure(figsize=(16,16))
for data, seg, labels in memmapGenerator(val_neg_memmap, val_pos_memmap, BATCH_SIZE, n_pos_val, n_neg_val):
if n_tneg < n_images:
pred = pred_fn(data).argmax(-1)
idx = np.where((pred == 0) & (labels == 0))[0]
for id in idx:
plt.subplot(int(np.ceil(np.sqrt(n_images))), int(np.ceil(np.sqrt(n_images))), n_tneg)
plt.imshow(data[id, 0, :, :], cmap="gray", interpolation="nearest")
plt.text(0, 0, labels[id], color='red', bbox=dict(facecolor='white', alpha=1))
plt.text(0, 12, pred[id], color='red', bbox=dict(facecolor='white', alpha=1))
plt.text(8, 0, 'true', color='blue')
plt.text(8, 12, 'predicted', color='blue')
n_tneg += 1
if n_tneg >= n_images:
break
else:
break
plt.savefig("../results/trueNegatives.png")
plt.close()
