This tool creates the datapoints necessary for a precision and recall curve figure. The tool samples a patch dataset
from the test and validation set (-data), and creates predictions using a trained model (-model). These predictions are
thresholded at several values. The binarized predictions and the label are then used to calculate the precision as well
as the recall. These values including the threshold amount, constitute a data point. Supplying a experiment id (-store_gui),
will store the datapoints in the web GUI.
It's worth noting that the measurements are relaxed. Relaxed precision and relaxed recall. This is implemented by the
image processing operation, dilation. The slack variable is set to 3 pixels.
'''
print_section("TOOLS: Measure precision and recall of model")
print("-data: path to dataset | -store: job_gui id to store curve in GUI "\
"| -store_path: store results locally | -model: stored model to use")
#====== Arguments ===============================================
is_dataset_path, dataset_path = get_command(
'-data', default='/home/olav/Pictures/Mass_roads_alpha')
store_gui, job_id = get_command('-store_gui', default='-1')
is_store_path, store_path = get_command('-store_path',
default='./pr_data.json')
is_model, model_path = get_command('-model', default='./results/params.pkl')
#==============================================================
store = ParamStorage()
data = store.load_params(path=model_path)
batch_size = data['optimization'].batch_size
measurer = PrecisionRecallCurve(dataset_path, data['params'], data['model'],
data['dataset'])
datapoints = measurer.get_curves_datapoints(batch_size)
if store_gui:
import os, sys
from PIL import Image
import numpy as np
sys.path.append(os.path.abspath("./"))
from interface.command import get_command
'''
This tool counts the percentage of road versus non-road pixels in the label set.
'''
is_dataset_path, dataset_path = get_command(
'-data', default='/home/olav/Pictures/Mass_roads_alpha')
is_set, set_type = get_command('-set', default='train')
def examine_label_dist(path):
image = Image.open(path, 'r').convert('L')
arr = np.array(image)
w, h = arr.shape
arr = np.reshape(arr, w * h)
positive = np.count_nonzero(arr)
negative = w * h - positive
ratio_pos = positive / float((w * h))
ratio_neg = negative / float((w * h))
print("%.4f" % ratio_pos, "area contains other stuff", "%.4f" % ratio_neg,
" Of area contains roads")
return ratio_pos
label_path = os.path.join(dataset_path, set_type, 'labels')
import numpy as np
import sys, os
from PIL import Image, ImageFilter
sys.path.append(os.path.abspath("./"))
from storage import ParamStorage
from interface import command
'''
This tool loads the model weight configuration stored in ./results/params.pkl and visualize the weights in the first layer.
The weight configuration of each kernel is converted to a RGB image. The tool assume there are only 64 kernels in the
first layer. A stored model can also be supplied (-model).
'''
is_model, model_path = command.get_command('-model',
default='./results/params.pkl')
def make_visual(layer_weights):
max_scale = layer_weights.max(axis=-1).max(axis=-1)[..., np.newaxis,
np.newaxis]
min_scale = layer_weights.min(axis=-1).min(axis=-1)[..., np.newaxis,
np.newaxis]
return (255 * (layer_weights - min_scale) /
(max_scale - min_scale)).astype(np.uint8)
store = ParamStorage()
data = store.load_params(path=model_path)
first_layer = np.array(data['params'][-2].eval())
print(first_layer.shape)
from interface.command import get_command
from printing import print_section, print_action
from storage import ParamStorage
from config import filename_params, dataset_params, pr_path, dataset_path
from augmenter.aerial import Creator
from data import AerialCurriculumDataset
import tools.util as util
'''
Create histograms of difference between prediction and label for dataset.
Allow finetuning of curriculum strategy.
'''
print_section(
'Generating plot of diff distribution between label and prediction')
#====== Arguments ===============================================
is_samples, samples = get_command('-samples', default="100")
samples = int(samples)
is_teacher_location, teacher_location = get_command(
'-teacher', default=filename_params.curriculum_teacher)
verify, stage = get_command('-verify', default="0")
stage = "stage" + stage
is_tradeoff, tradeoff = get_command('-tradeoff', default="0.5")
tradeoff = float(tradeoff)
#Dataset path. Config used if not supplied
is_alt_dataset, alt_dataset = get_command('-dataset')
if is_alt_dataset:
dataset_path = alt_dataset
thresholded at several values. The binarized predictions and the label are then used to calculate the precision as well
as the recall. These values including the threshold amount, constitute a data point. Supplying a experiment id (-store_gui),
will store the datapoints in the web GUI.
It's worth noting that the measurements are relaxed. Relaxed precision and relaxed recall. This is implemented by the
image processing operation, dilation. The slack variable is set to 3 pixels.
"""
print_section("TOOLS: Measure precision and recall of model")
print(
"-data: path to dataset | -store: job_gui id to store curve in GUI "
"| -store_path: store results locally | -model: stored model to use"
)
# ====== Arguments ===============================================
is_dataset_path, dataset_path = get_command("-data", default="/home/olav/Pictures/Mass_roads_alpha")
store_gui, job_id = get_command("-store_gui", default="-1")
is_store_path, store_path = get_command("-store_path", default="./pr_data.json")
is_model, model_path = get_command("-model", default="./results/params.pkl")
# ==============================================================
store = ParamStorage()
data = store.load_params(path=model_path)
batch_size = data["optimization"].batch_size
measurer = PrecisionRecallCurve(dataset_path, data["params"], data["model"], data["dataset"])
datapoints = measurer.get_curves_datapoints(batch_size)
if store_gui:
send_precision_recall_data(datapoints, None, job_id=job_id)
else:
from storage import ParamStorage
from config import filename_params, dataset_params, pr_path, dataset_path
from augmenter.aerial import Creator
from data import AerialCurriculumDataset
import tools.util as util
'''
Create histograms of difference between prediction and label for dataset.
Allow finetuning of curriculum strategy.
'''
print_section('Generating plot of diff distribution between label and prediction')
#====== Arguments ===============================================
is_samples, samples = get_command('-samples', default="100")
samples = int(samples)
is_teacher_location, teacher_location = get_command('-teacher', default=filename_params.curriculum_teacher)
verify, stage = get_command('-verify', default="0")
stage = "stage" + stage
is_tradeoff, tradeoff = get_command('-tradeoff', default="0.5")
tradeoff = float(tradeoff)
#Dataset path. Config used if not supplied
is_alt_dataset, alt_dataset = get_command('-dataset')
if is_alt_dataset:
dataset_path = alt_dataset
#==============================================================
def store_image(image, job_id, store_gui, name="image"):
out = Image.resize(image, 1.0)
if store_gui:
buf= StringIO.StringIO()
out.save(buf, format='JPEG')
send_result_image(job_id, buf.getvalue())
image.save('./tools/visualize/'+ name +'.jpg')
image.show()
print_section('TOOLS: Visualize result from model')
print("-data: Path to image in dataset you want visualization of | -store_gui: Upload images to exp with supplied id | \
-tradeoff: Threshold value associated with precision recall breakeven |-storeimage: Include aerial image")
is_image_path, image_path = get_command('-data', default='/home/olav/Pictures/Mass_roads/test/data/10378780_15.tiff')
store_data_image, temp = get_command('-storeimage')
store_gui, job_id = get_command('-store_gui', default="None")
is_tradeoff, bto = get_command('-tradeoff', default="0.5")
bto = float(bto)
is_model, model_path = get_command('-model', default="./results/params.pkl")
store = ParamStorage()
data = store.load_params(path=model_path)
batch_size = data['optimization'].batch_size
v = Visualizer(data['model'], data['params'], data['dataset'])
import numpy as np
import sys, os
from PIL import Image, ImageFilter
sys.path.append(os.path.abspath("./"))
from storage import ParamStorage
from interface import command
'''
This tool loads the model weight configuration stored in ./results/params.pkl and visualize the weights in the first layer.
The weight configuration of each kernel is converted to a RGB image. The tool assume there are only 64 kernels in the
first layer. A stored model can also be supplied (-model).
'''
is_model, model_path = command.get_command('-model', default='./results/params.pkl')
def make_visual(layer_weights):
max_scale = layer_weights.max(axis=-1).max(axis=-1)[...,
np.newaxis, np.newaxis]
min_scale = layer_weights.min(axis=-1).min(axis=-1)[...,
np.newaxis, np.newaxis]
return (255 * (layer_weights - min_scale) /
(max_scale - min_scale)).astype(np.uint8)
store = ParamStorage()
data = store.load_params(path=model_path)
first_layer = np.array(
data['params'][-2].eval())
-baseline: No difficulty estimation
-stages: Array with floats, setting difficulty threshold per stage. Please refrain from using space inside array.
-tradeoff: Previously trained curriculum teacher's best precision and recall tradeoff. (threshold value)
-dataset: Path to dataset. IE
-initsamples: Samples per image for first stage
-currsamples: Samples per image for remaining stages
-teacher: Curriculum teacher model
-save: Path to where pre-generated patch dataset should be stored.
REMEMBER: The patch creator is initialized using the config.py file.
'''
print_section("TOOLS: Creating curriculum learning dataset")
# Baseline will create a curriculum with no example ordering, but same amount of examples.
# Avoids results from curriculum learning to be caused by the model just having seen more examples.
is_baseline, baseline = get_command('-baseline')
is_stages, stages = get_command('-stages', default="[0.1, 1.0]")
stages = np.array(eval(stages))
#Precision recall breakeven point. 0.5 used as a default.
is_tradeoff, tradeoff = get_command('-tradeoff')
if is_tradeoff:
tradeoff = float(tradeoff)
#Dataset path. Config used if not supplied
is_alt_dataset, alt_dataset = get_command('-dataset')
if is_alt_dataset:
dataset_path = alt_dataset
#Initial stage sample size
buf = StringIO.StringIO()
out.save(buf, format='JPEG')
send_result_image(job_id, buf.getvalue())
image.save('./tools/visualize/' + name + '.jpg')
image.show()
print_section('TOOLS: Visualize result from model')
print(
"-data: Path to image in dataset you want visualization of | -store_gui: Upload images to exp with supplied id | \
-tradeoff: Threshold value associated with precision recall breakeven |-storeimage: Include aerial image"
)
is_image_path, image_path = get_command(
'-data',
default='/home/olav/Pictures/Mass_roads/test/data/10378780_15.tiff')
store_data_image, temp = get_command('-storeimage')
store_gui, job_id = get_command('-store_gui', default="None")
is_tradeoff, bto = get_command('-tradeoff', default="0.5")
bto = float(bto)
is_model, model_path = get_command('-model', default="./results/params.pkl")
store = ParamStorage()
data = store.load_params(path=model_path)
batch_size = data['optimization'].batch_size
import os,sys
from PIL import Image
import numpy as np
sys.path.append(os.path.abspath("./"))
from interface.command import get_command
'''
This tool counts the percentage of road versus non-road pixels in the label set.
'''
is_dataset_path, dataset_path = get_command('-data', default='/home/olav/Pictures/Mass_roads_alpha')
is_set, set_type = get_command('-set', default='train')
def examine_label_dist(path):
image = Image.open(path, 'r').convert('L')
arr = np.array(image)
w, h = arr.shape
arr = np.reshape(arr, w*h)
positive = np.count_nonzero(arr)
negative = w*h - positive
ratio_pos = positive/float((w*h))
ratio_neg = negative/float((w*h))
print( "%.4f" % ratio_pos, "area contains other stuff", "%.4f" % ratio_neg, " Of area contains roads")
return ratio_pos
label_path = os.path.join(dataset_path, set_type, 'labels')
included_extenstions = ['jpg','png', 'tiff', 'tif']
labels = [fn for fn in os.listdir(label_path) if any([fn.endswith(ext) for ext in included_extenstions])]
