This tool creates the datapoints necessary for a precision and recall curve figure. The tool samples a patch dataset from the test and validation set (-data), and creates predictions using a trained model (-model). These predictions are thresholded at several values. The binarized predictions and the label are then used to calculate the precision as well as the recall. These values including the threshold amount, constitute a data point. Supplying a experiment id (-store_gui), will store the datapoints in the web GUI. It's worth noting that the measurements are relaxed. Relaxed precision and relaxed recall. This is implemented by the image processing operation, dilation. The slack variable is set to 3 pixels. ''' print_section("TOOLS: Measure precision and recall of model") print("-data: path to dataset | -store: job_gui id to store curve in GUI "\ "| -store_path: store results locally | -model: stored model to use") #====== Arguments =============================================== is_dataset_path, dataset_path = get_command( '-data', default='/home/olav/Pictures/Mass_roads_alpha') store_gui, job_id = get_command('-store_gui', default='-1') is_store_path, store_path = get_command('-store_path', default='./pr_data.json') is_model, model_path = get_command('-model', default='./results/params.pkl') #============================================================== store = ParamStorage() data = store.load_params(path=model_path) batch_size = data['optimization'].batch_size measurer = PrecisionRecallCurve(dataset_path, data['params'], data['model'], data['dataset']) datapoints = measurer.get_curves_datapoints(batch_size) if store_gui:
import os, sys from PIL import Image import numpy as np sys.path.append(os.path.abspath("./")) from interface.command import get_command ''' This tool counts the percentage of road versus non-road pixels in the label set. ''' is_dataset_path, dataset_path = get_command( '-data', default='/home/olav/Pictures/Mass_roads_alpha') is_set, set_type = get_command('-set', default='train') def examine_label_dist(path): image = Image.open(path, 'r').convert('L') arr = np.array(image) w, h = arr.shape arr = np.reshape(arr, w * h) positive = np.count_nonzero(arr) negative = w * h - positive ratio_pos = positive / float((w * h)) ratio_neg = negative / float((w * h)) print("%.4f" % ratio_pos, "area contains other stuff", "%.4f" % ratio_neg, " Of area contains roads") return ratio_pos label_path = os.path.join(dataset_path, set_type, 'labels')
import numpy as np import sys, os from PIL import Image, ImageFilter sys.path.append(os.path.abspath("./")) from storage import ParamStorage from interface import command ''' This tool loads the model weight configuration stored in ./results/params.pkl and visualize the weights in the first layer. The weight configuration of each kernel is converted to a RGB image. The tool assume there are only 64 kernels in the first layer. A stored model can also be supplied (-model). ''' is_model, model_path = command.get_command('-model', default='./results/params.pkl') def make_visual(layer_weights): max_scale = layer_weights.max(axis=-1).max(axis=-1)[..., np.newaxis, np.newaxis] min_scale = layer_weights.min(axis=-1).min(axis=-1)[..., np.newaxis, np.newaxis] return (255 * (layer_weights - min_scale) / (max_scale - min_scale)).astype(np.uint8) store = ParamStorage() data = store.load_params(path=model_path) first_layer = np.array(data['params'][-2].eval()) print(first_layer.shape)
from interface.command import get_command from printing import print_section, print_action from storage import ParamStorage from config import filename_params, dataset_params, pr_path, dataset_path from augmenter.aerial import Creator from data import AerialCurriculumDataset import tools.util as util ''' Create histograms of difference between prediction and label for dataset. Allow finetuning of curriculum strategy. ''' print_section( 'Generating plot of diff distribution between label and prediction') #====== Arguments =============================================== is_samples, samples = get_command('-samples', default="100") samples = int(samples) is_teacher_location, teacher_location = get_command( '-teacher', default=filename_params.curriculum_teacher) verify, stage = get_command('-verify', default="0") stage = "stage" + stage is_tradeoff, tradeoff = get_command('-tradeoff', default="0.5") tradeoff = float(tradeoff) #Dataset path. Config used if not supplied is_alt_dataset, alt_dataset = get_command('-dataset') if is_alt_dataset: dataset_path = alt_dataset
thresholded at several values. The binarized predictions and the label are then used to calculate the precision as well as the recall. These values including the threshold amount, constitute a data point. Supplying a experiment id (-store_gui), will store the datapoints in the web GUI. It's worth noting that the measurements are relaxed. Relaxed precision and relaxed recall. This is implemented by the image processing operation, dilation. The slack variable is set to 3 pixels. """ print_section("TOOLS: Measure precision and recall of model") print( "-data: path to dataset | -store: job_gui id to store curve in GUI " "| -store_path: store results locally | -model: stored model to use" ) # ====== Arguments =============================================== is_dataset_path, dataset_path = get_command("-data", default="/home/olav/Pictures/Mass_roads_alpha") store_gui, job_id = get_command("-store_gui", default="-1") is_store_path, store_path = get_command("-store_path", default="./pr_data.json") is_model, model_path = get_command("-model", default="./results/params.pkl") # ============================================================== store = ParamStorage() data = store.load_params(path=model_path) batch_size = data["optimization"].batch_size measurer = PrecisionRecallCurve(dataset_path, data["params"], data["model"], data["dataset"]) datapoints = measurer.get_curves_datapoints(batch_size) if store_gui: send_precision_recall_data(datapoints, None, job_id=job_id) else:
from storage import ParamStorage from config import filename_params, dataset_params, pr_path, dataset_path from augmenter.aerial import Creator from data import AerialCurriculumDataset import tools.util as util ''' Create histograms of difference between prediction and label for dataset. Allow finetuning of curriculum strategy. ''' print_section('Generating plot of diff distribution between label and prediction') #====== Arguments =============================================== is_samples, samples = get_command('-samples', default="100") samples = int(samples) is_teacher_location, teacher_location = get_command('-teacher', default=filename_params.curriculum_teacher) verify, stage = get_command('-verify', default="0") stage = "stage" + stage is_tradeoff, tradeoff = get_command('-tradeoff', default="0.5") tradeoff = float(tradeoff) #Dataset path. Config used if not supplied is_alt_dataset, alt_dataset = get_command('-dataset') if is_alt_dataset: dataset_path = alt_dataset #==============================================================
def store_image(image, job_id, store_gui, name="image"): out = Image.resize(image, 1.0) if store_gui: buf= StringIO.StringIO() out.save(buf, format='JPEG') send_result_image(job_id, buf.getvalue()) image.save('./tools/visualize/'+ name +'.jpg') image.show() print_section('TOOLS: Visualize result from model') print("-data: Path to image in dataset you want visualization of | -store_gui: Upload images to exp with supplied id | \ -tradeoff: Threshold value associated with precision recall breakeven |-storeimage: Include aerial image") is_image_path, image_path = get_command('-data', default='/home/olav/Pictures/Mass_roads/test/data/10378780_15.tiff') store_data_image, temp = get_command('-storeimage') store_gui, job_id = get_command('-store_gui', default="None") is_tradeoff, bto = get_command('-tradeoff', default="0.5") bto = float(bto) is_model, model_path = get_command('-model', default="./results/params.pkl") store = ParamStorage() data = store.load_params(path=model_path) batch_size = data['optimization'].batch_size v = Visualizer(data['model'], data['params'], data['dataset'])
import numpy as np import sys, os from PIL import Image, ImageFilter sys.path.append(os.path.abspath("./")) from storage import ParamStorage from interface import command ''' This tool loads the model weight configuration stored in ./results/params.pkl and visualize the weights in the first layer. The weight configuration of each kernel is converted to a RGB image. The tool assume there are only 64 kernels in the first layer. A stored model can also be supplied (-model). ''' is_model, model_path = command.get_command('-model', default='./results/params.pkl') def make_visual(layer_weights): max_scale = layer_weights.max(axis=-1).max(axis=-1)[..., np.newaxis, np.newaxis] min_scale = layer_weights.min(axis=-1).min(axis=-1)[..., np.newaxis, np.newaxis] return (255 * (layer_weights - min_scale) / (max_scale - min_scale)).astype(np.uint8) store = ParamStorage() data = store.load_params(path=model_path) first_layer = np.array( data['params'][-2].eval())
-baseline: No difficulty estimation -stages: Array with floats, setting difficulty threshold per stage. Please refrain from using space inside array. -tradeoff: Previously trained curriculum teacher's best precision and recall tradeoff. (threshold value) -dataset: Path to dataset. IE -initsamples: Samples per image for first stage -currsamples: Samples per image for remaining stages -teacher: Curriculum teacher model -save: Path to where pre-generated patch dataset should be stored. REMEMBER: The patch creator is initialized using the config.py file. ''' print_section("TOOLS: Creating curriculum learning dataset") # Baseline will create a curriculum with no example ordering, but same amount of examples. # Avoids results from curriculum learning to be caused by the model just having seen more examples. is_baseline, baseline = get_command('-baseline') is_stages, stages = get_command('-stages', default="[0.1, 1.0]") stages = np.array(eval(stages)) #Precision recall breakeven point. 0.5 used as a default. is_tradeoff, tradeoff = get_command('-tradeoff') if is_tradeoff: tradeoff = float(tradeoff) #Dataset path. Config used if not supplied is_alt_dataset, alt_dataset = get_command('-dataset') if is_alt_dataset: dataset_path = alt_dataset #Initial stage sample size
buf = StringIO.StringIO() out.save(buf, format='JPEG') send_result_image(job_id, buf.getvalue()) image.save('./tools/visualize/' + name + '.jpg') image.show() print_section('TOOLS: Visualize result from model') print( "-data: Path to image in dataset you want visualization of | -store_gui: Upload images to exp with supplied id | \ -tradeoff: Threshold value associated with precision recall breakeven |-storeimage: Include aerial image" ) is_image_path, image_path = get_command( '-data', default='/home/olav/Pictures/Mass_roads/test/data/10378780_15.tiff') store_data_image, temp = get_command('-storeimage') store_gui, job_id = get_command('-store_gui', default="None") is_tradeoff, bto = get_command('-tradeoff', default="0.5") bto = float(bto) is_model, model_path = get_command('-model', default="./results/params.pkl") store = ParamStorage() data = store.load_params(path=model_path) batch_size = data['optimization'].batch_size
import os,sys from PIL import Image import numpy as np sys.path.append(os.path.abspath("./")) from interface.command import get_command ''' This tool counts the percentage of road versus non-road pixels in the label set. ''' is_dataset_path, dataset_path = get_command('-data', default='/home/olav/Pictures/Mass_roads_alpha') is_set, set_type = get_command('-set', default='train') def examine_label_dist(path): image = Image.open(path, 'r').convert('L') arr = np.array(image) w, h = arr.shape arr = np.reshape(arr, w*h) positive = np.count_nonzero(arr) negative = w*h - positive ratio_pos = positive/float((w*h)) ratio_neg = negative/float((w*h)) print( "%.4f" % ratio_pos, "area contains other stuff", "%.4f" % ratio_neg, " Of area contains roads") return ratio_pos label_path = os.path.join(dataset_path, set_type, 'labels') included_extenstions = ['jpg','png', 'tiff', 'tif'] labels = [fn for fn in os.listdir(label_path) if any([fn.endswith(ext) for ext in included_extenstions])]