def __verify_submission():
file_path = os.path.join(paths.DATA_FOLDER_PATH, "out", "predictions.txt")
colour_mapping = mapping.get_colour_mapping()
with open(file_path, "r") as fp:
for line in fp:
# parse file
file_id, data = parse_submission_line(line)
confidences = [d["confidence"] for d in data]
indices = np.argsort(confidences)
# draw data on image
colour_mask_1 = np.zeros((3024, 4032)).astype(np.uint8)
for index in indices:
polygon = np.array(data[index]["polygon"])
polygon = polygon.reshape((-1, 1, 2))
colour_mask_1 = cv2.fillPoly(
colour_mask_1, [polygon],
color=colour_mapping[data[index]["class_name"]])
# load original prediciton
prediction_file = os.path.join(paths.DATA_FOLDER_PATH,
"predictions", file_id + ".npy")
prediction = np.load(prediction_file)
class_id_mask = np.argmax(prediction, axis=-1)
colour_mask_2 = class_id_mask_to_colour_mask(class_id_mask)
plt.subplot(121)
plt.imshow(colour_mask_1)
plt.subplot(122)
plt.imshow(colour_mask_2)
plt.show()
def colour_mask_to_class_id_mask(colour_mask, data_folder_path=None):
"""
:param colour_mask:
:return:
"""
colour_mapping = mapping.get_colour_mapping(data_folder_path)
class_id_mask = np.zeros(colour_mask.shape[:2]).astype(np.uint8)
for i, k in enumerate(sorted(colour_mapping.keys())):
class_id_mask[colour_mask == colour_mapping[k]] = i
return class_id_mask
def class_id_mask_to_colour_mask(class_id_mask):
"""
:param class_id_mask:
:return:
"""
colour_mapping = mapping.get_colour_mapping()
colour_mask = np.zeros(class_id_mask.shape[:2]).astype(np.uint8)
for i, k in enumerate(sorted(colour_mapping.keys())):
colour_mask[class_id_mask == i] = colour_mapping[k]
return colour_mask
def create_submission_file(prediction_file_names, output_file_path):
"""
Creates a submission file in accordance with the rules of the competition based on the given files
:param prediction_file_names: paths to numpy files or arrays with size (h, w, class_count)
:param output_file_path: path to the file where the results will be stored
:return: Nothing
"""
colour_mapping = mapping.get_colour_mapping()
class_names = sorted(colour_mapping.keys())
out_lines = []
for i, file in enumerate(prediction_file_names):
print("file {} of {}\n".format(i + 1, len(prediction_file_names)))
prediction = np.load(file)
# start output row
line = "{};".format(os.path.split(file)[1][:-4])
class_id_mask = np.argmax(prediction, axis=-1)
class_ids = np.unique(class_id_mask).tolist()
# remove background class
if 0 in class_ids:
class_ids.remove(0)
pbar = tqdm(class_ids, ncols=50)
for class_id in pbar:
mask = ((class_id_mask == class_id) * 255).astype(np.uint8)
# apply opening to mask to remove small coral-areas
# TODO: the size needs to be set to something sensible!
kernel_size = (31, 31)
kernel = np.ones(kernel_size, np.uint8)
mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
kernel = np.ones(kernel_size, np.uint8)
mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel)
# if there are less than x pixel, skip the class
if np.sum(mask / 255) < 100:
continue
line += "{} ".format(class_names[class_id])
# find connected components
ret, labels = cv2.connectedComponents(mask)
for j, label in enumerate(
range(1, ret)
): # label 0 is for background of the connected components
cc_mask = ((labels == label) * 255).astype(np.uint8)
non_nulls = np.where(cc_mask)
cv2.floodFill(cc_mask, None,
(non_nulls[1][0], non_nulls[0][0]), 255)
polygon = _get_contour(cc_mask)
cc_mask = cc_mask / 255
confidence = (prediction[:, :, class_id] *
cc_mask).sum() / cc_mask.sum()
# print("confidence: {:.2f}".format(confidence))
if j > 0:
line += ","
line += "{:.2f}:".format(confidence)
line += _create_polygon_string(polygon)
line += ";"
line = line[:-1] # remove last semicolon
line += "\n"
out_lines.append(line)
with open(output_file_path, "w") as fp:
fp.writelines(out_lines)
def evaluate(image_file_paths,
gt_file_paths,
model_path,
nn_input_size,
window_sizes=None,
step_sizes=None,
device=None,
data_folder_path=None,
log_file_path=None):
"""
Evaluate model performance
:param image_file_paths: paths to images that will be predicted
:param gt_file_paths: paths to ground truth masks
:param model: model used for prediction
:param nn_input_size: size that the images will be scaled to before feeding them into the nn
:param num_classes: number of classes
:param window_sizes: list of sizes that determine how the image will be cut (for sliding window). Image will be cut
into squares
:param step_sizes: list of step sizes for sliding window
:param device: PyTorch device (cpu or gpu)
:return: list of prediction masks if res_fcn is None, else Nothing
"""
model = load_model(model_path)
colour_mapping = mapping.get_colour_mapping(
data_folder_path=data_folder_path)
classes_map = {x: y for x, y in enumerate(sorted(colour_mapping.keys()))}
with torch.no_grad():
predictions = predict.predict(image_file_paths=image_file_paths,
model=model,
nn_input_size=nn_input_size,
window_sizes=window_sizes,
step_sizes=step_sizes,
device=device)
# Calculates IoU looping over images and masks
intersection_per_substrate = defaultdict(int)
union_per_substrate = defaultdict(int)
for gt_file_path, prediction in zip(gt_file_paths, predictions):
pred_class_id_mask = np.argmax(prediction, axis=-1)
gt_colour_mask = cv2.imread(gt_file_path)[:, :, 0]
gt_class_id_mask = colour_mask_to_class_id_mask(gt_colour_mask)
for substrate_idx, substrate_name in classes_map.items():
intersection, union = calculate_agreement(
gt_class_id_mask,
pred_class_id_mask,
substrate_idx=substrate_idx)
if union:
intersection_per_substrate[substrate_name] += intersection
union_per_substrate[substrate_name] += union
iou_per_substrate, iou_average = _calculate_iou(
intersection_per_substrate, union_per_substrate, classes_map)
# Gradually append accuracy stats
if log_file_path:
with open(log_file_path, 'w') as f:
f.write(json.dumps(iou_per_substrate, indent=4))
f.write('\n')
f.write(f"avg: {iou_average}")
return iou_per_substrate, iou_average
def __init__(self,
data_train,
data_valid,
image_base_dir,
instructions,
models_folder_path=None,
data_folder_path=None,
checkpoint_file_path=None):
"""
:param data_train:
:param data_valid:
:param image_base_dir: A directory with all CLEF images (for training and validating)
:param instructions (dict): A dictionary containing instructions to train. It has
the following keys:
epochs
model_name
nn_input_shape
state_dict_file_path (default = None)
crops
images_per_batch
batch_size
backbone (default = resnet)
deeplab_output_stride (default = 16)
learning_rate (default = 1e-05)
multi_gpu (default = False)
class_stats_file_path
use_lr_scheduler (default = True)
"""
self.image_base_dir = image_base_dir
self.data_valid = data_valid
self.instructions = instructions
# specify model save dir
self.model_name = instructions[STR.MODEL_NAME]
# now = time.localtime()
# start_time = "{}-{}-{}T{}:{}:{}".format(now.tm_year, now.tm_mon, now.tm_mday, now.tm_hour, now.tm_min,
# now.tm_sec)
models_folder_path = models_folder_path or paths.MODELS_FOLDER_PATH
data_folder_path = data_folder_path or paths.DATA_FOLDER_PATH
experiment_folder_path = os.path.join(models_folder_path,
self.model_name)
if os.path.exists(experiment_folder_path):
Warning(
"Experiment folder exists already. Files might be overwritten")
os.makedirs(experiment_folder_path, exist_ok=True)
# define saver and save instructions
self.saver = Saver(folder_path=experiment_folder_path,
instructions=instructions)
self.saver.save_instructions()
# define Tensorboard Summary
self.writer = SummaryWriter(log_dir=experiment_folder_path)
nn_input_size = instructions[STR.NN_INPUT_SIZE]
state_dict_file_path = instructions.get(STR.STATE_DICT_FILE_PATH, None)
self.colour_mapping = mapping.get_colour_mapping(
data_folder_path=data_folder_path)
# define transformers for training
crops_per_image = instructions.get(STR.CROPS_PER_IMAGE, 10)
apply_random_cropping = (STR.CROPS_PER_IMAGE in instructions.keys()) and \
(STR.IMAGES_PER_BATCH in instructions.keys())
print("{}applying random cropping".format(
"" if apply_random_cropping else "_NOT_ "))
crop = RandomCrop(min_size=instructions.get(STR.CROP_SIZE_MIN, 400),
max_size=instructions.get(STR.CROP_SIZE_MAX, 1000),
crop_count=crops_per_image)
t = [Normalize()]
if apply_random_cropping:
t.append(crop)
t += [
Resize(nn_input_size),
Flip(p_vertical=0.2, p_horizontal=0.5),
ToTensor()
]
transformations_train = transforms.Compose(t)
# define transformers for validation
transformations_valid = transforms.Compose(
[Normalize(), crop,
Resize(nn_input_size),
ToTensor()])
# set up data loaders
dataset_train = DictArrayDataSet(image_base_dir=image_base_dir,
data=data_train,
data_folder_path=data_folder_path,
num_classes=len(
self.colour_mapping.keys()),
transformation=transformations_train)
# define batch sizes
self.batch_size = instructions[STR.BATCH_SIZE]
if apply_random_cropping:
self.data_loader_train = DataLoader(
dataset=dataset_train,
batch_size=instructions[STR.IMAGES_PER_BATCH],
shuffle=True,
collate_fn=custom_collate)
else:
self.data_loader_train = DataLoader(dataset=dataset_train,
batch_size=self.batch_size,
shuffle=True,
collate_fn=custom_collate)
dataset_valid = DictArrayDataSet(image_base_dir=image_base_dir,
data=data_valid,
data_folder_path=data_folder_path,
num_classes=len(
self.colour_mapping.keys()),
transformation=transformations_valid)
self.data_loader_valid = DataLoader(dataset=dataset_valid,
batch_size=self.batch_size,
shuffle=False,
collate_fn=custom_collate)
self.num_classes = dataset_train.num_classes()
# define model
print("Building model")
self.model = DeepLab(num_classes=self.num_classes,
backbone=instructions.get(STR.BACKBONE, "resnet"),
output_stride=instructions.get(
STR.DEEPLAB_OUTPUT_STRIDE, 16))
# load weights
if checkpoint_file_path is not None:
print("loading state_dict from:")
print(checkpoint_file_path)
load_state_dict(self.model, checkpoint_file_path)
learning_rate = instructions.get(STR.LEARNING_RATE, 1e-5)
train_params = [{
'params': self.model.get_1x_lr_params(),
'lr': learning_rate
}, {
'params': self.model.get_10x_lr_params(),
'lr': learning_rate
}]
# choose gpu or cpu
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
if instructions.get(STR.MULTI_GPU, False):
if torch.cuda.device_count() > 1:
print("Using ", torch.cuda.device_count(), " GPUs!")
self.model = nn.DataParallel(self.model)
print(f"Using {self.device}")
self.model.to(self.device)
# Define Optimizer
self.optimizer = torch.optim.SGD(train_params,
momentum=0.9,
weight_decay=5e-4,
nesterov=False)
# calculate class weights
if instructions.get(STR.CLASS_STATS_FILE_PATH, None):
class_weights = calculate_class_weights(
instructions[STR.CLASS_STATS_FILE_PATH],
self.colour_mapping,
modifier=instructions.get(STR.LOSS_WEIGHT_MODIFIER, 1.01))
class_weights = torch.from_numpy(class_weights.astype(np.float32))
else:
class_weights = None
self.criterion = SegmentationLosses(
weight=class_weights, cuda=self.device.type != "cpu").build_loss()
# Define Evaluator
self.evaluator = Evaluator(self.num_classes)
# Define lr scheduler
self.scheduler = None
if instructions.get(STR.USE_LR_SCHEDULER, True):
self.scheduler = LR_Scheduler(mode="cos",
base_lr=learning_rate,
num_epochs=instructions[STR.EPOCHS],
iters_per_epoch=len(
self.data_loader_train))
# print information before training start
print("-" * 60)
print("instructions")
pprint(instructions)
model_parameters = sum([p.nelement() for p in self.model.parameters()])
print("Model parameters: {:.2E}".format(model_parameters))
self.best_prediction = 0.0
def create_annotation_masks(data_folder_path, image_folder="images",
mask_folder="masks",
annotations_file="annotations_train.csv",
create_new_mapping=False):
"""
Create mask images acting as annotations from the annotations data file. Mask files will be stored in the
path.MASK_FOLDER_PATH folder
:return: Nothing
"""
# parse the annotations file to get the data
data_folder_path = (data_folder_path if data_folder_path else paths.DATA_FOLDER_PATH)
image_folder_path = os.path.join(data_folder_path, image_folder)
mask_folder_path = os.path.join(data_folder_path, mask_folder)
os.makedirs(data_folder_path, exist_ok=True)
os.makedirs(image_folder_path, exist_ok=True)
os.makedirs(mask_folder_path, exist_ok=True)
csv_file_path = os.path.join(data_folder_path, annotations_file)
data = parse_csv_data_file(csv_file_path)
# create a list containing all classes
classes = list(sorted(set([annotation["class"] for img_name in data.keys() for annotation in data[img_name]])))
# create a colour for each class, 0 is background
colours = np.linspace(0, 255, len(classes) + 1).astype(int).tolist()
if create_new_mapping:
class_mapping = {c: i + 1 for i, c in enumerate(classes)}
colour_mapping = {"background": 0}
colour_mapping.update({c: colours[class_mapping[c]] for c in classes})
with open(os.path.join(data_folder_path, "colour_mapping.json"), "w") as fp:
json.dump(colour_mapping, fp, indent=4)
else:
colour_mapping = mapping.get_colour_mapping(data_folder_path)
print("Creating masks")
for i, image_name in enumerate(data.keys()):
# create mask based on the size of the corresponding image
print(image_folder_path)
print(image_name)
image_path = os.path.join(image_folder_path, image_name)
# CLEF files has some images which are JPG in caps
try:
image_height, image_width = cv2.imread(image_path).shape[:2]
except AttributeError:
image_path_head, ext = image_path.split('.')
# Tries with the reverse capitalisation
image_path = image_path_head + ('.JPG' if ext.islower() else '.jpg')
image_height, image_width = cv2.imread(image_path).shape[:2]
mask = np.zeros((image_height, image_width), dtype=np.uint8)
# go through each annotation entry and fill the corresponding polygon. Color corresponds to class
for annotation in data[image_name]:
colour = colour_mapping[annotation["class"]]
points = annotation["coordinates"]
cv2.fillPoly(mask, [np.array(points)], color=colour)
# save the mask
name, _ = os.path.splitext(image_name)
out_name = name + "_mask.png"
print(f"Saving {os.path.join(mask_folder_path, out_name)}")
cv2.imwrite(os.path.join(mask_folder_path, out_name), mask)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
'--data_folder_path',
default=None,
type=str,
help='Path to the data directory, where to save the outputs.')
data_folder_path = parser.parse_args().data_folder_path
data_folder_path = (data_folder_path
if data_folder_path else paths.DATA_FOLDER_PATH)
mask_folder_path = os.path.join(data_folder_path, 'masks')
colour_mapping = mapping.get_colour_mapping(data_folder_path)
files_train, files_valid = calculate_split(mask_folder_path,
colour_mapping)
data_train = [{
"image_name": os.path.join("images", name[:-9] + ".JPG"),
"mask_name": os.path.join("masks", name)
} for name in files_train]
data_val = [{
"image_name": os.path.join("images", name[:-9] + ".JPG"),
"mask_name": os.path.join("masks", name)
} for name in files_valid]
data = data_train + data_val