def main():
gt = sorted(find_in_dir('c:\\Develop\\data\\inria\\train\\gt'))
pred = sorted(
find_in_dir('runs\\Apr10_23_47_hdfpn_resnext50_finetune\\evaluation'))
print(len(gt), len(pred))
thresholds, ious = optimize_threshold(gt, pred)
for t, iou in zip(thresholds, ious):
print(t, iou)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-m", "--model", type=str, default="unet", help="")
parser.add_argument("-dd",
"--data-dir",
type=str,
default=None,
required=True,
help="Data dir")
parser.add_argument(
"-c",
"--checkpoint",
type=str,
default=None,
required=True,
help="Checkpoint filename to use as initial model weights",
)
parser.add_argument("-b",
"--batch-size",
type=int,
default=16,
help="Batch size for inference")
parser.add_argument("-tta",
"--tta",
default=None,
type=str,
help="Type of TTA to use [fliplr, d4]")
args = parser.parse_args()
data_dir = args.data_dir
checkpoint_file = auto_file(args.checkpoint)
run_dir = os.path.dirname(os.path.dirname(checkpoint_file))
out_dir = os.path.join(run_dir, "submit")
os.makedirs(out_dir, exist_ok=True)
checkpoint = load_checkpoint(checkpoint_file)
checkpoint_epoch = checkpoint["epoch"]
print("Loaded model weights from", args.checkpoint)
print("Epoch :", checkpoint_epoch)
print(
"Metrics (Train):",
"IoU:",
checkpoint["epoch_metrics"]["train"]["jaccard"],
"Acc:",
checkpoint["epoch_metrics"]["train"]["accuracy"],
)
print(
"Metrics (Valid):",
"IoU:",
checkpoint["epoch_metrics"]["valid"]["jaccard"],
"Acc:",
checkpoint["epoch_metrics"]["valid"]["accuracy"],
)
model = get_model(args.model)
unpack_checkpoint(checkpoint, model=model)
# threshold = checkpoint["epoch_metrics"]["valid"].get("optimized_jaccard/threshold", 0.5)
threshold = 0.5
print("Using threshold", threshold)
model = nn.Sequential(PickModelOutput(model, OUTPUT_MASK_KEY),
nn.Sigmoid())
if args.tta == "fliplr":
model = TTAWrapper(model, fliplr_image2mask)
elif args.tta == "flipscale":
model = TTAWrapper(model, fliplr_image2mask)
model = MultiscaleTTAWrapper(model, size_offsets=[-128, -64, 64, 128])
elif args.tta == "d4":
model = TTAWrapper(model, d4_image2mask)
else:
pass
model = model.cuda()
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
model = model.eval()
mask = predict(model,
read_inria_image("sample_color.jpg"),
image_size=(512, 512),
batch_size=args.batch_size)
mask = ((mask > threshold) * 255).astype(np.uint8)
name = os.path.join(run_dir, "sample_color.jpg")
cv2.imwrite(name, mask)
test_predictions_dir = os.path.join(out_dir, "test_predictions")
test_predictions_dir_compressed = os.path.join(
out_dir, "test_predictions_compressed")
if args.tta is not None:
test_predictions_dir += f"_{args.tta}"
test_predictions_dir_compressed += f"_{args.tta}"
os.makedirs(test_predictions_dir, exist_ok=True)
os.makedirs(test_predictions_dir_compressed, exist_ok=True)
test_images = find_in_dir(os.path.join(data_dir, "test", "images"))
for fname in tqdm(test_images, total=len(test_images)):
image = read_inria_image(fname)
mask = predict(model,
image,
image_size=(512, 512),
batch_size=args.batch_size)
mask = ((mask > threshold) * 255).astype(np.uint8)
name = os.path.join(test_predictions_dir, os.path.basename(fname))
cv2.imwrite(name, mask)
name_compressed = os.path.join(test_predictions_dir_compressed,
os.path.basename(fname))
command = (
"gdal_translate --config GDAL_PAM_ENABLED NO -co COMPRESS=CCITTFAX4 -co NBITS=1 "
+ name + " " + name_compressed)
subprocess.call(command, shell=True)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-m', '--model', type=str, default='unet', help='')
parser.add_argument('-dd',
'--data-dir',
type=str,
default=None,
required=True,
help='Data dir')
parser.add_argument(
'-c',
'--checkpoint',
type=str,
default=None,
required=True,
help='Checkpoint filename to use as initial model weights')
parser.add_argument('-b',
'--batch-size',
type=int,
default=16,
help='Batch size for inference')
parser.add_argument('-tta',
'--tta',
default=None,
type=str,
help='Type of TTA to use [fliplr, d4]')
args = parser.parse_args()
data_dir = args.data_dir
checkpoint_file = auto_file(args.checkpoint)
run_dir = os.path.dirname(os.path.dirname(checkpoint_file))
out_dir = os.path.join(run_dir, 'evaluation')
os.makedirs(out_dir, exist_ok=True)
model = get_model(args.model)
checkpoint = UtilsFactory.load_checkpoint(checkpoint_file)
checkpoint_epoch = checkpoint['epoch']
print('Loaded model weights from', args.checkpoint)
print('Epoch :', checkpoint_epoch)
print('Metrics (Train):', 'IoU:',
checkpoint['epoch_metrics']['train']['jaccard'], 'Acc:',
checkpoint['epoch_metrics']['train']['accuracy'])
print('Metrics (Valid):', 'IoU:',
checkpoint['epoch_metrics']['valid']['jaccard'], 'Acc:',
checkpoint['epoch_metrics']['valid']['accuracy'])
UtilsFactory.unpack_checkpoint(checkpoint, model=model)
model = model.cuda().eval()
train_images = find_in_dir(os.path.join(data_dir, 'train', 'images'))
for fname in tqdm(train_images, total=len(train_images)):
image = read_rgb_image(fname)
mask = predict(model,
image,
tta=args.tta,
image_size=(512, 512),
batch_size=args.batch_size,
activation='sigmoid')
mask = (mask * 255).astype(np.uint8)
name = os.path.join(out_dir, os.path.basename(fname))
cv2.imwrite(name, mask)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("predictions", nargs="+")
parser.add_argument("-w", "--workers", type=int, default=0, help="")
parser.add_argument("-dd",
"--data-dir",
type=str,
default="data",
help="Data directory")
args = parser.parse_args()
targets = fs.find_in_dir(os.path.join(
args.data_dir, "tier3", "masks")) + fs.find_in_dir(
os.path.join(args.data_dir, "train", "masks"))
targets_post = dict((fs.id_from_fname(fname), fname) for fname in targets
if "_post_" in fname)
df = defaultdict(list)
postprocessings = {
"naive": make_predictions_naive,
"dominant": make_predictions_dominant,
"floodfill": make_predictions_floodfill,
}
for predictions_dir in args.predictions:
try:
prediction_files = fs.find_in_dir(predictions_dir)
prediction_files_post = dict((fs.id_from_fname(fname), fname)
for fname in prediction_files
if "_post_" in fname)
y_true_filenames = [
targets_post[image_id_post]
for image_id_post in prediction_files_post.keys()
]
y_pred_filenames = [
prediction_files_post[image_id_post]
for image_id_post in prediction_files_post.keys()
]
for name, fn in postprocessings.items():
score, localization_f1, damage_f1, damage_f1s = optimize_postprocessing(
y_pred_filenames,
y_true_filenames,
postprocessing_fn=fn,
workers=args.workers)
print(name, score)
df["samples"].append(len(y_pred_filenames))
df["predictions_dir"].append(predictions_dir)
df["postprocessing"].append(name)
df["score"].append(score)
df["localization_f1"].append(localization_f1)
df["damage_f1"].append(damage_f1)
except Exception as e:
print("Failed to process", predictions_dir, e)
df = pd.DataFrame.from_dict(df)
print(df)
current_time = datetime.now().strftime("%b%d_%H_%M")
df.to_csv(f"postprocessing_eval_{current_time}.csv", index=None)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("checkpoints", nargs="+")
parser.add_argument("-w", "--workers", type=int, default=1, help="")
parser.add_argument("-dd", "--data-dir", type=str, default="data", help="Data directory")
parser.add_argument("-a", "--activation", type=str, default="pre", help="")
args = parser.parse_args()
targets = fs.find_in_dir(os.path.join(args.data_dir, "tier3", "masks")) + fs.find_in_dir(
os.path.join(args.data_dir, "train", "masks")
)
targets_post = dict((fs.id_from_fname(fname), fname) for fname in targets if "_post_" in fname)
df = defaultdict(list)
current_time = datetime.now().strftime("%b%d_%H_%M")
print("Checkpoints ", args.checkpoints)
print("Activation ", args.activation)
for model_checkpoint in args.checkpoints:
model_checkpoint = fs.auto_file(model_checkpoint)
predictions_dir = os.path.join(
os.path.dirname(model_checkpoint), fs.id_from_fname(model_checkpoint) + "_oof_predictions"
)
prediction_files = fs.find_in_dir(predictions_dir)
prediction_files_post = dict(
(fs.id_from_fname(fname), fname) for fname in prediction_files if "_post_" in fname
)
y_true_filenames = [targets_post[image_id_post] for image_id_post in prediction_files_post.keys()]
y_pred_filenames = [prediction_files_post[image_id_post] for image_id_post in prediction_files_post.keys()]
rounder = OptimizedRounder(workers=args.workers, apply_softmax=args.activation)
raw_score, raw_localization_f1, raw_damage_f1, raw_damage_f1s = rounder.predict(
y_pred_filenames, y_true_filenames, np.array([1, 1, 1, 1, 1], dtype=np.float32)
)
rounder.fit(y_pred_filenames, y_true_filenames)
score, localization_f1, damage_f1, damage_f1s = rounder.predict(
y_pred_filenames, y_true_filenames, rounder.coefficients()
)
print(rounder.coefficients())
df["checkpoint"].append(fs.id_from_fname(model_checkpoint))
df["coefficients"].append(rounder.coefficients())
df["samples"].append(len(y_true_filenames))
df["raw_score"].append(raw_score)
df["raw_localization"].append(raw_localization_f1)
df["raw_damage"].append(raw_damage_f1)
df["opt_score"].append(score)
df["opt_localization"].append(localization_f1)
df["opt_damage"].append(damage_f1)
dataframe = pd.DataFrame.from_dict(df)
dataframe.to_csv(f"optimized_weights_{current_time}.csv", index=None)
print(df)
import os
import numpy as np
from pytorch_toolbelt.utils import fs
from scipy.stats import spearmanr
import matplotlib.pyplot as plt
from sklearn.metrics import ConfusionMatrixDisplay
import pandas as pd
submissions = [x for x in fs.find_in_dir(".") if str.endswith(x, ".csv")]
names = list(map(lambda x: fs.id_from_fname(x)[:32], submissions))
submissions = [pd.read_csv(x).sort_values(by="Id").reset_index() for x in submissions]
cm = np.zeros((len(submissions), len(submissions)))
for i in range(len(submissions)):
for j in range(len(submissions)):
cm[i, j] = spearmanr(submissions[i].Label, submissions[j].Label).correlation
print(cm)
plt.figure(figsize=(10 + len(submissions), 10 + len(submissions)))
disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=names)
disp.plot(include_values=True, cmap="Blues", ax=plt.gca(), xticks_rotation="45")
plt.tight_layout()
plt.show()
def convert_dir(dir, folder):
jsons_dir = os.path.join(dir, "labels")
masks_dir = os.path.join(dir, "masks")
os.makedirs(masks_dir, exist_ok=True)
jsons = [
fname for fname in fs.find_in_dir(jsons_dir) if fname.endswith(".json")
]
items = []
for json_fname in tqdm(jsons):
mask_fname = os.path.join(masks_dir,
fs.id_from_fname(json_fname) + ".png")
mask = create_inference_image(json_fname, mask_fname)
non_damaged_mask = mask == 1
light = mask == 2
medium = mask == 3
destroyed = mask == 4
non_damaged_pixels = non_damaged_mask.sum()
light_pixels = light.sum()
medium_pixels = medium.sum()
destroyed_pixels = destroyed.sum()
# guatemala-volcano_00000000_post_disaster
event_name, sample_id, event_type, disaster = fs.id_from_fname(
json_fname).split("_")
assert disaster == "disaster"
image_id = fs.id_from_fname(json_fname)
items.append({
"image_fname":
os.path.join(folder, "images", image_id + ".png"),
"mask_fname":
os.path.join(folder, "masks", image_id + ".png"),
"folder":
folder,
"image_id":
image_id,
"event_name":
event_name,
"sample_id":
sample_id,
"event_type":
event_type,
"non_damaged_pixels":
non_damaged_pixels,
"light_damaged_pixels":
light_pixels,
"medium_damaged_pixels":
medium_pixels,
"destroyed_pixels":
destroyed_pixels,
"non_damaged_buildings":
label(non_damaged_mask, return_num=True)[1],
"light_damaged_buildings":
label(light, return_num=True)[1],
"medium_damaged_buildings":
label(medium, return_num=True)[1],
"destroyed_buildings":
label(destroyed, return_num=True)[1],
})
return items
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-m", "--model", type=str, default="unet", help="")
parser.add_argument("-dd",
"--data-dir",
type=str,
default=None,
required=True,
help="Data dir")
parser.add_argument(
"-c",
"--checkpoint",
type=str,
default=None,
required=True,
help="Checkpoint filename to use as initial model weights",
)
parser.add_argument("-b",
"--batch-size",
type=int,
default=16,
help="Batch size for inference")
parser.add_argument("-tta",
"--tta",
default=None,
type=str,
help="Type of TTA to use [fliplr, d4]")
args = parser.parse_args()
data_dir = args.data_dir
checkpoint_file = auto_file(args.checkpoint)
run_dir = os.path.dirname(os.path.dirname(checkpoint_file))
out_dir = os.path.join(run_dir, "evaluation")
os.makedirs(out_dir, exist_ok=True)
model = get_model(args.model)
checkpoint = UtilsFactory.load_checkpoint(checkpoint_file)
checkpoint_epoch = checkpoint["epoch"]
print("Loaded model weights from", args.checkpoint)
print("Epoch :", checkpoint_epoch)
print(
"Metrics (Train):",
"IoU:",
checkpoint["epoch_metrics"]["train"]["jaccard"],
"Acc:",
checkpoint["epoch_metrics"]["train"]["accuracy"],
)
print(
"Metrics (Valid):",
"IoU:",
checkpoint["epoch_metrics"]["valid"]["jaccard"],
"Acc:",
checkpoint["epoch_metrics"]["valid"]["accuracy"],
)
UtilsFactory.unpack_checkpoint(checkpoint, model=model)
model = model.cuda().eval()
train_images = find_in_dir(os.path.join(data_dir, "train", "images"))
for fname in tqdm(train_images, total=len(train_images)):
image = read_inria_rgb(fname)
mask = predict(model,
image,
tta=args.tta,
image_size=(512, 512),
batch_size=args.batch_size,
activation="sigmoid")
mask = (mask * 255).astype(np.uint8)
name = os.path.join(out_dir, os.path.basename(fname))
cv2.imwrite(name, mask)