def run_dual_inference_on_dataset_oof(model,
dataset,
output_dir,
batch_size=1,
workers=0):
model = model.cuda()
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
model = model.eval()
data_loader = DataLoader(dataset,
batch_size=batch_size,
pin_memory=True,
num_workers=workers)
os.makedirs(output_dir, exist_ok=True)
for batch in tqdm(data_loader):
image_pre = batch[INPUT_IMAGE_PRE_KEY].cuda(non_blocking=True)
image_post = batch[INPUT_IMAGE_POST_KEY].cuda(non_blocking=True)
image_ids = batch[INPUT_IMAGE_ID_KEY]
output = model(image_pre=image_pre, image_post=image_post)
masks_pre = output[OUTPUT_MASK_PRE_KEY]
if masks_pre.size(2) != 1024 or masks_pre.size(3) != 1024:
masks_pre = F.interpolate(masks_pre,
size=(1024, 1024),
mode="bilinear",
align_corners=False)
masks_pre = to_numpy(masks_pre.squeeze(1)).astype(np.float32)
masks_post = output[OUTPUT_MASK_POST_KEY]
if masks_post.size(2) != 1024 or masks_post.size(3) != 1024:
masks_post = F.interpolate(masks_post,
size=(1024, 1024),
mode="bilinear",
align_corners=False)
masks_post = to_numpy(masks_post).astype(np.float32)
for i, image_id in enumerate(image_ids):
localization_image = masks_pre[i]
damage_image = masks_post[i]
localization_fname = os.path.join(
output_dir, fs.change_extension(image_id, ".npy"))
np.save(localization_fname, localization_image)
damage_fname = os.path.join(
output_dir,
fs.change_extension(image_id.replace("_pre", "_post"), ".npy"))
np.save(damage_fname, damage_image)
del data_loader
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-dd",
"--data-dir",
type=str,
default=os.environ.get("KAGGLE_2020_ALASKA2"))
args = parser.parse_args()
data_dir = args.data_dir
cover = fs.find_images_in_dir(os.path.join(data_dir, "Cover"))
jimi = fs.find_images_in_dir(os.path.join(data_dir, "JMiPOD"))
juni = fs.find_images_in_dir(os.path.join(data_dir, "JUNIWARD"))
uerd = fs.find_images_in_dir(os.path.join(data_dir, "UERD"))
for cover_fname, jimi_fname, juni_fname, uerd_fname in zip(
tqdm(cover), jimi, juni, uerd):
cover = decode_bgr_from_dct(fs.change_extension(cover_fname, ".npz"))
jimi = decode_bgr_from_dct(fs.change_extension(jimi_fname, ".npz"))
juni = decode_bgr_from_dct(fs.change_extension(juni_fname, ".npz"))
uerd = decode_bgr_from_dct(fs.change_extension(uerd_fname, ".npz"))
jimi_mask = block8_sum(np.abs(cover - jimi).sum(axis=2)) > 0
juni_mask = block8_sum(np.abs(cover - juni).sum(axis=2)) > 0
uerd_mask = block8_sum(np.abs(cover - uerd).sum(axis=2)) > 0
cover_mask = jimi_mask | juni_mask | uerd_mask
cv2.imwrite(fs.change_extension(cover_fname, ".png"), cover_mask * 255)
cv2.imwrite(fs.change_extension(jimi_fname, ".png"), jimi_mask * 255)
cv2.imwrite(fs.change_extension(juni_fname, ".png"), juni_mask * 255)
cv2.imwrite(fs.change_extension(uerd_fname, ".png"), uerd_mask * 255)
def compute_mean_std(dataset):
"""
https://stats.stackexchange.com/questions/25848/how-to-sum-a-standard-deviation
"""
global_mean = np.zeros(3, dtype=np.float64)
global_var = np.zeros(3, dtype=np.float64)
n_items = 0
for image_fname in dataset:
dct_file = np.load(fs.change_extension(image_fname, ".npz"))
# This normalization roughly puts values into zero mean and unit variance
y = idct8v2(dct_file["dct_y"])
cb = idct8v2(dct_file["dct_cb"])
cr = idct8v2(dct_file["dct_cr"])
global_mean[0] += y.mean()
global_mean[1] += cb.mean()
global_mean[2] += cr.mean()
global_var[0] += y.std()**2
global_var[1] += cb.std()**2
global_var[2] += cr.std()**2
n_items += 1
return global_mean / n_items, np.sqrt(global_var / n_items)
def compute_mean_std(dataset):
"""
https://stats.stackexchange.com/questions/25848/how-to-sum-a-standard-deviation
"""
# global_mean = np.zeros((3 * 64), dtype=np.float64)
# global_var = np.zeros((3 * 64), dtype=np.float64)
n_items = 0
s = RunningStatistics()
for image_fname in dataset:
dct_file = np.load(fs.change_extension(image_fname, ".npz"))
y = torch.from_numpy(dct_file["dct_y"])
cb = torch.from_numpy(dct_file["dct_cb"])
cr = torch.from_numpy(dct_file["dct_cr"])
dct = torch.stack([y, cb, cr], dim=0).unsqueeze(0).float()
dct = sd2(dct)[0]
s.update(dct)
# dct = to_numpy()
# global_mean += dct.mean(axis=(1, 2))
# global_var += dct.std(axis=(1, 2)) ** 2
# n_items += 1
return s.mean, s.std
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-dd",
"--data-dir",
type=str,
default=os.environ.get("KAGGLE_2020_ALASKA2"))
args = parser.parse_args()
data_dir = args.data_dir
test_dir = os.path.join(data_dir, "Test")
dataset = fs.find_images_in_dir(test_dir)
# dataset = dataset[:500]
df = defaultdict(list)
for image_fname in tqdm(dataset):
dct_fname = fs.change_extension(image_fname, ".npz")
dct_data = np.load(dct_fname)
qm0 = dct_data["qm0"]
qm1 = dct_data["qm1"]
qf = quality_factror_from_qm(qm0)
fsize = os.stat(image_fname).st_size
df["image_id"].append(os.path.basename(image_fname))
df["quality"].append(qf)
df["qm0"].append(qm0.flatten().tolist())
df["qm1"].append(qm1.flatten().tolist())
df["file_size"].append(fsize)
df = pd.DataFrame.from_dict(df)
df.to_csv("test_dataset_qf_qt.csv", index=False)
def compute_statistics(cover_fname):
results_df = defaultdict(list)
# cover_dct = np.load(fs.change_extension(cover_fname, ".npz"))
cover = read_from_dct(cover_fname)
for method_name in ["JMiPOD", "JUNIWARD", "UERD"]:
stego_fname = cover_fname.replace("Cover", method_name)
stego = read_from_dct(stego_fname)
mask_fname = fs.change_extension(stego_fname, ".png")
mask = compute_mask(cover, stego)
results_df["image"].append(os.path.basename(cover_fname))
results_df["method"].append(os.path.basename(method_name))
results_df["pd"].append(count_pixel_difference(cover, stego))
cv2.imwrite(mask_fname, ((mask > 0) * 255).astype(np.uint8))
# stego_dct = np.load(fs.change_extension(stego_fname, ".npz"))
# dct_y, dct_cr, dct_cb = count_dct_difference(cover_dct, stego_dct)
# results_df["dct_total"].append(dct_y + dct_cr + dct_cb)
# results_df["dct_y"].append(dct_y)
# results_df["dct_cr"].append(dct_cr)
# results_df["dct_cb"].append(dct_cb)
#
# dct_y, dct_cr, dct_cb = count_dct_difference_bits(cover_dct, stego_dct)
# results_df["dct_bits_total"].append(dct_y + dct_cr + dct_cb)
# results_df["dct_bits_y"].append(dct_y)
# results_df["dct_bits_cr"].append(dct_cr)
# results_df["dct_bits_cb"].append(dct_cb)
return results_df
def main():
parser = argparse.ArgumentParser()
parser.add_argument('input', nargs='+')
parser.add_argument('--need-features', action='store_true')
parser.add_argument('-b',
'--batch-size',
type=int,
default=multiprocessing.cpu_count(),
help='Batch Size during training, e.g. -b 64')
parser.add_argument('-w', '--workers', type=int, default=4, help='')
args = parser.parse_args()
need_features = args.need_features
batch_size = args.batch_size
num_workers = args.workers
checkpoint_fname = args.input # pass just single checkpoint filename as arg
'''Not Changing variables'''
data_dir = '/opt/ml/code/'
checkpoint_path = os.path.join(data_dir, 'model', checkpoint_fname)
current_milli_time = lambda: str(round(time.time() * 1000))
if torch.cuda.is_available():
checkpoint = torch.load(checkpoint_path)
else:
checkpoint = torch.load(checkpoint_path,
map_location=lambda storage, loc: storage)
params = checkpoint['checkpoint_data']['cmd_args']
# Make OOF predictions
images_dir = os.path.join(data_dir, "ratinopathy", current_milli_time())
retino = pd.read_csv(os.path.join(data_dir, 'aptos-2019', 'test.csv'))
'''Downloading fundus photography files'''
for id_code in retino['id_code']:
download_from_s3(s3_filename="aptos-2019/train.csv",
local_path=os.path.join(images_dir, id_code))
image_paths = retino['id_code'].apply(
lambda x: image_with_name_in_dir(images_dir, x))
# Now run inference on Aptos2019 public test, will return a pd.DataFrame having image_id, logits, regrssions, ordinal, features
ratinopathy = run_model_inference(checkpoint=checkpoint,
params=params,
apply_softmax=True,
need_features=need_features,
retino=retino,
image_paths=image_paths,
batch_size=batch_size,
tta='fliplr',
workers=num_workers,
crop_black=True)
ratinopathy.to_pickle(
fs.change_extension(checkpoint_fname, '_ratinopathy_predictions.pkl'))
def csv_from_excel(fname, sheet_name='Feuil1') -> pd.DataFrame:
wb = xlrd.open_workbook(fname)
sh = wb.sheet_by_name(sheet_name)
csv_file = change_extension(fname, '.csv')
your_csv_file = open(csv_file, 'w', encoding='utf-8')
wr = csv.writer(your_csv_file, quoting=csv.QUOTE_ALL)
for rownum in range(sh.nrows):
wr.writerow(sh.row_values(rownum))
your_csv_file.close()
return pd.read_csv(csv_file)
def compute_mean_std(dataset):
"""
https://stats.stackexchange.com/questions/25848/how-to-sum-a-standard-deviation
"""
# global_mean = np.zeros((3 * 64), dtype=np.float64)
# global_var = np.zeros((3 * 64), dtype=np.float64)
n_items = 0
s = RunningStatistics()
for image_fname in dataset:
dct_file = fs.change_extension(image_fname, ".npz")
residual = compute_decoding_residual(cv2.imread(image_fname), dct_file)
s.update(torch.from_numpy(residual).permute(2, 0, 1))
return s.mean, s.std
def get_predictions(models: List[str], datasets: List[str]) -> List[str]:
models_predictions = []
for dataset in datasets:
assert dataset in {
'aptos2015_test_private', 'aptos2015_test_public',
'aptos2015_train', 'aptos2019_test', 'messidor2_train',
'idrid_test'
}
for model_name in models:
if model_name in MODELS:
model_checkpoints = MODELS[model_name] # Well-known models
else:
model_checkpoints = [model_name] # Random stuff
for model_checkpoint in model_checkpoints:
predictions = fs.change_extension(
model_checkpoint, f'_{dataset}_predictions.pkl')
models_predictions.append(predictions)
return models_predictions
def compute_features_proc(image_fname):
dct_file = fs.change_extension(image_fname, ".npz")
image = 2 * (decode_bgr_from_dct(dct_file) / 140 - 0.5)
entropy_per_channel = [
entropy(image[..., 0].flatten()),
entropy(image[..., 1].flatten()),
entropy(image[..., 2].flatten()),
]
f = [
image[..., 0].mean(),
image[..., 1].mean(),
image[..., 2].mean(),
image[..., 0].std(),
image[..., 1].std(),
image[..., 2].std(),
entropy_per_channel[0],
entropy_per_channel[1],
entropy_per_channel[2],
]
return f
def get_predictions_csv(experiment,
metric: str,
type: str,
tta: str = None,
need_embedding=False):
if isinstance(experiment, list):
return [
get_predictions_csv(x,
metric=metric,
type=type,
tta=tta,
need_embedding=need_embedding)
for x in experiment
]
embedding_suffix = "_w_emb" if need_embedding else ""
assert type in {"test", "holdout", "oof", "train"}
assert metric in {"loss", "bauc", "cauc"}
assert tta in {None, "d4", "hv"}
checkpoints_dir = {
"loss": "checkpoints",
"bauc": "checkpoints_auc",
"cauc": "checkpoints_auc_classifier"
}[metric]
csv = os.path.join("models", experiment, "main", checkpoints_dir,
f"best_{type}_predictions{embedding_suffix}.csv")
if tta == "d4":
csv = as_d4_tta([csv])[0]
elif tta == "hv":
csv = as_hv_tta([csv])[0]
if need_embedding:
csv = fs.change_extension(csv, ".pkl")
return csv
def as_d4_tta(predictions):
return [fs.change_extension(x, "_d4_tta.csv") for x in predictions]
def main():
output_dir = os.path.dirname(__file__)
checksum = "DCTR_JRM_B4_B5_B6_MixNet_XL_SRNET"
columns = [
# "DCTR",
# "JRM",
# "MixNet_xl_pc",
# "MixNet_xl_pjm",
# "MixNet_xl_pjuni",
# "MixNet_xl_puerd",
# "efn_b4_pc",
# "efn_b4_pjm",
# "efn_b4_pjuni",
# "efn_b4_puerd",
# "efn_b2_pc",
# "efn_b2_pjm",
# "efn_b2_pjuni",
# "efn_b2_puerd",
# "MixNet_s_pc",
# "MixNet_s_pjm",
# "MixNet_s_pjuni",
# "MixNet_s_puerd",
# "SRNet_pc",
# "SRNet_pjm",
# "SRNet_pjuni",
# "SRNet_puerd",
# "SRNet_noPC70_pc",
# "SRNet_noPC70_pjm",
# "SRNet_noPC70_pjuni",
# "SRNet_noPC70_puerd",
"efn_b4_mish_pc",
"efn_b4_mish_pjm",
"efn_b4_mish_pjuni",
"efn_b4_mish_puerd",
"efn_b5_mish_pc",
"efn_b5_mish_pjm",
"efn_b5_mish_pjuni",
"efn_b5_mish_puerd",
# "efn_b2_NR_mish_pc",
# "efn_b2_NR_mish_pjm",
# "efn_b2_NR_mish_pjuni",
# "efn_b2_NR_mish_puerd",
"MixNet_xl_mish_pc",
"MixNet_xl_mish_pjm",
"MixNet_xl_mish_pjuni",
"MixNet_xl_mish_puerd",
"efn_b6_NR_mish_pc",
"efn_b6_NR_mish_pjm",
"efn_b6_NR_mish_pjuni",
"efn_b6_NR_mish_puerd",
"SRNet_noPC70_mckpt_pc",
"SRNet_noPC70_mckpt_pjm",
"SRNet_noPC70_mckpt_pjuni",
"SRNet_noPC70_mckpt_puerd",
]
x, y, quality_h, image_ids = get_x_y_for_stacking(
"probabilities_zoo_holdout_0718.csv", columns)
print(x.shape, y.shape)
x_test, _, quality_t, image_ids_test = get_x_y_for_stacking(
"probabilities_zoo_lb_0718.csv", columns)
print(x_test.shape)
if True:
sc = StandardScaler()
x = sc.fit_transform(x)
x_test = sc.transform(x_test)
if False:
sc = PCA(n_components=16)
x = sc.fit_transform(x)
x_test = sc.transform(x_test)
if True:
quality_h = F.one_hot(torch.tensor(quality_h).long(),
3).numpy().astype(np.float32)
quality_t = F.one_hot(torch.tensor(quality_t).long(),
3).numpy().astype(np.float32)
x = np.column_stack([x, quality_h])
x_test = np.column_stack([x_test, quality_t])
group_kfold = GroupKFold(n_splits=5)
params = {
"booster": ["gbtree", "gblinear"],
"min_child_weight": [1, 5, 10],
# L2 reg
"lambda": [0, 0.01, 0.1, 1],
# L1 reg
"alpha": [0, 0.01, 0.1, 1],
"subsample": [0.6, 0.8, 1.0],
"colsample_bytree": [0.6, 0.8, 1.0],
"max_depth": [2, 3, 4, 5, 6],
"n_estimators": [16, 32, 64, 128, 256, 1000],
"learning_rate": [0.001, 0.01, 0.05, 0.2, 1],
}
xgb = XGBClassifier(objective="binary:logistic", gamma=1e-4, nthread=1)
random_search = RandomizedSearchCV(
xgb,
param_distributions=params,
scoring=make_scorer(alaska_weighted_auc,
greater_is_better=True,
needs_proba=True),
n_jobs=4,
n_iter=100,
cv=group_kfold.split(x, y, groups=image_ids),
verbose=3,
random_state=42,
)
# Here we go
random_search.fit(x, y)
print("\n All results:")
print(random_search.cv_results_)
print("\n Best estimator:")
print(random_search.best_estimator_)
print(random_search.best_score_)
print("\n Best hyperparameters:")
print(random_search.best_params_)
results = pd.DataFrame(random_search.cv_results_)
results.to_csv("xgb-2-random-grid-search-results-01.csv", index=False)
test_pred = random_search.predict_proba(x_test)[:, 1]
submit_fname = os.path.join(
output_dir,
f"xgb_cls2_gs_{random_search.best_score_:.4f}_{checksum}_.csv")
df = {}
df["Label"] = test_pred
df["Id"] = image_ids_test
pd.DataFrame.from_dict(df).to_csv(submit_fname, index=False)
print("Saved submission to ", submit_fname)
import json
with open(fs.change_extension(submit_fname, ".json"), "w") as f:
json.dump(random_search.best_params_, f, indent=2)
print("Features importance")
print(random_search.best_estimator_.feature_importances_)
def convert_dir(df: pd.DataFrame, dir) -> pd.DataFrame:
crops_dir = os.path.join(dir, "crops")
os.makedirs(crops_dir, exist_ok=True)
building_crops = []
global_crop_index = 0
for i, row in tqdm(df.iterrows(), total=len(df)):
image_fname_pre = read_image(os.path.join(dir, row["image_fname_pre"]))
image_fname_post = read_image(
os.path.join(dir, row["image_fname_post"]))
mask_fname_post = row["mask_fname_post"]
json_fname_post = fs.change_extension(
mask_fname_post.replace("masks", "labels"), ".json")
inference_data = open_json(os.path.join(dir, json_fname_post))
instance_image, labels = create_instance_image(inference_data)
for label_index, damage_label in zip(
range(1,
instance_image.max() + 1), labels):
try:
instance_mask = instance_image == label_index
rmin, rmax, cmin, cmax = bbox1(instance_mask)
max_size = max(rmax - rmin, cmax - cmin)
if max_size < 16:
print("Skipping crop since it's too small",
fs.id_from_fname(mask_fname_post), "label_index",
label_index, "min_size", max_size)
continue
rpadding = (rmax - rmin) // 4
cpadding = (cmax - cmin) // 4
pre_crop = image_fname_pre[max(0, rmin -
rpadding):rmax + rpadding,
max(0, cmin - cpadding):cmax +
cpadding]
post_crop = image_fname_post[max(0, rmin -
rpadding):rmax + rpadding,
max(0, cmin - cpadding):cmax +
cpadding]
image_id_pre = row["image_id_pre"]
image_id_post = row["image_id_post"]
pre_crop_fname = f"{global_crop_index:06}_{image_id_pre}.png"
post_crop_fname = f"{global_crop_index:06}_{image_id_post}.png"
global_crop_index += 1
cv2.imwrite(os.path.join(crops_dir, pre_crop_fname), pre_crop)
cv2.imwrite(os.path.join(crops_dir, post_crop_fname),
post_crop)
building_crops.append({
"pre_crop_fname": pre_crop_fname,
"post_crop": post_crop_fname,
"label": damage_label,
"event_name": row["event_name_post"],
"fold": row["fold_post"],
"rmin": rmin,
"rmax": rmax,
"cmin": cmin,
"cmax": cmax,
"max_size": max_size,
"rpadding": rpadding,
"cpadding": cpadding
})
except Exception as e:
print(e)
print(mask_fname_post)
df = pd.DataFrame.from_records(building_crops)
return df
def main():
# Give no chance to randomness
torch.manual_seed(0)
np.random.seed(0)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
parser = argparse.ArgumentParser()
parser.add_argument("checkpoint", type=str, nargs="+")
parser.add_argument("-dd",
"--data-dir",
type=str,
default=os.environ.get("KAGGLE_2020_ALASKA2"))
parser.add_argument("-b", "--batch-size", type=int, default=1)
parser.add_argument("-w", "--workers", type=int, default=0)
parser.add_argument("-d4", "--d4-tta", action="store_true")
parser.add_argument("-hv", "--hv-tta", action="store_true")
parser.add_argument("-f", "--force-recompute", action="store_true")
parser.add_argument("-fp16", "--fp16", action="store_true")
args = parser.parse_args()
checkpoint_fnames = args.checkpoint
data_dir = args.data_dir
batch_size = args.batch_size
workers = args.workers
fp16 = args.fp16
d4_tta = args.d4_tta
force_recompute = args.force_recompute
need_embedding = True
outputs = [
OUTPUT_PRED_MODIFICATION_FLAG, OUTPUT_PRED_MODIFICATION_TYPE,
OUTPUT_PRED_EMBEDDING
]
embedding_suffix = "_w_emb" if need_embedding else ""
for checkpoint_fname in checkpoint_fnames:
model, checkpoints, required_features = ensemble_from_checkpoints(
[checkpoint_fname],
strict=True,
outputs=outputs,
activation=None,
tta=None,
need_embedding=need_embedding)
report_checkpoint(checkpoints[0])
model = model.cuda()
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
model = model.eval()
if fp16:
model = model.half()
train_ds = get_train_except_holdout(data_dir,
features=required_features)
holdout_ds = get_holdout(data_dir, features=required_features)
test_ds = get_test_dataset(data_dir, features=required_features)
if d4_tta:
model = wrap_model_with_tta(model,
"d4",
inputs=required_features,
outputs=outputs).eval()
tta_suffix = "_d4_tta"
else:
tta_suffix = ""
# Train
trn_predictions_csv = fs.change_extension(
checkpoint_fname,
f"_train_predictions{embedding_suffix}{tta_suffix}.pkl")
if force_recompute or not os.path.exists(trn_predictions_csv):
trn_predictions = compute_trn_predictions(model,
train_ds,
fp16=fp16,
batch_size=batch_size,
workers=workers)
trn_predictions.to_pickle(trn_predictions_csv)
# Holdout
hld_predictions_csv = fs.change_extension(
checkpoint_fname,
f"_holdout_predictions{embedding_suffix}{tta_suffix}.pkl")
if force_recompute or not os.path.exists(hld_predictions_csv):
hld_predictions = compute_trn_predictions(model,
holdout_ds,
fp16=fp16,
batch_size=batch_size,
workers=workers)
hld_predictions.to_pickle(hld_predictions_csv)
# Test
tst_predictions_csv = fs.change_extension(
checkpoint_fname,
f"_test_predictions{embedding_suffix}{tta_suffix}.pkl")
if force_recompute or not os.path.exists(tst_predictions_csv):
tst_predictions = compute_trn_predictions(model,
test_ds,
fp16=fp16,
batch_size=batch_size,
workers=workers)
tst_predictions.to_pickle(tst_predictions_csv)
def main():
# Give no chance to randomness
torch.manual_seed(0)
np.random.seed(0)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
parser = argparse.ArgumentParser()
parser.add_argument("checkpoint", type=str, nargs="+")
parser.add_argument("-dd", "--data-dir", type=str, default=os.environ.get("KAGGLE_2020_ALASKA2"))
parser.add_argument("-b", "--batch-size", type=int, default=1)
parser.add_argument("-w", "--workers", type=int, default=0)
parser.add_argument("-d4", "--d4-tta", action="store_true")
parser.add_argument("-hv", "--hv-tta", action="store_true")
parser.add_argument("-f", "--force-recompute", action="store_true")
parser.add_argument("-oof", "--need-oof", action="store_true")
args = parser.parse_args()
checkpoint_fnames = args.checkpoint
data_dir = args.data_dir
batch_size = args.batch_size
workers = args.workers
d4_tta = args.d4_tta
hv_tta = args.hv_tta
force_recompute = args.force_recompute
outputs = [OUTPUT_PRED_MODIFICATION_FLAG, OUTPUT_PRED_MODIFICATION_TYPE]
for checkpoint_fname in checkpoint_fnames:
model, checkpoints, required_features = ensemble_from_checkpoints(
[checkpoint_fname], strict=True, outputs=outputs, activation=None, tta=None
)
report_checkpoint(checkpoints[0])
model = model.cuda()
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
model = model.eval()
# Holdout
variants = {
"istego100k_test_same_center_crop": get_istego100k_test_same(
data_dir, features=required_features, output_size="center_crop"
),
"istego100k_test_same_full": get_istego100k_test_same(
data_dir, features=required_features, output_size="full"
),
"istego100k_test_other_center_crop": get_istego100k_test_other(
data_dir, features=required_features, output_size="center_crop"
),
"istego100k_test_other_full": get_istego100k_test_other(
data_dir, features=required_features, output_size="full"
),
"holdout": get_holdout("d:\datasets\ALASKA2", features=required_features),
}
for name, dataset in variants.items():
print("Making predictions for ", name, len(dataset))
predictions_csv = fs.change_extension(checkpoint_fname, f"_{name}_predictions.csv")
if force_recompute or not os.path.exists(predictions_csv):
holdout_predictions = compute_oof_predictions(
model, dataset, batch_size=batch_size // 4 if "full" in name else batch_size, workers=workers
)
holdout_predictions.to_csv(predictions_csv, index=False)
holdout_predictions = pd.read_csv(predictions_csv)
print(name)
print(
"\tbAUC",
alaska_weighted_auc(
holdout_predictions[INPUT_TRUE_MODIFICATION_FLAG].values,
holdout_predictions[OUTPUT_PRED_MODIFICATION_FLAG].apply(sigmoid).values,
),
)
print(
"\tcAUC",
alaska_weighted_auc(
holdout_predictions[INPUT_TRUE_MODIFICATION_FLAG].values,
holdout_predictions[OUTPUT_PRED_MODIFICATION_TYPE].apply(parse_classifier_probas).values,
),
)
def main():
start = time.time()
torch.set_num_threads(1)
torch.set_num_interop_threads(1)
parser = argparse.ArgumentParser()
parser.add_argument("pre_image", type=str)
parser.add_argument("post_image", type=str)
parser.add_argument("loc_image", type=str)
parser.add_argument("dmg_image", type=str)
parser.add_argument("--raw", action="store_true")
parser.add_argument("--color-mask", action="store_true")
parser.add_argument("--gpu", action="store_true")
args = parser.parse_args()
pre_image = args.pre_image
post_image = args.post_image
localization_fname = args.loc_image
damage_fname = args.dmg_image
save_raw = args.raw
color_mask = args.color_mask
use_gpu = args.gpu
size = 1024
postprocess = "naive"
image_size = size, size
print("pre_image ", pre_image)
print("post_image ", post_image)
print("loc_image ", localization_fname)
print("dmg_image ", damage_fname)
print("Size ", image_size)
print("Postprocess ", postprocess)
print("Colorize ", color_mask)
raw_predictions_file = fs.change_extension(damage_fname, ".npy")
print("raw_predictions_file", raw_predictions_file)
print(*torch.__config__.show().split("\n"), sep="\n")
if not os.path.isdir(os.path.dirname(localization_fname)):
print("Output directory does not exists", localization_fname)
return -1
if not os.access(os.path.dirname(localization_fname), os.W_OK):
print("Output directory does not have write access",
localization_fname)
return -2
if not os.path.isdir(os.path.dirname(damage_fname)):
print("Output directory does not exists", damage_fname)
return -1
if not os.access(os.path.dirname(damage_fname), os.W_OK):
print("Output directory does not have write access", damage_fname)
return -2
fold_0_models_dict = [
# (
# "Dec15_21_41_resnet101_fpncatv2_256_512_fold0_fp16_crops.pth",
# [0.45136154, 1.4482629, 1.42098208, 0.6839698, 0.96800456],
# ),
# (
# "Dec16_08_26_resnet34_unet_v2_512_fold0_fp16_crops.pth",
# [0.92919105, 1.03831743, 1.03017048, 0.98257118, 1.0241164],
# ),
# (
# "Dec21_21_54_densenet161_deeplab256_512_fold0_fp16_crops.pth",
# [0.48157651, 1.02084685, 1.36264406, 1.03175205, 1.11758873],
# ),
# 0.762814651939279 0.854002889559006 0.7237339786736817 [0.9186602573598759, 0.5420118318644089, 0.7123870673168781, 0.8405837378060299] coeffs [0.51244243 1.42747062 1.23648384 0.90290896 0.88912514]
(
"Dec30_15_34_resnet34_unet_v2_512_fold0_fp16_pseudo_crops.pth",
[0.51244243, 1.42747062, 1.23648384, 0.90290896, 0.88912514],
),
# 0.7673669954814148 0.8582940771677703 0.7283982461872626 [0.919932857782992, 0.5413880912001547, 0.731840942842999, 0.8396640419159087] coeffs [0.50847073 1.15392272 1.2059733 1.1340391 1.03196719]
(
"Dec30_15_34_resnet101_fpncatv2_256_512_fold0_fp16_pseudo_crops.pth",
[0.50847073, 1.15392272, 1.2059733, 1.1340391, 1.03196719],
),
]
fold_1_models_dict = [
# (
# "Dec16_18_59_densenet201_fpncatv2_256_512_fold1_fp16_crops.pth",
# [0.64202075, 1.04641224, 1.23015655, 1.03203408, 1.12505602],
# ),
# (
# "Dec17_01_52_resnet34_unet_v2_512_fold1_fp16_crops.pth",
# [0.69605759, 0.89963168, 0.9232137, 0.92938775, 0.94460875],
# ),
(
"Dec22_22_24_seresnext50_unet_v2_512_fold1_fp16_crops.pth",
[0.54324459, 1.76890163, 1.20782899, 0.85128004, 0.83100698],
),
(
"Dec31_02_09_resnet34_unet_v2_512_fold1_fp16_pseudo_crops.pth",
# Maybe suboptimal
[0.48269921, 1.22874469, 1.38328066, 0.96695393, 0.91348539],
),
("Dec31_03_55_densenet201_fpncatv2_256_512_fold1_fp16_pseudo_crops.pth",
[0.48804137, 1.14809462, 1.24851827, 1.11798428, 1.00790482])
]
fold_2_models_dict = [
# (
# "Dec17_19_19_resnet34_unet_v2_512_fold2_fp16_crops.pth",
# [0.65977938, 1.50252452, 0.97098732, 0.74048182, 1.08712367],
# ),
# 0.7674290884579319 0.8107652756500724 0.7488564368041575 [0.9228529822124596, 0.5900700454049471, 0.736806959757804, 0.8292099253270483] coeffs [0.34641084 1.63486251 1.14186036 0.86668715 1.12193125]
(
"Dec17_19_12_inceptionv4_fpncatv2_256_512_fold2_fp16_crops.pth",
[0.34641084, 1.63486251, 1.14186036, 0.86668715, 1.12193125],
),
# 0.7683650436367244 0.8543981047493 0.7314937317313349 [0.9248137307721042, 0.5642011151253543, 0.7081016179096937, 0.831720163492164] coeffs [0.51277498 1.4475809 0.8296623 0.97868596 1.34180805]
(
"Dec27_14_08_densenet169_unet_v2_512_fold2_fp16_crops.pth",
[0.55429115, 1.34944309, 1.1087044, 0.89542089, 1.17257541],
),
(
"Dec31_12_45_resnet34_unet_v2_512_fold2_fp16_pseudo_crops.pth",
# Copied from Dec17_19_19_resnet34_unet_v2_512_fold2_fp16_crops
[0.65977938, 1.50252452, 0.97098732, 0.74048182, 1.08712367],
)
]
fold_3_models_dict = [
(
"Dec15_23_24_resnet34_unet_v2_512_fold3_crops.pth",
[0.84090623, 1.02953555, 1.2526516, 0.9298182, 0.94053529],
),
# (
# "Dec18_12_49_resnet34_unet_v2_512_fold3_fp16_crops.pth",
# [0.55555375, 1.18287119, 1.10997173, 0.85927596, 1.18145368],
# ),
# (
# "Dec19_14_59_efficientb4_fpncatv2_256_512_fold3_fp16_crops.pth",
# [0.59338243, 1.17347438, 1.186104, 1.06860638, 1.03041829],
# ),
(
"Dec21_11_50_seresnext50_unet_v2_512_fold3_fp16_crops.pth",
[0.43108046, 1.30222898, 1.09660616, 0.94958969, 1.07063753],
),
(
"Dec31_18_17_efficientb4_fpncatv2_256_512_fold3_fp16_pseudo_crops.pth",
# Copied from Dec19_14_59_efficientb4_fpncatv2_256_512_fold3_fp16_crops
[0.59338243, 1.17347438, 1.186104, 1.06860638, 1.03041829])
]
fold_4_models_dict = [
(
"Dec19_06_18_resnet34_unet_v2_512_fold4_fp16_crops.pth",
[0.83915734, 1.02560309, 0.77639015, 1.17487775, 1.05632771],
),
(
"Dec27_14_37_resnet101_unet_v2_512_fold4_fp16_crops.pth",
[0.57414314, 1.19599486, 1.05561912, 0.98815567, 1.2274592],
),
]
infos = []
resize = A.Resize(1024, 1024)
normalize = A.Normalize(mean=(0.485, 0.456, 0.406, 0.485, 0.456, 0.406),
std=(0.229, 0.224, 0.225, 0.229, 0.224, 0.225))
transform = A.Compose([resize, normalize])
# Very dumb way but it matches 1:1 with inferencing
pre, post = read_image(pre_image), read_image(post_image)
image = np.dstack([pre, post])
image = transform(image=image)["image"]
pre_image = image[..., 0:3]
post_image = image[..., 3:6]
models = []
for models_dict in [
fold_0_models_dict,
fold_1_models_dict,
fold_2_models_dict,
fold_3_models_dict,
fold_4_models_dict,
]:
for checkpoint, weights in models_dict:
model, info = weighted_model(checkpoint,
weights,
activation="model")
models.append(model)
infos.append(info)
model = Ensembler(models, outputs=[OUTPUT_MASK_KEY])
model = HFlipTTA(model, outputs=[OUTPUT_MASK_KEY], average=True)
model = MultiscaleTTA(model,
outputs=[OUTPUT_MASK_KEY],
size_offsets=[-128, +128],
average=True)
model = model.eval()
df = pd.DataFrame.from_records(infos)
pd.set_option("display.max_rows", None)
pd.set_option("display.max_columns", None)
pd.set_option("display.width", None)
pd.set_option("display.max_colwidth", -1)
print(df)
print("score ", df["score"].mean(), df["score"].std())
print("localization ", df["localization"].mean(), df["localization"].std())
print("damage ", df["damage"].mean(), df["damage"].std())
input_image = tensor_from_rgb_image(np.dstack([pre_image,
post_image])).unsqueeze(0)
if use_gpu:
print("Using GPU for inference")
input_image = input_image.cuda()
model = model.cuda()
output = model(input_image)
masks = output[OUTPUT_MASK_KEY]
predictions = to_numpy(masks.squeeze(0)).astype(np.float32)
if save_raw:
np.save(raw_predictions_file, predictions)
localization_image, damage_image = make_predictions_naive(predictions)
if color_mask:
localization_image = colorize_mask(localization_image)
localization_image.save(localization_fname)
damage_image = colorize_mask(damage_image)
damage_image.save(damage_fname)
else:
cv2.imwrite(localization_fname, localization_image)
cv2.imwrite(damage_fname, damage_image)
print("Saved output to ", localization_fname, damage_fname)
done = time.time()
elapsed = done - start
print("Inference time", elapsed, "(s)")
def run_inference_on_dataset_oof(model,
dataset,
output_dir,
batch_size=1,
workers=0,
save=True,
fp16=False):
model = model.cuda()
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
model = model.eval()
if fp16:
model = model.half()
data_loader = DataLoader(dataset,
batch_size=batch_size,
pin_memory=True,
num_workers=workers)
if save:
os.makedirs(output_dir, exist_ok=True)
allrows = []
for batch in tqdm(data_loader):
image = batch[INPUT_IMAGE_KEY]
if fp16:
image = image.half()
image = image.cuda(non_blocking=True)
image_ids = batch[INPUT_IMAGE_ID_KEY]
dmg_true = to_numpy(batch[INPUT_MASK_KEY]).astype(np.float32)
output = model(image)
masks = output[OUTPUT_MASK_KEY]
masks = to_numpy(masks)
for i, image_id in enumerate(image_ids):
damage_mask = masks[i]
if save:
damage_fname = os.path.join(
output_dir,
fs.change_extension(image_id.replace("_pre", "_post"),
".npy"))
np.save(damage_fname, damage_mask.astype(np.float16))
loc_pred, dmg_pred = make_predictions_naive(damage_mask)
row = CompetitionMetricCallback.get_row_pair(
loc_pred, dmg_pred, dmg_true[i], dmg_true[i])
allrows.append(row)
if save:
if DAMAGE_TYPE_KEY in output:
damage_type = to_numpy(
output[DAMAGE_TYPE_KEY].sigmoid()).astype(np.float32)
for i, image_id in enumerate(image_ids):
damage_fname = os.path.join(
output_dir,
fs.change_extension(
image_id.replace("_pre", "_damage_type"), ".npy"))
np.save(damage_fname, damage_type[i])
del data_loader
return CompetitionMetricCallback.compute_metrics(allrows)
def read_from_dct(image_fname):
return decode_bgr_from_dct(fs.change_extension(image_fname, ".npz"))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("experiments", nargs="+", type=str)
parser.add_argument("-o", "--output", type=str, required=False)
parser.add_argument("-dd",
"--data-dir",
type=str,
default=os.environ.get("KAGGLE_2020_ALASKA2"))
args = parser.parse_args()
output_dir = os.path.dirname(__file__)
data_dir = args.data_dir
experiments = args.experiments
output_file = args.output
holdout_predictions = get_predictions_csv(experiments, "cauc", "holdout",
"d4")
test_predictions = get_predictions_csv(experiments, "cauc", "test", "d4")
checksum = compute_checksum_v2(experiments)
holdout_ds = get_holdout("", features=[INPUT_IMAGE_KEY])
image_ids_h = [fs.id_from_fname(x) for x in holdout_ds.images]
quality_h = F.one_hot(torch.tensor(holdout_ds.quality).long(),
3).numpy().astype(np.float32)
test_ds = get_test_dataset("", features=[INPUT_IMAGE_KEY])
quality_t = F.one_hot(torch.tensor(test_ds.quality).long(),
3).numpy().astype(np.float32)
with_logits = True
x, y = get_x_y_for_stacking(holdout_predictions,
with_logits=with_logits,
tta_logits=with_logits)
# Force target to be binary
y = (y > 0).astype(int)
print(x.shape, y.shape)
x_test, _ = get_x_y_for_stacking(test_predictions,
with_logits=with_logits,
tta_logits=with_logits)
print(x_test.shape)
if False:
image_fnames_h = [
os.path.join(data_dir, INDEX_TO_METHOD[method], f"{image_id}.jpg")
for (image_id, method) in zip(image_ids_h, y)
]
test_image_ids = pd.read_csv(test_predictions[0]).image_id.tolist()
image_fnames_t = [
os.path.join(data_dir, "Test", image_id)
for image_id in test_image_ids
]
entropy_t = compute_image_features(image_fnames_t)
x_test = np.column_stack([x_test, entropy_t])
# entropy_h = entropy_t.copy()
# x = x_test.copy()
entropy_h = compute_image_features(image_fnames_h)
x = np.column_stack([x, entropy_h])
print("Added image features", entropy_h.shape, entropy_t.shape)
if True:
sc = StandardScaler()
x = sc.fit_transform(x)
x_test = sc.transform(x_test)
if False:
sc = PCA(n_components=16)
x = sc.fit_transform(x)
x_test = sc.transform(x_test)
if True:
x = np.column_stack([x, quality_h])
x_test = np.column_stack([x_test, quality_t])
group_kfold = GroupKFold(n_splits=5)
params = {
"min_child_weight": [1, 5, 10],
"gamma": [1e-3, 1e-2, 1e-2, 0.5, 2],
"subsample": [0.6, 0.8, 1.0],
"colsample_bytree": [0.6, 0.8, 1.0],
"max_depth": [2, 3, 4, 5, 6],
"n_estimators": [16, 32, 64, 128, 256, 1000],
"learning_rate": [0.001, 0.01, 0.05, 0.2, 1],
}
xgb = XGBClassifier(objective="binary:logistic", nthread=1)
random_search = RandomizedSearchCV(
xgb,
param_distributions=params,
scoring=make_scorer(alaska_weighted_auc,
greater_is_better=True,
needs_proba=True),
n_jobs=4,
n_iter=25,
cv=group_kfold.split(x, y, groups=image_ids_h),
verbose=3,
random_state=42,
)
# Here we go
random_search.fit(x, y)
print("\n All results:")
print(random_search.cv_results_)
print("\n Best estimator:")
print(random_search.best_estimator_)
print(random_search.best_score_)
print("\n Best hyperparameters:")
print(random_search.best_params_)
results = pd.DataFrame(random_search.cv_results_)
results.to_csv("xgb-random-grid-search-results-01.csv", index=False)
test_pred = random_search.predict_proba(x_test)[:, 1]
if output_file is None:
with_logits_sfx = "_with_logits" if with_logits else ""
submit_fname = os.path.join(
output_dir,
f"xgb_cls_gs_{random_search.best_score_:.4f}_{checksum}{with_logits_sfx}.csv"
)
else:
submit_fname = output_file
df = pd.read_csv(test_predictions[0]).rename(columns={"image_id": "Id"})
df["Label"] = test_pred
df[["Id", "Label"]].to_csv(submit_fname, index=False)
print("Saved submission to ", submit_fname)
import json
with open(fs.change_extension(submit_fname, ".json"), "w") as f:
json.dump(random_search.best_params_, f, indent=2)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('input', nargs='+')
parser.add_argument('--need-features', action='store_true')
parser.add_argument('-b',
'--batch-size',
type=int,
default=None,
help='Batch Size during training, e.g. -b 64')
parser.add_argument('-w', '--workers', type=int, default=4, help='')
args = parser.parse_args()
need_features = args.need_features
batch_size = args.batch_size
num_workers = args.workers
checkpoints = args.input
for i, checkpoint_fname in enumerate(checkpoints):
print(i, checkpoint_fname)
# Make OOF predictions
checkpoint = torch.load(checkpoint_fname)
params = checkpoint['checkpoint_data']['cmd_args']
image_size = params['size']
data_dir = params['data_dir']
# train_ds, valid_ds, train_sizes = get_datasets(data_dir=params['data_dir'],
# use_aptos2019=params['use_aptos2019'],
# use_aptos2015=params['use_aptos2015'],
# use_idrid=params['use_idrid'],
# use_messidor=params['use_messidor'],
# use_unsupervised=False,
# image_size=(image_size, image_size),
# augmentation=params['augmentations'],
# preprocessing=params['preprocessing'],
# target_dtype=int,
# coarse_grading=params.get('coarse', False),
# fold=i,
# folds=4)
# print(len(valid_ds))
# oof_predictions = run_model_inference_via_dataset(checkpoint_fname,
# valid_ds,
# apply_softmax=True,
# need_features=need_features,
# batch_size=batch_size,
# workers=num_workers)
# dst_fname = fs.change_extension(checkpoint_fname, '_oof_predictions.pkl')
# oof_predictions.to_pickle(dst_fname)
# Now run inference on holdout IDRID Test dataset
idrid_test = run_model_inference(
model_checkpoint=checkpoint_fname,
apply_softmax=True,
need_features=need_features,
test_csv=pd.read_csv(
os.path.join(data_dir, 'idrid', 'test_labels.csv')),
data_dir=os.path.join(data_dir, 'idrid'),
images_dir='test_images_768',
batch_size=batch_size,
tta='fliplr',
workers=num_workers,
crop_black=True)
idrid_test.to_pickle(
fs.change_extension(checkpoint_fname,
'_idrid_test_predictions.pkl'))
# Now run inference on Messidor 2 Test dataset
messidor2_train = run_model_inference(
model_checkpoint=checkpoint_fname,
apply_softmax=True,
need_features=need_features,
test_csv=pd.read_csv(
os.path.join(data_dir, 'messidor_2', 'train_labels.csv')),
data_dir=os.path.join(data_dir, 'messidor_2'),
images_dir='train_images_768',
batch_size=batch_size,
tta='fliplr',
workers=num_workers,
crop_black=True)
messidor2_train.to_pickle(
fs.change_extension(checkpoint_fname,
'_messidor2_train_predictions.pkl'))
# Now run inference on Aptos2019 public test
aptos2019_test = run_model_inference(
model_checkpoint=checkpoint_fname,
apply_softmax=True,
need_features=need_features,
test_csv=pd.read_csv(
os.path.join(data_dir, 'aptos-2019', 'test.csv')),
data_dir=os.path.join(data_dir, 'aptos-2019'),
images_dir='test_images_768',
batch_size=batch_size,
tta='fliplr',
workers=num_workers,
crop_black=True)
aptos2019_test.to_pickle(
fs.change_extension(checkpoint_fname,
'_aptos2019_test_predictions.pkl'))
# Now run inference on Aptos2015 private test
if True:
aptos2015_df = pd.read_csv(
os.path.join(data_dir, 'aptos-2015', 'test_labels.csv'))
aptos2015_df = aptos2015_df[aptos2015_df['Usage'] == 'Private']
aptos2015_test = run_model_inference(
model_checkpoint=checkpoint_fname,
apply_softmax=True,
need_features=need_features,
test_csv=aptos2015_df,
data_dir=os.path.join(data_dir, 'aptos-2015'),
images_dir='test_images_768',
batch_size=batch_size,
tta='fliplr',
workers=num_workers,
crop_black=True)
aptos2015_test.to_pickle(
fs.change_extension(checkpoint_fname,
'_aptos2015_test_private_predictions.pkl'))
if False:
aptos2015_df = pd.read_csv(
os.path.join(data_dir, 'aptos-2015', 'test_labels.csv'))
aptos2015_df = aptos2015_df[aptos2015_df['Usage'] == 'Public']
aptos2015_test = run_model_inference(
model_checkpoint=checkpoint_fname,
apply_softmax=True,
need_features=need_features,
test_csv=aptos2015_df,
data_dir=os.path.join(data_dir, 'aptos-2015'),
images_dir='test_images_768',
batch_size=batch_size,
tta='fliplr',
workers=num_workers,
crop_black=True)
aptos2015_test.to_pickle(
fs.change_extension(checkpoint_fname,
'_aptos2015_test_public_predictions.pkl'))
if False:
aptos2015_df = pd.read_csv(
os.path.join(data_dir, 'aptos-2015', 'train_labels.csv'))
aptos2015_test = run_model_inference(
model_checkpoint=checkpoint_fname,
apply_softmax=True,
need_features=need_features,
test_csv=aptos2015_df,
data_dir=os.path.join(data_dir, 'aptos-2015'),
images_dir='train_images_768',
batch_size=batch_size,
tta='fliplr',
workers=num_workers,
crop_black=True)
aptos2015_test.to_pickle(
fs.change_extension(checkpoint_fname,
'_aptos2015_train_predictions.pkl'))