def predict_results(config_types: [type(BaseClassificationTrainConfig)], output_file: str, threshold: float):
dataset = create_dataset(is_test=True, include_negatives=True)
output_dir = os.path.dirname(output_file)
if not os.path.exists(output_dir) and not os.path.isdir(output_dir):
os.makedirs(output_dir)
predictors = []
for config_type in config_types:
fsm = FileStructManager(base_dir=config_type.experiment_dir, is_continue=True)
predictors.append(Predictor(config_type.create_model().cuda(), fsm=fsm))
with open(output_file, 'w') as out_file:
out_file.write("ImageId,EncodedPixels\n")
out_file.flush()
images_paths = dataset.get_items()
for i, data in enumerate(tqdm(dataset)):
data = cv2.resize(data, (512, 512))
img_tensor = torch.from_numpy(np.expand_dims(np.expand_dims(data.astype(np.float32), 0) / 128 - 1, 0)).cuda()
res = []
for predictor in predictors:
res.append(np.squeeze(predictor.predict({'data': img_tensor}).data.cpu().numpy()))
res = np.median(res)
res = np.where(res > threshold, 1, 0).astype(np.int)
out_file.write("{},{}\n".format(os.path.splitext(os.path.basename(images_paths[i]))[0], float(res)))
out_file.flush()
def predict(config_type: type(BaseSegmentationTrainConfig), output_file: str):
# dataset = create_dataset(is_test=False, indices_path='data/indices/train.npy')
dataset = create_dataset(is_test=True,
for_segmentation=True,
include_negatives=False,
indices_path='data/indices/test_seg.npy')
output_dir = os.path.dirname(output_file)
if not os.path.exists(output_dir) and not os.path.isdir(output_dir):
os.makedirs(output_dir)
fsm = FileStructManager(base_dir=config_type.experiment_dir,
is_continue=True)
predictor = Predictor(config_type.create_model().cuda(), fsm=fsm)
with open(output_file, 'w') as out_file:
out_file.write("ImageId,EncodedPixels\n")
out_file.flush()
images_paths = dataset.get_items()
for i, data in enumerate(tqdm(dataset)):
# img = data['data'].copy()
# target = data['target'].copy()
data = cv2.resize(data, (512, 512))
img_tensor = torch.from_numpy(
np.expand_dims(
np.expand_dims(data.astype(np.float32), 0) / 128 - 1,
0)).cuda()
res = np.squeeze(
predictor.predict({
'data': img_tensor
}).data.cpu().numpy())
res[res < 0.7] = 0
if res[res > 0].size < 101:
rle = -1
else:
res = (res * 255).astype(np.uint8)
res = cv2.resize(res, (1024, 1024))
# res_cntrs, _ = cv2.findContours(res, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
# target_cntrs, _ = cv2.findContours((target * 255).astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
#
# img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
# img = cv2.drawContours(img, res_cntrs, -1, (0, 0, 255))
# img = cv2.drawContours(img, target_cntrs, -1, (0, 255, 0))
# cv2.imshow("img", img)
# cv2.waitKey(0)
# res = cv2.flip(res, 1)
# res = cv2.rotate(res, cv2.ROTATE_90_COUNTERCLOCKWISE)
res[res > 0] = 255
rle = mask2rle(res)
out_file.write("{},{}\n".format(
os.path.splitext(os.path.basename(images_paths[i]))[0], rle))
out_file.flush()
def predict_on_test_set(config_type: type(BaseClassificationTrainConfig)):
dataset = create_dataset(is_test=False, include_negatives=True, indices_path='data/indices/test_class.npy')
fsm = FileStructManager(base_dir=config_type.experiment_dir, is_continue=True)
predictor = Predictor(config_type.create_model().cuda(), fsm=fsm)
predicts, targets = [], []
for i, data in enumerate(tqdm(dataset)):
targets.append(ClassificationAugmentations.mask2class(data['target']))
data = cv2.resize(data['data'], (512, 512))
img_tensor = torch.from_numpy(np.expand_dims(np.expand_dims(data.astype(np.float32), 0) / 128 - 1, 0)).cuda()
res = np.squeeze(predictor.predict({'data': img_tensor}).data.cpu().numpy())
predicts.append(res)
return np.array(predicts), np.array(targets)
def predict(config_types: [type(BaseSegmentationTrainConfig)], output_file: str, class_predicts: {}):
dataset = create_dataset(is_test=True, include_negatives=False)
output_dir = os.path.dirname(output_file)
if not os.path.exists(output_dir) and not os.path.isdir(output_dir):
os.makedirs(output_dir)
predictors = []
for config_type in config_types:
fsm = FileStructManager(base_dir=config_type.experiment_dir, is_continue=True)
predictors.append(Predictor(config_type.create_model().cuda(), fsm=fsm))
with open(output_file, 'w') as out_file:
out_file.write("ImageId,EncodedPixels\n")
out_file.flush()
images_paths = dataset.get_items()
for i, data in enumerate(tqdm(dataset)):
cur_img_path = os.path.splitext(os.path.basename(images_paths[i]))[0]
data = cv2.resize(data, (512, 512))
img_tensor = torch.from_numpy(np.expand_dims(np.expand_dims(data.astype(np.float32), 0) / 128 - 1, 0)).cuda()
res = []
for predictor in predictors:
res.append(np.squeeze(predictor.predict({'data': img_tensor}).data.cpu().numpy()))
res = np.median(res, axis=0)
res[res < 0.7] = 0
if not class_predicts[cur_img_path]:
rle = '-1'
else:
res = (res * 255).astype(np.uint8)
res = cv2.resize(res, (1024, 1024))
res[res > 0] = 255
rle = mask2rle(res)
if len(rle) < 1:
rle = '-1'
out_file.write("{},{}\n".format(cur_img_path, rle))
out_file.flush()
import os
from pietoolbelt.steps.stratification import DatasetStratification
from train_config.config import folds_num
from train_config.dataset import create_dataset
from config import INDICES_DIR
if __name__ == '__main__':
if not os.path.exists(INDICES_DIR):
os.makedirs(INDICES_DIR)
test_part = 0.1
folds_dict = {'fold_{}.npy'.format(i): (1 - test_part) / folds_num for i in range(folds_num)}
strat = DatasetStratification(create_dataset(), lambda x: x['target'])
strat.run(dict(folds_dict, **{'test.npy': test_part}), INDICES_DIR)
predicts_from_file = np.loadtxt(os.path.join(predicts_path, f),
delimiter=',')[1:]
predicts_from_file = {v[0]: v[1] for v in predicts_from_file}
all_predicts.append({'preds': predicts_from_file, 'net': f})
for num in range(2, len(all_predicts) + 1):
for cmb in list(combinations(all_predicts, num)):
all_predicts.append({
'preds':
unite_predicts([all_predicts[i]['preds'] for i in cmb]),
'net':
','.join([all_predicts[i]['net'] for i in cmb])
})
dataset = create_dataset(is_test=False,
for_segmentation=True,
indices_path='data/out/test_class.npy')
best_thresh, best_pred, best_metric = None, None, 0
for pred in all_predicts:
cur_metric, thresh = get_best_thresh(pred, dataset)
if cur_metric > best_metric:
best_metric = cur_metric
best_thresh = thresh
best_pred = pred
with open(os.path.join(predicts_path, 'class_best_predict_config.json'),
'w') as out:
json.dump(out, {'net': best_pred['net'], 'thresh': best_thresh})
import numpy as np
import os
from pietoolbelt.steps.stratification import DatasetStratification
from train_config.dataset import create_dataset
from train_config.train_config import MyTrainConfig
def calc_label(x):
return int(10 * np.count_nonzero(x) / x.size)
if __name__ == '__main__':
out_dir = os.path.join('data', 'indices')
if not os.path.exists(out_dir):
os.makedirs(out_dir)
test_part = 0.1
folds_dict = {
'fold_{}.npy'.format(i): (1 - test_part) / MyTrainConfig.folds_num
for i in range(MyTrainConfig.folds_num)
}
strat = DatasetStratification(create_dataset(), calc_label, workers_num=12)
strat.run(dict(folds_dict, **{'test.npy': test_part}), out_dir)