def visualize_predictions(image_logger, max_samples, metric_fn, logits, gt): num_samples = min(len(gt), max_samples) metrics = to_numpy( metric_fn(from_numpy(logits), from_numpy(gt), average=False)) order = np.argsort(metrics) gt = gt[order][:num_samples] logits = logits[order][:num_samples] metrics = metrics[order][:num_samples] probs = 1 / (1 + np.exp(-logits.squeeze())) samples_per_row = 16 num_rows = int(np.ceil(num_samples / samples_per_row)) * 2 plt.figure(figsize=(6, 1 * num_rows)) for i in range(num_samples): plt.subplot(num_rows, samples_per_row, (i // samples_per_row) * samples_per_row + i + 1) plt.title(f'{metrics[i]:.1f}') plt.imshow(probs[i], vmin=0, vmax=1) plt.xticks([]) plt.yticks([]) plt.subplot(num_rows, samples_per_row, (i // samples_per_row + 1) * samples_per_row + i + 1) plt.imshow(gt[i]) plt.xticks([]) plt.yticks([]) plt.gcf().tight_layout() plt.subplots_adjust(hspace=0.1, wspace=0.1) image_logger(plt.gcf())
def find_outliers(checkpoint_path, num_folds=5, fold_ids=[0, 1, 2, 3], batch_size=1, limit=None, tta=False): model = load_checkpoint(checkpoint_path) model = as_cuda(model) torch.set_grad_enabled(False) model.eval() mask_db = get_mask_db('data/train.csv') all_image_ids, all_fold_ids = get_area_stratified_split(mask_db, num_folds) image_ids = all_image_ids[np.isin(all_fold_ids, fold_ids)] losses = [] generator = get_validation_generator(num_folds, fold_ids, batch_size, limit) for inputs, gt in tqdm(generator, total=len(generator)): inputs, gt = from_numpy(inputs), from_numpy(gt) outputs = model(inputs) if tta: flipped_outputs = model(inputs.flip(dims=(3, ))) outputs = (outputs + flipped_outputs.flip(dims=(3, ))) / 2 batch_losses = to_numpy(lovasz_hinge_loss(outputs, gt, average=False)) losses.extend(batch_losses) image_ids = image_ids[:len(losses)] import pdb pdb.set_trace() stats = pd.DataFrame(data={'image_id': image_ids, 'loss': losses}) print(stats.sort_values('loss'))
def on_validation_end(self, logs, outputs, gt): losses = to_numpy(self.loss_fn(from_numpy(outputs), from_numpy(gt))) losses = losses.mean(axis=0) losses = uniform_filter(losses, size=6, mode='nearest') x, y = np.meshgrid(np.arange(losses.shape[0]), np.arange(losses.shape[1])) fig = plt.figure() ax = fig.gca(projection='3d') ax.plot_surface(x, y, losses, linewidth=0, antialiased=True, cmap=plt.get_cmap('viridis'), edgecolor='none') ax.view_init(60, 35) self.image_logger(fig)
def on_validation_end(self, logs, outputs, gt): values = to_numpy(self.metric_fn(from_numpy(outputs), from_numpy(gt), average=False)) plt.hist(values, bins=20) plt.title(self.metric_fn.__name__) self.image_logger(plt.gcf())
def predict(checkpoint_path, batch_size=1, limit=None, tta=False): blacklist = [ '0035c56490', '005855cd72', '00b6d3a31f', '00c473f654', '00f12566b9', '01a40c3405', '01d6be9b57', '0279cf7419', '02c9416d86', '0300ad09f3', '0350a9e3bd', '06b664b866', '06fa0b053b', '0794c37f5a', '09ce1453ea', '0a7c09181a', '0a90963914', '0bda89116e', '0be00ec340', '0da877df19', '0f13aea58f', '0fcf26daaf', '0fd7ee2ea9', '1081dc0cb9', '108fe2c3d8', '10947d16b1', '10f1d4a32c', '10f59e0caf', '11a0fc8072', '12de44ce0c', '131fc7cbf2', '132cea196e', '1409ddabbf', '1471bc7b6d', '147a36429e', '153558a258', '1557139238', '162f55f72b', '162f738a03', '1817464894', '186e591d03', '1915af8856', '1aaffdb790', '1bb08d1d48', '1bd9b1a1c8', '1c16231286', '1c165d61d6', '1c21b0e9bb', '1c920e3604', '1dac95d5b7', '1df8119d15', '1df8d91491', '1e10c8b214', '1e95bad8bb', '21beba74c7', '21c45388e8', '228e88f048', '22a9fdff9e', '22ca26f94a', '23062a0de4', '242f1fac62', '2454a76962', '245daa2004', '24a18f95ed', '24f57fec17', '2567b0c9de', '2662f5581a', '270dd2b8e6', '27a79442e2', '27b98d2fb7', '2849772dd6', '29d32c7561', '29fdaa3c4e', '2a0583533f', '2aa30fc1d4', '2cf9b202bb', '2e40cf4b88', '2f85d3f736', '2f8b6beb17', '2fac24e793', '30e5c4d227', '31c6a2860f', '31d4b415e4', '328d1497af', '3336643b56', '33cb556649', '3474bba21d', '34f546117e', '353e010b7b', '35b730549d', '36c0313b9e', '37062e1138', '3746667eb4', '3780e66035', '37ab36b8b8', '38bda63402', '38cc48d708', '39aa1f3a2b', '3c3dedc0cc', '3c8cf08665', '3ce8a8bdf4', '3cf20ce659', '3d05dc9c29', '3d21e1dc58', '3d9c671ce1', '3e7fa000c7', '3e871346f4', '3e991d73fa', '3fe3373b67', '4062ebb8f8', '40e3262fb1', '4150e85a49', '42e3dc40ba', '4303cf97d3', '432f07b09e', '447b0d530d', '44df56fa12', '457c69bcc8', '45e3277f3f', '4622cee4b9', '46a93d2bd4', '4775e574dd', '48132be0ac', '485e869caf', '48a6b2914d', '49174ccdd9', '49ba3412bd', '49f33f1832', '4a96727bdf', '4b478173aa', '4b64c96dbe', '4c6c90518d', '4d1825be4e', '4d3eeda971', '4e70e4d96c', '4f18b39baa', '4f29c4cd04', '4fb2778316', '503f08307e', '50ac343646', '5119fdbfde', '51806f45ce', '5241080fcb', '52cfb25e88', '52dbe09f4c', '52f369a734', '536decbbed', '54fbd50faa', '55b9cb8acd', '571e7f5a50', '578cae22ea', '5791c70c05', '5850ff1f52', '5895535b14', '58b6687314', '58d550a35d', '58ee6464fd', '5a06c45958', '5af890206e', '5b7ebdc259', '5c0acc3f31', '5d049a35b3', '5d36a9659e', '5d4578efd7', '5e52f098d9', '5ef842ff3e', '5f3f6d6ca6', '60611407d3', '60ed5847a1', '61a0a17dcd', '629f639f2c', '62ab055b81', '63660a3693', '63a371159d', '648ac9c05a', '65556f12d3', '6654fbf093', '66b8883dd0', '67d21b9e92', '68a1443ac6', '68a469ef7e', '68ff47165b', '69c7de5d26', '6a7d046783', '6ae646336d', '6afae09aa9', '6c6f886709', '6d0ab73c29', '6d23b68142', '6d3209c6d5', '6d75fcd108', '6e28a340f7', '6e470db51c', '6e5e055e6f', '6e67571b91', '6f0d7bdd29', '6fc195d3de', '706321cca9', '70b96ce692', '717bd5d6b1', '71b7cc2fdc', '71e937b285', '7304dcacee', '7370b31cd0', '75d8153a52', '75f0269699', '7671c2d961', '76b26d2e39', '76de89d0eb', '7785879425', '77b35c01c7', '78e2b8371a', '792cef86c2', '7940a73429', '7959b9bd53', '7968eec1f1', '79949a4117', '79bb9dc447', '7a60a99114', '7ae7a91efc', '7bf1f7ff2c', '7ceede7d55', '7e94eb71be', '7f6d43202d', '800b9f8b72', '800d311316', '8016056c46', '808c63ed8f', '8135ca6dde', '81ad4fed2b', '82db07ac64', '837543dddb', '84d6b146a8', '8507aeb1cb', '85ef988548', '85fe1cb502', '868bb336b9', '86bb8ddfe4', '86e7716f79', '8869f3399c', '88e2efec9e', '8920afbcd2', '897d7e821c', '8a39b1ce9f', '8a6401e9d5', '8c1cdc6be2', '8c4081751a', '8cb2bbf1a8', '8e48845dd4', '8fc26c0caa', '90519d19c7', '906c1ff22e', '90c214f1e5', '90efc61382', '91791bd48c', '923aca4789', '9290c42ee0', '92b1a7eec7', '934fcb3879', '93c3bd5f5c', '93ff5d63de', '94c1d3a759', '9511d7e887', '953d0eb2ab', '96798b90f9', '967f41605c', '96d2ddd94f', '988251c854', '98a174a0d4', '98a9095250', '98b6e72be0', '995c49297f', '998e781159', '9aad618687', '9b66bcf23c', '9b86b8a5fe', '9b952b1af0', '9c14cb5581', '9c58e81c46', '9cf74c6432', '9d96a157ca', '9f3cc74e77', '9f5029183b', '9f757eb55b', 'a01b8c4af5', 'a16611557b', 'a16bbd70da', 'a17c4dc16f', 'a21e1cf8d4', 'a4554beb3c', 'a4763177ef', 'a4af4ec79e', 'a89e1ab744', 'a8a86d7d11', 'aa10246dda', 'ac42154416', 'acdbb294e4', 'ad5d638041', 'ad91e70d5c', 'ad972c7127', 'ade51acc52', 'b0fda65353', 'b180ac0113', 'b3459251e5', 'b3be6d0b23', 'b3c8932dc8', 'b5f5cb0885', 'b72181a9d5', 'b77b60349e', 'b935f60197', 'bc09c0039c', 'be02cfe5da', 'be56c6732d', 'be8fb89ea9', 'bf0f9e16e7', 'bf66dafb31', 'c1466f194c', 'c19559637d', 'c1dbd8af8d', 'c1f15b6967', 'c20570220b', 'c215f25359', 'c25ec5b066', 'c28f0753df', 'c2cf683cdc', 'c329b6d198', 'c3bafa1d78', 'c4291c0396', 'c4a43ba621', 'c59e59474e', 'c5dad43641', 'c64b87ba5a', 'c7bfd8548e', 'c8409479b3', 'c877496b7a', 'c89bda5c02', 'c9526a0744', 'c96f52ba7a', 'c9867c4064', 'ca4d59fef3', 'ca9f801c0a', 'cae26f3214', 'cc2fc654f5', 'cc5b03f643', 'ccc645a996', 'cd22ea7236', 'ce96a07785', 'cee5f80b1c', 'd075ac58be', 'd0c4763c9b', 'd2066b2414', 'd27db3dcc0', 'd2e4b1e381', 'd2f25a78a6', 'd3acb2d561', 'd4a84662ee', 'd4f2ed2ce2', 'd795565f86', 'd8e7f62b4e', 'dba4475686', 'ddaa8f2cc9', 'ddd3e5ca3f', 'de6e6ed26a', 'de96056281', 'df0027e3ab', 'dfab2c098a', 'e0257b4c20', 'e0ad4ecf12', 'e18e04a6ff', 'e1a0800dc7', 'e224e91c50', 'e25d11adc9', 'e576adfb4f', 'e634f95937', 'e699288e54', 'e7688312ec', 'e7afd37c7f', 'e84940ac6a', 'e86a7ecaa5', 'e88de6b8fc', 'e8957ff25d', 'e902cfed5a', 'e925b2a8ff', 'ea1488cd35', 'ea3b072e78', 'ea5cc18a2d', 'ea6dfe38ae', 'eac66ccf10', 'eae2a8b2ef', 'eaf44cece7', 'ebeeab3a36', 'ec712f4129', 'ed8fe4f4e0', 'ee54465e3d', 'ee9f7913a1', 'eeb676e37b', 'f2a038cbd1', 'f2fd5c81ac', 'f38d7d303f', 'f3b6e11340', 'f3bf3050d0', 'f43de2bb2a', 'f560c14550', 'f5b747d45b', 'f5d2d4d974', 'f5ddab9cab', 'f5f5901c55', 'f63f03c926', 'f70a411f05', 'f7ed9b7ab7', 'f8088ca24a', 'f8a05eac67', 'f8afabcb4c', 'f8f9e5ba3b', 'f96265d40e', 'f99a792e1c', 'f9c42b8d50', 'fae8a1af69', 'fbac994408', 'fbc8a6a4f9', 'fbf6226410', 'fbf7b17153', 'fd1f3e5caa', 'fd2642606b', 'fd5cc36d93', 'fdad2f99d8', 'fdda40625e', 'ff06e0f167', 'ff608418c1', 'ffb3e66bbe', ] model = load_checkpoint(checkpoint_path) model = as_cuda(model) torch.set_grad_enabled(False) model.eval() records = [] ids = list( map(lambda path: path.split('/')[-1].split('.')[0], get_images_in('data/test/images')))[:limit] test_generator = get_test_generator(batch_size, limit) for inputs, _ in tqdm(test_generator, total=len(test_generator)): inputs = from_numpy(inputs) outputs = model(inputs) if tta: flipped_outputs = model(inputs.flip(dims=(3, ))) outputs = (torch.sigmoid(outputs) + torch.sigmoid(flipped_outputs.flip(dims=(3, )))) / 2 # TODO AS: Ignoring reflected regions, since they are cut on submission masks = to_numpy(outputs[:, 0, :, :].round().long())[:, 13:-14, 13:-14] for mask in masks: _id = ids.pop(0) if mask.sum() <= 3 or _id in blacklist: records.append((_id, None)) else: records.append((_id, encode_rle(mask))) image_ids, encoded_pixels = zip(*records) df = pd.DataFrame({'id': image_ids, 'rle_mask': encoded_pixels}) df.to_csv('./data/submissions/__latest.csv', index=False)
def fit_model(model, train_generator, validation_generator, optimizer, loss_fn, num_epochs, logger, callbacks=[], metrics=[]): for epoch in tqdm(range(num_epochs)): num_batches = len(train_generator) logs = {} logs['train_loss'] = 0 for func in metrics: logs[f'train_{func.__name__}'] = 0 model.train() torch.set_grad_enabled(True) for callback in callbacks: callback.on_train_begin() for inputs, gt in tqdm(train_generator, total=num_batches): inputs, gt = from_numpy(inputs), from_numpy(gt) optimizer.zero_grad() outputs = model(inputs) loss = loss_fn(outputs, gt) loss.backward() optimizer.step() logs['train_loss'] += loss.data[0] for func in metrics: logs[f'train_{func.__name__}'] += func(outputs.detach(), gt) for callback in callbacks: callback.on_train_batch_end(loss.data[0]) logs['train_loss'] /= num_batches for func in metrics: logs[f'train_{func.__name__}'] /= num_batches logs['val_loss'] = 0 for func in metrics: logs[f'val_{func.__name__}'] = 0 all_outputs = [] all_gt = [] num_batches = len(validation_generator) model.eval() torch.set_grad_enabled(False) for inputs, gt in tqdm(validation_generator, total=num_batches): all_gt.append(gt) inputs, gt = from_numpy(inputs), from_numpy(gt) outputs = model(inputs) # TODO AS: Extract as cmd opt flipped_outputs = torch.sigmoid( model(inputs.flip(dims=(3, ))).flip(dims=(3, ))) outputs = torch.sigmoid(outputs) outputs = (outputs + flipped_outputs) / 2 outputs = torch.log(outputs / (1 - outputs)) logs['val_loss'] += loss_fn(outputs, gt).data[0] for func in metrics: logs[f'val_{func.__name__}'] += func(outputs.detach(), gt) if isinstance(outputs, tuple): all_outputs.append(list(map(to_numpy, outputs))) else: all_outputs.append(to_numpy(outputs)) logs['val_loss'] /= num_batches for func in metrics: logs[f'val_{func.__name__}'] /= num_batches if isinstance(all_outputs[0], tuple): all_outputs = list(map(np.concatenate, zip(*all_outputs))) else: all_outputs = np.concatenate(all_outputs) all_gt = np.concatenate(all_gt) for callback in callbacks: callback.on_validation_end(logs, all_outputs, all_gt) epoch_rows = [['epoch', epoch]] for name, value in logs.items(): epoch_rows.append([name, f'{value:.3f}']) logger(tabulate(epoch_rows))