def plot_hist_groups(pred, y, metric, bins=None, figsize=(16, 16)): TP = to_np((pred.mean(dim=0).argmax(dim=1) == y) & (y == 1)) TN = to_np((pred.mean(dim=0).argmax(dim=1) == y) & (y == 0)) FP = to_np((pred.mean(dim=0).argmax(dim=1) != y) & (y == 0)) FN = to_np((pred.mean(dim=0).argmax(dim=1) != y) & (y == 1)) result = metric(pred) TP_result = result[TP] TN_result = result[TN] FP_result = result[FP] FN_result = result[FN] fig, ax = plt.subplots(2, 2, figsize=figsize) sns.distplot(TP_result, ax=ax[0, 0], bins=bins) ax[0, 0].set_title(f"True positive") sns.distplot(TN_result, ax=ax[0, 1], bins=bins) ax[0, 1].set_title(f"True negative") sns.distplot(FP_result, ax=ax[1, 0], bins=bins) ax[1, 0].set_title(f"False positive") sns.distplot(FN_result, ax=ax[1, 1], bins=bins) ax[1, 1].set_title(f"False negative")
def old_analyze_low_confidence(interp, thresh=0.0, thresh_min=0.0, display_mode='delta'): assert display_mode in ['delta', 'index', 'prediction'] p = to_np(interp.preds) y_true = to_np(interp.y_true) wrong = [(j, p[j, i], p[j, y_true[j]], p[j, i] - p[j, y_true[j]]) for j in range(len(y_true)) for i in [np.argmax(p[j])] if i != y_true[j] and p[j, i] < thresh and p[j, i] >= thresh_min] med_delta = np.median([w[3] for w in wrong]) print( f'Total predictions in range: {len(wrong)} Median delta: {med_delta:3.2f}' ) if display_mode == 'index': print('\nLow Confidence Predictions sorted by image number') elif display_mode == 'delta': wrong = sorted(wrong, key=lambda x: x[3]) print( '\nLow Confidence Predictions sorted by difference from correct answer' ) elif display_mode == 'prediction': wrong = sorted(wrong, key=lambda x: x[1]) print('\nLow Confidence Predictions sorted by prediction confidence') #print('\nLow Confidence Predictions sorted by image number') for j, p_predict, p_correct, delta in wrong: print( f'{j:4d}: p_predict: {p_predict:3.2f} p_correct: {p_correct:3.2f} delta: {delta:3.2f}' ) return
def threshold_confusion_matrix(interp, thresh=0): preds = to_np(interp.preds) y_true = to_np(interp.y_true) n = preds.shape[1] cm = np.zeros((n, n), dtype=np.int64) for j, p in enumerate(preds): i = np.argmax(p) y = y_true[j] if p[i] >= thresh: cm[y, i] += 1 return cm
def test_stft(n_fft, n_hop): x_tensor, sr = get_data() x_np = to_np(x_tensor) stft_tensor = stft(x_tensor, n_fft=n_fft, n_hop=n_hop) stft_np = compute_np_batch(x_np, ref_stft, n_fft=n_fft, n_hop=n_hop) check_isclose(stft_tensor, stft_np)
def validate_w_dropout( model: nn.Module, dl: DataLoader, loss_func: OptLossFunc = None, cb_handler: Optional[CallbackHandler] = None, pbar: Optional[PBar] = None, average=True, n_batch: Optional[int] = None ) -> Iterator[Tuple[Union[Tensor, int], ...]]: "Calculate `loss_func` of `model` on `dl` in evaluation mode." model.train() with torch.no_grad(): val_losses, nums = [], [] if cb_handler: cb_handler.set_dl(dl) for xb, yb in progress_bar(dl, parent=pbar, leave=(pbar is not None)): if cb_handler: xb, yb = cb_handler.on_batch_begin(xb, yb, train=False) val_loss = loss_batch(model, xb, yb, loss_func, cb_handler=cb_handler) val_losses.append(val_loss) if not is_listy(yb): yb = [yb] nums.append(yb[0].shape[0]) if cb_handler and cb_handler.on_batch_end(val_losses[-1]): break if n_batch and (len(nums) >= n_batch): break nums = np.array(nums, dtype=np.float32) if average: return (to_np(torch.stack(val_losses)) * nums).sum() / nums.sum() else: return val_losses model.eval()
def silent_validate( model: nn.Module, dl: DataLoader, loss_func: OptLossFunc = None, cb_handler: Optional[CallbackHandler] = None, average=True, n_batch: Optional[int] = None ) -> Iterator[Tuple[Union[Tensor, int], ...]]: """Calculate `loss_func` of `model` on `dl` in evaluation mode. Note: This version does not overwrite results from training""" model.eval() with torch.no_grad(): val_losses, nums = [], [] if cb_handler: cb_handler.set_dl(dl) for xb, yb in dl: if cb_handler: xb, yb = cb_handler.on_batch_begin(xb, yb, train=False) val_loss = loss_batch(model, xb, yb, loss_func, cb_handler=cb_handler) val_losses.append(val_loss) if not is_listy(yb): yb = [yb] nums.append(first_el(yb).shape[0]) if cb_handler and cb_handler.on_batch_end(val_losses[-1]): break if n_batch and (len(nums) >= n_batch): break nums = np.array(nums, dtype=np.float32) if average: return (to_np(torch.stack(val_losses)) * nums).sum() / nums.sum() else: return val_losses
def hook(self, m, i, o): mean = o.mean().item() std = o.std().item() z = to_np(o.mean(dim=0)) i = hex(id(m)) self.stats.append({"m": mean, "s": std, "z": z, "module": i})
def neighbor_gen(at, distance_expansion=None, cutoff=5.0, n_gaussians=25, trainable_gaussians=False, environment_provider=ASEEnvironmentProvider(5.0), collect_triples=False, pair_provider=None, center_positions=True): properties = {} properties[Structure.Z] = tensor(at.numbers.astype(np.int)).unsqueeze(0) positions = at.positions.astype(np.float32) if center_positions: positions -= at.get_center_of_mass() properties[Structure.R] = tensor(positions).unsqueeze(0) properties[Structure.cell] = tensor(at.cell.astype( np.float32)).unsqueeze(0) # get atom environment idx = 0 nbh_idx, offsets = environment_provider.get_environment(idx, at) properties[Structure.neighbors] = tensor(nbh_idx.astype( np.int)).unsqueeze(0) properties[Structure.cell_offset] = tensor(offsets.astype( np.float32)).unsqueeze(0) properties[Structure.neighbor_mask] = None properties['_idx'] = tensor(np.array([idx], dtype=np.int)).unsqueeze(0) if collect_triples: nbh_idx_j, nbh_idx_k = collect_atom_triples(nbh_idx) properties[Structure.neighbor_pairs_j] = tensor( nbh_idx_j.astype(np.int)) properties[Structure.neighbor_pairs_k] = tensor( nbh_idx_k.astype(np.int)) model = spk.custom.representation.RBF( distance_expansion=distance_expansion, cutoff=cutoff, n_gaussians=n_gaussians, trainable_gaussians=trainable_gaussians) model = to_device(model) r, f = model.forward(properties) return to_np(r.squeeze()), to_np(f.squeeze())
def test_to_decibel(ref, top_db, amin): x_tensor, sr = get_data() stft_tensor = stft(x_tensor) stft_np = to_np(stft_tensor) db_tensor = power_to_db(stft_tensor, ref=ref, top_db=top_db, amin=amin) db_np = compute_np_batch(stft_np, ref_power_to_db, ref=ref, top_db=top_db, amin=amin) check_isclose(db_tensor, db_np, atol=amin*10)
def BALD(probs): """Information Gain, distance between the entropy of averages and average of entropy""" entrop1 = entropy(probs) probs = to_np(probs) entrop2 = -(np.log(probs) * probs).sum(axis=2) entrop2 = entrop2.mean(axis=0) ig = entrop1 - entrop2 return ig
def test_freq_to_mel(n_fft, n_mels, f_min, f_max): x_tensor, sr = get_data() stft_tensor = stft(x_tensor, n_fft=n_fft) stft_np = to_np(stft_tensor) mel_tensor = freq_to_mel(stft_tensor, n_mels=n_mels, n_fft=n_fft, sr=sr, f_min=f_min, f_max=f_max) mel_np = compute_np_batch(stft_np, ref_freq_to_mel, n_fft=n_fft, n_mels=n_mels, sr=sr, f_min=f_min, f_max=f_max) check_isclose(mel_tensor, mel_np)
def combine_predictions(all_interp): y_true = to_np(all_interp[0][1].y_true) all_preds = np.stack([to_np(interp.preds) for _, interp in all_interp]) preds = np.mean(all_preds, axis=0) acc_m = _compute_acc(preds, y_true) preds = np.median(all_preds, axis=0) acc_med = _compute_acc(preds, y_true) preds = gmean(all_preds, axis=0) acc_g = _compute_acc(preds, y_true) preds = hmean(all_preds, axis=0) acc_h = _compute_acc(preds, y_true) print( f'accuracy -- mean: {acc_m:0.3f} median: {acc_med:0.3f} gmean: {acc_g:0.3f} hmean: {acc_h:0.3f}' ) return acc_m, acc_med, acc_g, acc_h
def entropy(probs, softmax=False): """Return the prediction of a T*N*C tensor with : - T : the number of samples - N : the batch size - C : the number of classes """ probs = to_np(probs) prob = probs.mean(axis=0) entrop = -(np.log(prob) * prob).sum(axis=1) return entrop
def analyze_low_confidence(interp, thresh=0.0, thresh_min=0.0, display_mode='delta'): assert display_mode in ['delta', 'index', 'prediction'] p = to_np(interp.preds) y_true = to_np(interp.y_true) wrong = [ analyze_row(j, p[j], y_true[j]) for j in range(len(y_true)) for i in [np.argmax(p[j])] if i != y_true[j] and p[j, i] < thresh and p[j, i] >= thresh_min ] med_delta = np.median([w[3] for w in wrong]) med_delta_next = np.median([w[6] for w in wrong]) print( f'Total predictions in range: {len(wrong)} Median del_act: {med_delta:3.2f} Median del_next: {med_delta_next:3.2f}' ) if display_mode == 'index': print('\nLow Confidence Predictions sorted by image number') elif display_mode == 'delta': wrong = sorted(wrong, key=lambda x: x[3]) print( '\nLow Confidence Predictions sorted by difference from correct answer' ) elif display_mode == 'prediction': wrong = sorted(wrong, key=lambda x: x[1]) print('\nLow Confidence Predictions sorted by prediction confidence') for j, p_predict, p_correct, delta_actual, is_second, p_second, delta_second in wrong: is_sec = 'T' if is_second else 'F' print( f'{j:4d}: p_pred: {p_predict:3.2f} p_corr: {p_correct:3.2f} del_act: {delta_actual:3.2f}', end='') print( f' is_sec: {is_sec} p_sec: {p_second:3.2f} del_sec: {delta_second:3.2f}' )
def model_params_to_request_params(training_type, model_params): if model_params is None: return {} if training_type == TrainingType.MNIST: numpy_params = to_np(model_params) return { 'weights': numpy_params[0].tolist(), 'bias': numpy_params[1].tolist() } elif training_type == TrainingType.CHEST_X_RAY_PNEUMONIA: weights_array = [] for i, weights in enumerate(model_params): print('model params SHAPE:', weights.shape) weights_array.append(np.array(weights).tolist()) return {'weights': weights_array} else: raise ValueError('Unsupported training type', training_type)
def inference(cfg, model, data_bunch, test_labels, num_query): logger = logging.getLogger("reid_baseline.inference") logger.info("Start inferencing") pids = [] camids = [] for p, c in test_labels: pids.append(p) camids.append(c) q_pids = np.asarray(pids[:num_query]) g_pids = np.asarray(pids[num_query:]) q_camids = np.asarray(camids[:num_query]) g_camids = np.asarray(camids[num_query:]) feats = [] model.eval() for imgs, _ in data_bunch.test_dl: with torch.no_grad(): feat = model(imgs) feats.append(feat) feats = torch.cat(feats, dim=0) qf = feats[:num_query] gf = feats[num_query:] m, n = qf.shape[0], gf.shape[0] # Cosine distance distmat = torch.mm(F.normalize(qf), F.normalize(gf).t()) # Euclid distance # distmat = torch.pow(qf,2).sum(dim=1,keepdim=True).expand(m,n) + \ # torch.pow(gf,2).sum(dim=1,keepdim=True).expand(n,m).t() # distmat.addmm_(1, -2, qf, gf.t()) distmat = to_np(distmat) # Compute CMC and mAP. cmc, mAP = evaluate(-distmat, q_pids, g_pids, q_camids, g_camids) logger.info('Compute CMC Curve') logger.info("mAP: {:.1%}".format(mAP)) for r in [1, 5, 10]: logger.info("CMC curve, Rank-{:<3}:{:.1%}".format(r, cmc[r - 1]))
fig=plt.figure(figsize=(columns*4, rows*4)) for i in range(rows): fig.add_subplot(rows, columns, 3*i+1) plt.axis('off') plt.imshow(x[i]) fig.add_subplot(rows, columns, 3*i+2) plt.axis('off') plt.imshow(yp[i]) fig.add_subplot(rows, columns, 3*i+3) plt.axis('off') plt.imshow(yt[i]) plt.show() learn.model.eval(); x,y = next(iter(md.val_dl)) yp = to_np(F.sigmoid(learn.model(V(x)))) Show_images(np.asarray(md.val_ds.denorm(x)), yp, y) sz = 384 #image size bs = 32 #batch size md = get_data(sz,bs) learn.set_data(md) learn.unfreeze() learn.bn_freeze(True) learn.fit(lrs/5,1,wds=wd,cycle_len=2,use_clr=(10,8)) learn.save('Unet34_384_1') learn.model.eval(); x,y = next(iter(md.val_dl))
def analyze_confidence(interp, thresh=0.0, do_plot=True, plot_args={'figsize': (10, 5)}, return_raw=False): p = to_np(interp.preds) y_true = to_np(interp.y_true) all_correct = [ p[j, i] for j in range(len(y_true)) for i in [np.argmax(p[j])] if i == y_true[j] and p[j, i] > thresh ] all_wrong = [ p[j, i] for j in range(len(y_true)) for i in [np.argmax(p[j])] if i != y_true[j] and p[j, i] > thresh ] total_predicted = len(all_correct) + len(all_wrong) acc = len(all_correct) / total_predicted missing = len(y_true) - total_predicted pct_unknown = missing / len(y_true) if do_plot and not return_raw: print( f'Accuracy: {100*acc:3.2f}% Error: {100*(1-acc):3.2f}% Unknown: {100*pct_unknown:3.2f}% @ Threshold: {thresh:0.2f}' ) print('') colors = ['green', 'red'] print(f'Confidence Histograms @ t={thresh}') fig, axs = plt.subplots(2, 2, **plot_args) ax = axs[0, 0] ax.tick_params(axis='both', which='major', labelsize=7) ax.set_title(f'Correct & Incorrect') ax.hist([all_correct, all_wrong], range=(0, 1), bins=20, stacked=True, color=colors) ax.set_xlim(thresh, 1) ax = axs[0, 1] ax.tick_params(axis='both', which='major', labelsize=7) ax.set_title(f'Incorrect Only') ax.hist(all_wrong, range=(0, 1), bins=20, color=colors[1]) ax.set_xlim(thresh, 1) ax = axs[1, 0] ax.tick_params(axis='both', which='major', labelsize=7) ax.set_title(f'Log Correct & Incorrect') ax.hist([all_correct, all_wrong], range=(0, 1), bins=20, log=True, stacked=True, color=colors) ax.set_xlim(thresh, 1) ax = axs[1, 1] ax.tick_params(axis='both', which='major', labelsize=7) ax.set_title(f'Log Incorrect Only') ax.hist(all_wrong, range=(0, 1), bins=20, log=True, color=colors[1]) ax.set_xlim(thresh, 1) fig.tight_layout() plt.show() elif return_raw: return total_predicted, len(all_correct), len(all_wrong), missing else: return acc, pct_unknown
def get_bounding_box_predictions(learn, dataloader, anchors, original_images, verticalCropIndex, horizontalCropIndex, detect_threshold=0.5, nms_threshold=0.1, model_input_size=256): """ Generates bounding box predictions for an entire epoch of a provided Dataloader """ all_imgs = [] all_bboxes = [] all_scores = [] batch_index = 0 for img_batch, target_batch in dataloader: with torch.no_grad(): prediction_batch = learn.model(img_batch) class_pred_batch, bbox_pred_batch = prediction_batch[:2] for index, (img, clas_pred, bbox_pred) in enumerate( zip(img_batch, class_pred_batch, bbox_pred_batch)): original_image = original_images[batch_index + index] all_imgs.append(original_image) #Filter out predictions below detect_thresh bbox_pred, scores, preds = process_output( clas_pred, bbox_pred, anchors, detect_threshold) #If there are no bounding boxes, we're done if len(bbox_pred) <= 0: all_bboxes.append([]) all_scores.append([]) continue #Only keep most likely bounding boxes to_keep = nms(bbox_pred, scores, nms_threshold) if len(to_keep) <= 0: all_bboxes.append([]) all_scores.append([]) continue bbox_pred, preds, scores = bbox_pred[to_keep].cpu( ), preds[to_keep].cpu(), scores[to_keep].cpu() #Change back to pixel values height = img.shape[1] width = img.shape[2] t_sz = torch.Tensor([height, width])[None].cpu() bbox_pred = to_np(rescale_boxes(bbox_pred, t_sz)) #Get crop location crop_y, crop_x = get_crop_coordinates(original_image.shape[1], original_image.shape[2], verticalCropIndex, horizontalCropIndex, model_input_size) # change from CTWH to TLRB bbox_pred[:, :2] = bbox_pred[:, :2] - bbox_pred[:, 2:] / 2 #Account for offset due to cropping bbox_pred[:, 0] = bbox_pred[:, 0] + crop_y bbox_pred[:, 2] = bbox_pred[:, 2] + bbox_pred[:, 0] bbox_pred[:, 1] = bbox_pred[:, 1] + crop_x bbox_pred[:, 3] = bbox_pred[:, 3] + bbox_pred[:, 1] all_bboxes.append(bbox_pred) all_scores.append(scores.numpy()) #After completing a batch, we have to keep track the total number of images we've processed batch_index = batch_index + index + 1 return all_imgs, all_bboxes, all_scores
def check_isclose(tensor, nparray, atol=1e-8): c = np.isclose(to_np(tensor), nparray, atol=atol) print(c.size, c.sum()) assert np.isclose(to_np(tensor), nparray, atol=atol).all()