def plot_hist_groups(pred, y, metric, bins=None, figsize=(16, 16)):
TP = to_np((pred.mean(dim=0).argmax(dim=1) == y) & (y == 1))
TN = to_np((pred.mean(dim=0).argmax(dim=1) == y) & (y == 0))
FP = to_np((pred.mean(dim=0).argmax(dim=1) != y) & (y == 0))
FN = to_np((pred.mean(dim=0).argmax(dim=1) != y) & (y == 1))
result = metric(pred)
TP_result = result[TP]
TN_result = result[TN]
FP_result = result[FP]
FN_result = result[FN]
fig, ax = plt.subplots(2, 2, figsize=figsize)
sns.distplot(TP_result, ax=ax[0, 0], bins=bins)
ax[0, 0].set_title(f"True positive")
sns.distplot(TN_result, ax=ax[0, 1], bins=bins)
ax[0, 1].set_title(f"True negative")
sns.distplot(FP_result, ax=ax[1, 0], bins=bins)
ax[1, 0].set_title(f"False positive")
sns.distplot(FN_result, ax=ax[1, 1], bins=bins)
ax[1, 1].set_title(f"False negative")
def old_analyze_low_confidence(interp,
thresh=0.0,
thresh_min=0.0,
display_mode='delta'):
assert display_mode in ['delta', 'index', 'prediction']
p = to_np(interp.preds)
y_true = to_np(interp.y_true)
wrong = [(j, p[j, i], p[j, y_true[j]], p[j, i] - p[j, y_true[j]])
for j in range(len(y_true)) for i in [np.argmax(p[j])]
if i != y_true[j] and p[j, i] < thresh and p[j, i] >= thresh_min]
med_delta = np.median([w[3] for w in wrong])
print(
f'Total predictions in range: {len(wrong)} Median delta: {med_delta:3.2f}'
)
if display_mode == 'index':
print('\nLow Confidence Predictions sorted by image number')
elif display_mode == 'delta':
wrong = sorted(wrong, key=lambda x: x[3])
print(
'\nLow Confidence Predictions sorted by difference from correct answer'
)
elif display_mode == 'prediction':
wrong = sorted(wrong, key=lambda x: x[1])
print('\nLow Confidence Predictions sorted by prediction confidence')
#print('\nLow Confidence Predictions sorted by image number')
for j, p_predict, p_correct, delta in wrong:
print(
f'{j:4d}: p_predict: {p_predict:3.2f} p_correct: {p_correct:3.2f} delta: {delta:3.2f}'
)
return
def threshold_confusion_matrix(interp, thresh=0):
preds = to_np(interp.preds)
y_true = to_np(interp.y_true)
n = preds.shape[1]
cm = np.zeros((n, n), dtype=np.int64)
for j, p in enumerate(preds):
i = np.argmax(p)
y = y_true[j]
if p[i] >= thresh:
cm[y, i] += 1
return cm
def test_stft(n_fft, n_hop):
x_tensor, sr = get_data()
x_np = to_np(x_tensor)
stft_tensor = stft(x_tensor, n_fft=n_fft, n_hop=n_hop)
stft_np = compute_np_batch(x_np, ref_stft, n_fft=n_fft, n_hop=n_hop)
check_isclose(stft_tensor, stft_np)
def validate_w_dropout(
model: nn.Module,
dl: DataLoader,
loss_func: OptLossFunc = None,
cb_handler: Optional[CallbackHandler] = None,
pbar: Optional[PBar] = None,
average=True,
n_batch: Optional[int] = None
) -> Iterator[Tuple[Union[Tensor, int], ...]]:
"Calculate `loss_func` of `model` on `dl` in evaluation mode."
model.train()
with torch.no_grad():
val_losses, nums = [], []
if cb_handler: cb_handler.set_dl(dl)
for xb, yb in progress_bar(dl, parent=pbar, leave=(pbar is not None)):
if cb_handler:
xb, yb = cb_handler.on_batch_begin(xb, yb, train=False)
val_loss = loss_batch(model,
xb,
yb,
loss_func,
cb_handler=cb_handler)
val_losses.append(val_loss)
if not is_listy(yb): yb = [yb]
nums.append(yb[0].shape[0])
if cb_handler and cb_handler.on_batch_end(val_losses[-1]): break
if n_batch and (len(nums) >= n_batch): break
nums = np.array(nums, dtype=np.float32)
if average:
return (to_np(torch.stack(val_losses)) * nums).sum() / nums.sum()
else:
return val_losses
model.eval()
def silent_validate(
model: nn.Module,
dl: DataLoader,
loss_func: OptLossFunc = None,
cb_handler: Optional[CallbackHandler] = None,
average=True,
n_batch: Optional[int] = None
) -> Iterator[Tuple[Union[Tensor, int], ...]]:
"""Calculate `loss_func` of `model` on `dl` in evaluation mode.
Note: This version does not overwrite results from training"""
model.eval()
with torch.no_grad():
val_losses, nums = [], []
if cb_handler: cb_handler.set_dl(dl)
for xb, yb in dl:
if cb_handler:
xb, yb = cb_handler.on_batch_begin(xb, yb, train=False)
val_loss = loss_batch(model,
xb,
yb,
loss_func,
cb_handler=cb_handler)
val_losses.append(val_loss)
if not is_listy(yb): yb = [yb]
nums.append(first_el(yb).shape[0])
if cb_handler and cb_handler.on_batch_end(val_losses[-1]): break
if n_batch and (len(nums) >= n_batch): break
nums = np.array(nums, dtype=np.float32)
if average:
return (to_np(torch.stack(val_losses)) * nums).sum() / nums.sum()
else:
return val_losses
def hook(self, m, i, o):
mean = o.mean().item()
std = o.std().item()
z = to_np(o.mean(dim=0))
i = hex(id(m))
self.stats.append({"m": mean, "s": std, "z": z, "module": i})
def neighbor_gen(at,
distance_expansion=None,
cutoff=5.0,
n_gaussians=25,
trainable_gaussians=False,
environment_provider=ASEEnvironmentProvider(5.0),
collect_triples=False,
pair_provider=None,
center_positions=True):
properties = {}
properties[Structure.Z] = tensor(at.numbers.astype(np.int)).unsqueeze(0)
positions = at.positions.astype(np.float32)
if center_positions:
positions -= at.get_center_of_mass()
properties[Structure.R] = tensor(positions).unsqueeze(0)
properties[Structure.cell] = tensor(at.cell.astype(
np.float32)).unsqueeze(0)
# get atom environment
idx = 0
nbh_idx, offsets = environment_provider.get_environment(idx, at)
properties[Structure.neighbors] = tensor(nbh_idx.astype(
np.int)).unsqueeze(0)
properties[Structure.cell_offset] = tensor(offsets.astype(
np.float32)).unsqueeze(0)
properties[Structure.neighbor_mask] = None
properties['_idx'] = tensor(np.array([idx], dtype=np.int)).unsqueeze(0)
if collect_triples:
nbh_idx_j, nbh_idx_k = collect_atom_triples(nbh_idx)
properties[Structure.neighbor_pairs_j] = tensor(
nbh_idx_j.astype(np.int))
properties[Structure.neighbor_pairs_k] = tensor(
nbh_idx_k.astype(np.int))
model = spk.custom.representation.RBF(
distance_expansion=distance_expansion,
cutoff=cutoff,
n_gaussians=n_gaussians,
trainable_gaussians=trainable_gaussians)
model = to_device(model)
r, f = model.forward(properties)
return to_np(r.squeeze()), to_np(f.squeeze())
def test_to_decibel(ref, top_db, amin):
x_tensor, sr = get_data()
stft_tensor = stft(x_tensor)
stft_np = to_np(stft_tensor)
db_tensor = power_to_db(stft_tensor,
ref=ref, top_db=top_db, amin=amin)
db_np = compute_np_batch(stft_np, ref_power_to_db,
ref=ref, top_db=top_db, amin=amin)
check_isclose(db_tensor, db_np, atol=amin*10)
def BALD(probs):
"""Information Gain, distance between the entropy of averages and average of entropy"""
entrop1 = entropy(probs)
probs = to_np(probs)
entrop2 = -(np.log(probs) * probs).sum(axis=2)
entrop2 = entrop2.mean(axis=0)
ig = entrop1 - entrop2
return ig
def test_freq_to_mel(n_fft, n_mels, f_min, f_max):
x_tensor, sr = get_data()
stft_tensor = stft(x_tensor, n_fft=n_fft)
stft_np = to_np(stft_tensor)
mel_tensor = freq_to_mel(stft_tensor, n_mels=n_mels, n_fft=n_fft, sr=sr,
f_min=f_min, f_max=f_max)
mel_np = compute_np_batch(stft_np, ref_freq_to_mel, n_fft=n_fft,
n_mels=n_mels, sr=sr,
f_min=f_min, f_max=f_max)
check_isclose(mel_tensor, mel_np)
def combine_predictions(all_interp):
y_true = to_np(all_interp[0][1].y_true)
all_preds = np.stack([to_np(interp.preds) for _, interp in all_interp])
preds = np.mean(all_preds, axis=0)
acc_m = _compute_acc(preds, y_true)
preds = np.median(all_preds, axis=0)
acc_med = _compute_acc(preds, y_true)
preds = gmean(all_preds, axis=0)
acc_g = _compute_acc(preds, y_true)
preds = hmean(all_preds, axis=0)
acc_h = _compute_acc(preds, y_true)
print(
f'accuracy -- mean: {acc_m:0.3f} median: {acc_med:0.3f} gmean: {acc_g:0.3f} hmean: {acc_h:0.3f}'
)
return acc_m, acc_med, acc_g, acc_h
def entropy(probs, softmax=False):
"""Return the prediction of a T*N*C tensor with :
- T : the number of samples
- N : the batch size
- C : the number of classes
"""
probs = to_np(probs)
prob = probs.mean(axis=0)
entrop = -(np.log(prob) * prob).sum(axis=1)
return entrop
def analyze_low_confidence(interp,
thresh=0.0,
thresh_min=0.0,
display_mode='delta'):
assert display_mode in ['delta', 'index', 'prediction']
p = to_np(interp.preds)
y_true = to_np(interp.y_true)
wrong = [
analyze_row(j, p[j], y_true[j]) for j in range(len(y_true))
for i in [np.argmax(p[j])]
if i != y_true[j] and p[j, i] < thresh and p[j, i] >= thresh_min
]
med_delta = np.median([w[3] for w in wrong])
med_delta_next = np.median([w[6] for w in wrong])
print(
f'Total predictions in range: {len(wrong)} Median del_act: {med_delta:3.2f} Median del_next: {med_delta_next:3.2f}'
)
if display_mode == 'index':
print('\nLow Confidence Predictions sorted by image number')
elif display_mode == 'delta':
wrong = sorted(wrong, key=lambda x: x[3])
print(
'\nLow Confidence Predictions sorted by difference from correct answer'
)
elif display_mode == 'prediction':
wrong = sorted(wrong, key=lambda x: x[1])
print('\nLow Confidence Predictions sorted by prediction confidence')
for j, p_predict, p_correct, delta_actual, is_second, p_second, delta_second in wrong:
is_sec = 'T' if is_second else 'F'
print(
f'{j:4d}: p_pred: {p_predict:3.2f} p_corr: {p_correct:3.2f} del_act: {delta_actual:3.2f}',
end='')
print(
f' is_sec: {is_sec} p_sec: {p_second:3.2f} del_sec: {delta_second:3.2f}'
)
def model_params_to_request_params(training_type, model_params):
if model_params is None:
return {}
if training_type == TrainingType.MNIST:
numpy_params = to_np(model_params)
return {
'weights': numpy_params[0].tolist(),
'bias': numpy_params[1].tolist()
}
elif training_type == TrainingType.CHEST_X_RAY_PNEUMONIA:
weights_array = []
for i, weights in enumerate(model_params):
print('model params SHAPE:', weights.shape)
weights_array.append(np.array(weights).tolist())
return {'weights': weights_array}
else:
raise ValueError('Unsupported training type', training_type)
def inference(cfg, model, data_bunch, test_labels, num_query):
logger = logging.getLogger("reid_baseline.inference")
logger.info("Start inferencing")
pids = []
camids = []
for p, c in test_labels:
pids.append(p)
camids.append(c)
q_pids = np.asarray(pids[:num_query])
g_pids = np.asarray(pids[num_query:])
q_camids = np.asarray(camids[:num_query])
g_camids = np.asarray(camids[num_query:])
feats = []
model.eval()
for imgs, _ in data_bunch.test_dl:
with torch.no_grad():
feat = model(imgs)
feats.append(feat)
feats = torch.cat(feats, dim=0)
qf = feats[:num_query]
gf = feats[num_query:]
m, n = qf.shape[0], gf.shape[0]
# Cosine distance
distmat = torch.mm(F.normalize(qf), F.normalize(gf).t())
# Euclid distance
# distmat = torch.pow(qf,2).sum(dim=1,keepdim=True).expand(m,n) + \
# torch.pow(gf,2).sum(dim=1,keepdim=True).expand(n,m).t()
# distmat.addmm_(1, -2, qf, gf.t())
distmat = to_np(distmat)
# Compute CMC and mAP.
cmc, mAP = evaluate(-distmat, q_pids, g_pids, q_camids, g_camids)
logger.info('Compute CMC Curve')
logger.info("mAP: {:.1%}".format(mAP))
for r in [1, 5, 10]:
logger.info("CMC curve, Rank-{:<3}:{:.1%}".format(r, cmc[r - 1]))
fig=plt.figure(figsize=(columns*4, rows*4))
for i in range(rows):
fig.add_subplot(rows, columns, 3*i+1)
plt.axis('off')
plt.imshow(x[i])
fig.add_subplot(rows, columns, 3*i+2)
plt.axis('off')
plt.imshow(yp[i])
fig.add_subplot(rows, columns, 3*i+3)
plt.axis('off')
plt.imshow(yt[i])
plt.show()
learn.model.eval();
x,y = next(iter(md.val_dl))
yp = to_np(F.sigmoid(learn.model(V(x))))
Show_images(np.asarray(md.val_ds.denorm(x)), yp, y)
sz = 384 #image size
bs = 32 #batch size
md = get_data(sz,bs)
learn.set_data(md)
learn.unfreeze()
learn.bn_freeze(True)
learn.fit(lrs/5,1,wds=wd,cycle_len=2,use_clr=(10,8))
learn.save('Unet34_384_1')
learn.model.eval();
x,y = next(iter(md.val_dl))
def analyze_confidence(interp,
thresh=0.0,
do_plot=True,
plot_args={'figsize': (10, 5)},
return_raw=False):
p = to_np(interp.preds)
y_true = to_np(interp.y_true)
all_correct = [
p[j, i] for j in range(len(y_true)) for i in [np.argmax(p[j])]
if i == y_true[j] and p[j, i] > thresh
]
all_wrong = [
p[j, i] for j in range(len(y_true)) for i in [np.argmax(p[j])]
if i != y_true[j] and p[j, i] > thresh
]
total_predicted = len(all_correct) + len(all_wrong)
acc = len(all_correct) / total_predicted
missing = len(y_true) - total_predicted
pct_unknown = missing / len(y_true)
if do_plot and not return_raw:
print(
f'Accuracy: {100*acc:3.2f}% Error: {100*(1-acc):3.2f}% Unknown: {100*pct_unknown:3.2f}% @ Threshold: {thresh:0.2f}'
)
print('')
colors = ['green', 'red']
print(f'Confidence Histograms @ t={thresh}')
fig, axs = plt.subplots(2, 2, **plot_args)
ax = axs[0, 0]
ax.tick_params(axis='both', which='major', labelsize=7)
ax.set_title(f'Correct & Incorrect')
ax.hist([all_correct, all_wrong],
range=(0, 1),
bins=20,
stacked=True,
color=colors)
ax.set_xlim(thresh, 1)
ax = axs[0, 1]
ax.tick_params(axis='both', which='major', labelsize=7)
ax.set_title(f'Incorrect Only')
ax.hist(all_wrong, range=(0, 1), bins=20, color=colors[1])
ax.set_xlim(thresh, 1)
ax = axs[1, 0]
ax.tick_params(axis='both', which='major', labelsize=7)
ax.set_title(f'Log Correct & Incorrect')
ax.hist([all_correct, all_wrong],
range=(0, 1),
bins=20,
log=True,
stacked=True,
color=colors)
ax.set_xlim(thresh, 1)
ax = axs[1, 1]
ax.tick_params(axis='both', which='major', labelsize=7)
ax.set_title(f'Log Incorrect Only')
ax.hist(all_wrong, range=(0, 1), bins=20, log=True, color=colors[1])
ax.set_xlim(thresh, 1)
fig.tight_layout()
plt.show()
elif return_raw:
return total_predicted, len(all_correct), len(all_wrong), missing
else:
return acc, pct_unknown
def get_bounding_box_predictions(learn,
dataloader,
anchors,
original_images,
verticalCropIndex,
horizontalCropIndex,
detect_threshold=0.5,
nms_threshold=0.1,
model_input_size=256):
"""
Generates bounding box predictions for an entire epoch of a provided Dataloader
"""
all_imgs = []
all_bboxes = []
all_scores = []
batch_index = 0
for img_batch, target_batch in dataloader:
with torch.no_grad():
prediction_batch = learn.model(img_batch)
class_pred_batch, bbox_pred_batch = prediction_batch[:2]
for index, (img, clas_pred, bbox_pred) in enumerate(
zip(img_batch, class_pred_batch, bbox_pred_batch)):
original_image = original_images[batch_index + index]
all_imgs.append(original_image)
#Filter out predictions below detect_thresh
bbox_pred, scores, preds = process_output(
clas_pred, bbox_pred, anchors, detect_threshold)
#If there are no bounding boxes, we're done
if len(bbox_pred) <= 0:
all_bboxes.append([])
all_scores.append([])
continue
#Only keep most likely bounding boxes
to_keep = nms(bbox_pred, scores, nms_threshold)
if len(to_keep) <= 0:
all_bboxes.append([])
all_scores.append([])
continue
bbox_pred, preds, scores = bbox_pred[to_keep].cpu(
), preds[to_keep].cpu(), scores[to_keep].cpu()
#Change back to pixel values
height = img.shape[1]
width = img.shape[2]
t_sz = torch.Tensor([height, width])[None].cpu()
bbox_pred = to_np(rescale_boxes(bbox_pred, t_sz))
#Get crop location
crop_y, crop_x = get_crop_coordinates(original_image.shape[1],
original_image.shape[2],
verticalCropIndex,
horizontalCropIndex,
model_input_size)
# change from CTWH to TLRB
bbox_pred[:, :2] = bbox_pred[:, :2] - bbox_pred[:, 2:] / 2
#Account for offset due to cropping
bbox_pred[:, 0] = bbox_pred[:, 0] + crop_y
bbox_pred[:, 2] = bbox_pred[:, 2] + bbox_pred[:, 0]
bbox_pred[:, 1] = bbox_pred[:, 1] + crop_x
bbox_pred[:, 3] = bbox_pred[:, 3] + bbox_pred[:, 1]
all_bboxes.append(bbox_pred)
all_scores.append(scores.numpy())
#After completing a batch, we have to keep track the total number of images we've processed
batch_index = batch_index + index + 1
return all_imgs, all_bboxes, all_scores
def check_isclose(tensor, nparray, atol=1e-8):
c = np.isclose(to_np(tensor), nparray, atol=atol)
print(c.size, c.sum())
assert np.isclose(to_np(tensor), nparray, atol=atol).all()
