def _run_inference(self, dataset, summary, threshod=0.5):
"""
Run inference for the dataset and write the inference related data into summary.
Args:
dataset (`ds`): the parsed dataset
summary (`SummaryRecord`): the summary object to store the data
threshold (float): the threshold for prediction.
Returns:
imageid_labels (dict): a dict that maps image_id and the union of its ground truth and predicted labels.
"""
imageid_labels = {}
ds.config.set_seed(58)
self._count = 0
for j, next_element in enumerate(dataset):
now = time()
inputs, labels, _ = self._unpack_next_element(next_element)
prob = self._model(inputs).asnumpy()
for idx, inp in enumerate(inputs):
gt_labels = labels[idx]
gt_probs = [float(prob[idx][i]) for i in gt_labels]
data_np = _convert_image_format(
np.expand_dims(inp.asnumpy(), 0), 'NCHW')
_, _, _, image_string = _make_image(_normalize(data_np))
predicted_labels = [
int(i) for i in (prob[idx] > threshod).nonzero()[0]
]
predicted_probs = [
float(prob[idx][i]) for i in predicted_labels
]
union_labs = list(set(gt_labels + predicted_labels))
imageid_labels[str(self._count)] = union_labs
explain = Explain()
explain.image_id = str(self._count)
explain.image_data = image_string
summary.add_value("explainer", "image", explain)
explain = Explain()
explain.image_id = str(self._count)
explain.ground_truth_label.extend(gt_labels)
explain.inference.ground_truth_prob.extend(gt_probs)
explain.inference.predicted_label.extend(predicted_labels)
explain.inference.predicted_prob.extend(predicted_probs)
summary.add_value("explainer", "inference", explain)
summary.record(1)
self._count += 1
print(
"Finish running and writing {}-th batch inference data. Time elapsed: {}s"
.format(j,
time() - now))
return imageid_labels
def _run_inference(self, summary, threshold=0.5):
"""
Run inference for the dataset and write the inference related data into summary.
Args:
summary (SummaryRecord): The summary object to store the data
threshold (float): The threshold for prediction.
Returns:
dict, The map of sample d to the union of its ground truth and predicted labels.
"""
sample_id_labels = {}
self._sample_index = 0
ds.config.set_seed(self._DATASET_SEED)
for j, next_element in enumerate(self._dataset):
now = time()
inputs, labels, _ = self._unpack_next_element(next_element)
prob = self._full_network(inputs).asnumpy()
for idx, inp in enumerate(inputs):
gt_labels = labels[idx]
gt_probs = [float(prob[idx][i]) for i in gt_labels]
data_np = _convert_image_format(np.expand_dims(inp.asnumpy(), 0), 'NCHW')
original_image = _np_to_image(_normalize(data_np), mode='RGB')
original_image_path = self._save_original_image(self._sample_index, original_image)
predicted_labels = [int(i) for i in (prob[idx] > threshold).nonzero()[0]]
predicted_probs = [float(prob[idx][i]) for i in predicted_labels]
union_labs = list(set(gt_labels + predicted_labels))
sample_id_labels[str(self._sample_index)] = union_labs
explain = Explain()
explain.sample_id = self._sample_index
explain.image_path = original_image_path
summary.add_value("explainer", "sample", explain)
explain = Explain()
explain.sample_id = self._sample_index
explain.ground_truth_label.extend(gt_labels)
explain.inference.ground_truth_prob.extend(gt_probs)
explain.inference.predicted_label.extend(predicted_labels)
explain.inference.predicted_prob.extend(predicted_probs)
summary.add_value("explainer", "inference", explain)
summary.record(1)
self._sample_index += 1
self._spaced_print("Finish running and writing {}-th batch inference data."
" Time elapsed: {:.3f} s".format(j, time() - now),
end='')
return sample_id_labels
def _run_inference(self, summary, threshold=0.5):
"""
Run inference for the dataset and write the inference related data into summary.
Args:
summary (SummaryRecord): The summary object to store the data.
threshold (float): The threshold for prediction.
Returns:
dict, The map of sample d to the union of its ground truth and predicted labels.
"""
has_uncertainty_rec = False
sample_id_labels = {}
self._sample_index = 0
ds.config.set_seed(self._DATASET_SEED)
for j, next_element in enumerate(self._dataset):
now = time()
inputs, labels, _ = self._unpack_next_element(next_element)
prob = self._full_network(inputs).asnumpy()
if self._uncertainty is not None:
prob_var = self._uncertainty.eval_epistemic_uncertainty(inputs)
else:
prob_var = None
for idx, inp in enumerate(inputs):
gt_labels = labels[idx]
gt_probs = [float(prob[idx][i]) for i in gt_labels]
if prob_var is not None:
gt_prob_vars = [float(prob_var[idx][i]) for i in gt_labels]
gt_itl_lows, gt_itl_his, gt_prob_sds = \
self._calc_beta_intervals(gt_probs, gt_prob_vars)
data_np = _convert_image_format(np.expand_dims(inp.asnumpy(), 0), 'NCHW')
original_image = _np_to_image(_normalize(data_np), mode='RGB')
original_image_path = self._save_original_image(self._sample_index, original_image)
predicted_labels = [int(i) for i in (prob[idx] > threshold).nonzero()[0]]
predicted_probs = [float(prob[idx][i]) for i in predicted_labels]
if prob_var is not None:
predicted_prob_vars = [float(prob_var[idx][i]) for i in predicted_labels]
predicted_itl_lows, predicted_itl_his, predicted_prob_sds = \
self._calc_beta_intervals(predicted_probs, predicted_prob_vars)
union_labs = list(set(gt_labels + predicted_labels))
sample_id_labels[str(self._sample_index)] = union_labs
explain = Explain()
explain.sample_id = self._sample_index
explain.image_path = original_image_path
summary.add_value("explainer", "sample", explain)
explain = Explain()
explain.sample_id = self._sample_index
explain.ground_truth_label.extend(gt_labels)
explain.inference.ground_truth_prob.extend(gt_probs)
explain.inference.predicted_label.extend(predicted_labels)
explain.inference.predicted_prob.extend(predicted_probs)
if prob_var is not None:
explain.inference.ground_truth_prob_sd.extend(gt_prob_sds)
explain.inference.ground_truth_prob_itl95_low.extend(gt_itl_lows)
explain.inference.ground_truth_prob_itl95_hi.extend(gt_itl_his)
explain.inference.predicted_prob_sd.extend(predicted_prob_sds)
explain.inference.predicted_prob_itl95_low.extend(predicted_itl_lows)
explain.inference.predicted_prob_itl95_hi.extend(predicted_itl_his)
has_uncertainty_rec = True
summary.add_value("explainer", "inference", explain)
summary.record(1)
if self._is_hoc_registered:
self._run_hoc(summary, self._sample_index, inputs[idx], prob[idx])
self._sample_index += 1
self._spaced_print("Finish running and writing {}-th batch inference data."
" Time elapsed: {:.3f} s".format(j, time() - now))
if has_uncertainty_rec:
self._manifest["uncertainty"] = True
return sample_id_labels
def _run_inference(self, dataset, summary, threshold=0.5):
"""
Run inference for the dataset and write the inference related data into summary.
Args:
dataset (`ds`): the parsed dataset
summary (`SummaryRecord`): the summary object to store the data
threshold (float): the threshold for prediction.
Returns:
imageid_labels (dict): a dict that maps image_id and the union of its ground truth and predicted labels.
"""
spacer = '{:120}\r'
imageid_labels = {}
ds.config.set_seed(_SEED)
self._count = 0
for j, next_element in enumerate(dataset):
now = time()
inputs, labels, _ = self._unpack_next_element(next_element)
prob = self._model(inputs).asnumpy()
if self._uncertainty is not None:
prob_var = self._uncertainty.eval_epistemic_uncertainty(inputs)
prob_sd = np.sqrt(prob_var)
else:
prob_var = prob_sd = None
for idx, inp in enumerate(inputs):
gt_labels = labels[idx]
gt_probs = [float(prob[idx][i]) for i in gt_labels]
data_np = _convert_image_format(
np.expand_dims(inp.asnumpy(), 0), 'NCHW')
original_image = _np_to_image(_normalize(data_np), mode='RGB')
original_image_path = self._save_original_image(
self._count, original_image)
predicted_labels = [
int(i) for i in (prob[idx] > threshold).nonzero()[0]
]
predicted_probs = [
float(prob[idx][i]) for i in predicted_labels
]
has_uncertainty = False
gt_prob_sds = gt_prob_itl95_lows = gt_prob_itl95_his = None
predicted_prob_sds = predicted_prob_itl95_lows = predicted_prob_itl95_his = None
if prob_var is not None:
gt_prob_sds = [float(prob_sd[idx][i]) for i in gt_labels]
predicted_prob_sds = [
float(prob_sd[idx][i]) for i in predicted_labels
]
try:
gt_prob_itl95_lows, gt_prob_itl95_his = \
_calc_prob_interval(0.95, gt_probs, [float(prob_var[idx][i]) for i in gt_labels])
predicted_prob_itl95_lows, predicted_prob_itl95_his = \
_calc_prob_interval(0.95, predicted_probs, [float(prob_var[idx][i])
for i in predicted_labels])
has_uncertainty = True
except ValueError:
log.error(traceback.format_exc())
log.error("Error on calculating uncertainty")
union_labs = list(set(gt_labels + predicted_labels))
imageid_labels[str(self._count)] = union_labs
explain = Explain()
explain.sample_id = self._count
explain.image_path = original_image_path
summary.add_value("explainer", "sample", explain)
explain = Explain()
explain.sample_id = self._count
explain.ground_truth_label.extend(gt_labels)
explain.inference.ground_truth_prob.extend(gt_probs)
explain.inference.predicted_label.extend(predicted_labels)
explain.inference.predicted_prob.extend(predicted_probs)
if has_uncertainty:
explain.inference.ground_truth_prob_sd.extend(gt_prob_sds)
explain.inference.ground_truth_prob_itl95_low.extend(
gt_prob_itl95_lows)
explain.inference.ground_truth_prob_itl95_hi.extend(
gt_prob_itl95_his)
explain.inference.predicted_prob_sd.extend(
predicted_prob_sds)
explain.inference.predicted_prob_itl95_low.extend(
predicted_prob_itl95_lows)
explain.inference.predicted_prob_itl95_hi.extend(
predicted_prob_itl95_his)
summary.add_value("explainer", "inference", explain)
summary.record(1)
self._count += 1
print(spacer.format(
"Finish running and writing {}-th batch inference data."
" Time elapsed: {:.3f} s".format(j,
time() - now)),
end='')
return imageid_labels
def _run_sample(self, summary, next_element, sample_id_labels, threshold):
"""
Run inference for a sample.
Args:
summary (SummaryRecord): The summary object to store the data.
next_element (tuple): The next dataset sample.
sample_id_labels (dict): The sample id to labels dictionary.
threshold (float): The threshold for prediction.
"""
inputs, labels, _ = self._unpack_next_element(next_element)
prob = self._full_network(inputs).asnumpy()
if self._uncertainty is not None:
prob_var = self._uncertainty.eval_epistemic_uncertainty(inputs)
else:
prob_var = None
for idx, inp in enumerate(inputs):
gt_labels = labels[idx]
gt_probs = [float(prob[idx][i]) for i in gt_labels]
if prob_var is not None:
gt_prob_vars = [float(prob_var[idx][i]) for i in gt_labels]
gt_itl_lows, gt_itl_his, gt_prob_sds = \
self._calc_beta_intervals(gt_probs, gt_prob_vars)
data_np = _convert_image_format(np.expand_dims(inp.asnumpy(), 0),
'NCHW')
original_image = _np_to_image(_normalize(data_np), mode='RGB')
original_image_path = self._save_original_image(
self._sample_index, original_image)
predicted_labels = [
int(i) for i in (prob[idx] > threshold).nonzero()[0]
]
predicted_probs = [float(prob[idx][i]) for i in predicted_labels]
if prob_var is not None:
predicted_prob_vars = [
float(prob_var[idx][i]) for i in predicted_labels
]
predicted_itl_lows, predicted_itl_his, predicted_prob_sds = \
self._calc_beta_intervals(predicted_probs, predicted_prob_vars)
union_labs = list(set(gt_labels + predicted_labels))
sample_id_labels[str(self._sample_index)] = union_labs
explain = Explain()
explain.sample_id = self._sample_index
explain.image_path = original_image_path
summary.add_value("explainer", "sample", explain)
explain = Explain()
explain.sample_id = self._sample_index
explain.ground_truth_label.extend(gt_labels)
explain.inference.ground_truth_prob.extend(gt_probs)
explain.inference.predicted_label.extend(predicted_labels)
explain.inference.predicted_prob.extend(predicted_probs)
if prob_var is not None:
explain.inference.ground_truth_prob_sd.extend(gt_prob_sds)
explain.inference.ground_truth_prob_itl95_low.extend(
gt_itl_lows)
explain.inference.ground_truth_prob_itl95_hi.extend(gt_itl_his)
explain.inference.predicted_prob_sd.extend(predicted_prob_sds)
explain.inference.predicted_prob_itl95_low.extend(
predicted_itl_lows)
explain.inference.predicted_prob_itl95_hi.extend(
predicted_itl_his)
self._manifest["uncertainty"] = True
summary.add_value("explainer", "inference", explain)
summary.record(1)
if self._is_hoc_registered:
self._run_hoc(summary, self._sample_index, inputs[idx],
prob[idx])
