def _run_exp_step(self, next_element, explainer, sample_id_labels, summary): """ Run the explanation for each step and write explanation results into summary. Args: next_element (Tuple): Data of one step explainer (_Attribution): An Attribution object to generate saliency maps. sample_id_labels (dict): A dict that maps the sample id and its union labels. summary (SummaryRecord): The summary object to store the data Returns: list, List of dict that maps label to its corresponding saliency map. """ inputs, labels, _ = self._unpack_next_element(next_element) sample_index = self._sample_index unions = [] for _ in range(len(labels)): unions_labels = sample_id_labels[str(sample_index)] unions.append(unions_labels) sample_index += 1 batch_unions = self._make_label_batch(unions) saliency_dict_lst = [] if isinstance(explainer, RISE): batch_saliency_full = explainer(inputs, batch_unions) else: batch_saliency_full = [] for i in range(len(batch_unions[0])): batch_saliency = explainer(inputs, batch_unions[:, i]) batch_saliency_full.append(batch_saliency) concat = ms.ops.operations.Concat(1) batch_saliency_full = concat(tuple(batch_saliency_full)) for idx, union in enumerate(unions): saliency_dict = {} explain = Explain() explain.sample_id = self._sample_index for k, lab in enumerate(union): saliency = batch_saliency_full[idx:idx + 1, k:k + 1] saliency_dict[lab] = saliency saliency_np = _normalize(saliency.asnumpy().squeeze()) saliency_image = _np_to_image(saliency_np, mode='L') heatmap_path = self._save_heatmap(explainer.__class__.__name__, lab, self._sample_index, saliency_image) explanation = explain.explanation.add() explanation.explain_method = explainer.__class__.__name__ explanation.heatmap_path = heatmap_path explanation.label = lab summary.add_value("explainer", "explanation", explain) summary.record(1) self._sample_index += 1 saliency_dict_lst.append(saliency_dict) return saliency_dict_lst
def _run_exp_step(self, next_element, explainer, imageid_labels, summary): """ Run the explanation for each step and write explanation results into summary. Args: next_element (Tuple): data of one step explainer (_Attribution): an Attribution object to generate saliency maps. imageid_labels (dict): a dict that maps the image_id and its union labels. summary (SummaryRecord): the summary object to store the data Returns: List of dict that maps label to its corresponding saliency map. """ inputs, labels, _ = self._unpack_next_element(next_element) count = self._count unions = [] for _ in range(len(labels)): unions_labels = imageid_labels[str(count)] unions.append(unions_labels) count += 1 batch_unions = self._make_label_batch(unions) saliency_dict_lst = [] batch_saliency_full = [] for i in range(len(batch_unions[0])): batch_saliency = explainer(inputs, batch_unions[:, i]) batch_saliency_full.append(batch_saliency) for idx, union in enumerate(unions): saliency_dict = {} explain = Explain() explain.image_id = str(self._count) for k, lab in enumerate(union): saliency = batch_saliency_full[k][idx:idx + 1] saliency_dict[lab] = saliency saliency_np = _make_rgba(saliency) _, _, _, saliency_string = _make_image(_normalize(saliency_np)) explanation = explain.explanation.add() explanation.explain_method = explainer.__class__.__name__ explanation.label = lab explanation.heatmap = saliency_string summary.add_value("explainer", "explanation", explain) summary.record(1) self._count += 1 saliency_dict_lst.append(saliency_dict) return saliency_dict_lst
def _run_hoc(self, summary, sample_id, sample_input, prob): """ Run HOC search for a sample image, and then save the result to summary. Args: summary (SummaryRecord): The summary object to store the data. sample_id (int): The sample ID. sample_input (Union[Tensor, np.ndarray]): Sample image tensor in CHW or NCWH(N=1). prob (Union[Tensor, np.ndarray]): List of sample's classification prediction output, HOC will run for labels with prediction output strictly larger then HOC searcher's threshold(0.5 by default). """ if isinstance(sample_input, ms.Tensor): sample_input = sample_input.asnumpy() if len(sample_input.shape) == 3: sample_input = np.expand_dims(sample_input, axis=0) has_rec = False explain = Explain() explain.sample_id = sample_id str_mask = hoc.auto_str_mask(sample_input) compiled_mask = None for label_idx, label_prob in enumerate(prob): if label_prob > self._hoc_searcher.threshold: if compiled_mask is None: compiled_mask = hoc.compile_mask(str_mask, sample_input) try: edit_tree, layer_outputs = self._hoc_searcher.search( sample_input, label_idx, compiled_mask) except hoc.NoValidResultError: log.warning( f"No Hierarchical Occlusion result was found in sample#{sample_id} " f"label:{self._labels[label_idx]}, skipped.") continue has_rec = True hoc_rec = explain.hoc.add() hoc_rec.label = label_idx hoc_rec.mask = str_mask layer_count = edit_tree.max_layer + 1 for layer in range(layer_count): steps = edit_tree.get_layer_or_leaf_steps(layer) layer_output = layer_outputs[layer] hoc_layer = hoc_rec.layer.add() hoc_layer.prob = layer_output for step in steps: hoc_layer.box.extend(list(step.box)) if has_rec: summary.add_value("explainer", "hoc", explain) summary.record(1) self._manifest['hierarchical_occlusion'] = True
def _run_saliency(self, summary, sample_id_labels): """Run the saliency explanations.""" for explainer in self._explainers: explain = Explain() if self._benchmarkers: for bench in self._benchmarkers: bench.reset() print( f"Start running and writing explanation for {explainer.__class__.__name__}......" ) self._sample_index = 0 start = time() ds.config.set_seed(self._DATASET_SEED) self._run_explainer(summary, sample_id_labels, explainer) if not self._benchmarkers: continue for bench in self._benchmarkers: benchmark = explain.benchmark.add() benchmark.explain_method = explainer.__class__.__name__ benchmark.benchmark_method = bench.__class__.__name__ benchmark.total_score = bench.performance if isinstance(bench, LabelSensitiveMetric): benchmark.label_score.extend(bench.class_performances) self._spaced_print( "Finish running and writing explanation and benchmark data for {}. " "Time elapsed: {:.3f} s".format(explainer.__class__.__name__, time() - start)) summary.add_value('explainer', 'benchmark', explain) summary.record(1)
def _run_saliency(self, summary, sample_id_labels): """Run the saliency explanations.""" if self._benchmarkers is None or not self._benchmarkers: for exp in self._explainers: start = time() print("Start running and writing explanation data for {}......".format(exp.__class__.__name__)) self._sample_index = 0 ds.config.set_seed(self._DATASET_SEED) for idx, next_element in enumerate(self._dataset): now = time() self._spaced_print("Start running {}-th explanation data for {}......".format( idx, exp.__class__.__name__)) self._run_exp_step(next_element, exp, sample_id_labels, summary) self._spaced_print("Finish writing {}-th explanation data for {}. Time elapsed: " "{:.3f} s".format(idx, exp.__class__.__name__, time() - now)) self._spaced_print( "Finish running and writing explanation data for {}. Time elapsed: {:.3f} s".format( exp.__class__.__name__, time() - start)) else: for exp in self._explainers: explain = Explain() for bench in self._benchmarkers: bench.reset() print(f"Start running and writing explanation and " f"benchmark data for {exp.__class__.__name__}......") self._sample_index = 0 start = time() ds.config.set_seed(self._DATASET_SEED) for idx, next_element in enumerate(self._dataset): now = time() self._spaced_print("Start running {}-th explanation data for {}......".format( idx, exp.__class__.__name__)) saliency_dict_lst = self._run_exp_step(next_element, exp, sample_id_labels, summary) self._spaced_print( "Finish writing {}-th batch explanation data for {}. Time elapsed: {:.3f} s".format( idx, exp.__class__.__name__, time() - now)) for bench in self._benchmarkers: now = time() self._spaced_print( "Start running {}-th batch {} data for {}......".format( idx, bench.__class__.__name__, exp.__class__.__name__)) self._run_exp_benchmark_step(next_element, exp, bench, saliency_dict_lst) self._spaced_print( "Finish running {}-th batch {} data for {}. Time elapsed: {:.3f} s".format( idx, bench.__class__.__name__, exp.__class__.__name__, time() - now)) for bench in self._benchmarkers: benchmark = explain.benchmark.add() benchmark.explain_method = exp.__class__.__name__ benchmark.benchmark_method = bench.__class__.__name__ benchmark.total_score = bench.performance if isinstance(bench, LabelSensitiveMetric): benchmark.label_score.extend(bench.class_performances) self._spaced_print("Finish running and writing explanation and benchmark data for {}. " "Time elapsed: {:.3f} s".format(exp.__class__.__name__, time() - start)) summary.add_value('explainer', 'benchmark', explain) summary.record(1)
def _run_hoc(self, summary, sample_id, sample_input, prob): """ Run HOC search for a sample image, and then save the result to summary. Args: summary (SummaryRecord): The summary object to store the data. sample_id (int): The sample ID. sample_input (Union[Tensor, np.ndarray]): Sample image tensor in CHW or NCWH(N=1). prob (Union[Tensor, np.ndarray]): List of sample's classification prediction output, HOC will run for labels with prediction output strictly larger then HOC searcher's threshold(0.5 by default). """ if isinstance(sample_input, ms.Tensor): sample_input = sample_input.asnumpy() if len(sample_input.shape) == 3: sample_input = np.expand_dims(sample_input, axis=0) explain = None str_mask = hoc.auto_str_mask(sample_input) compiled_mask = None for label_idx, label_prob in enumerate(prob): if label_prob <= self._hoc_searcher.threshold: continue if compiled_mask is None: compiled_mask = hoc.compile_mask(str_mask, sample_input) try: edit_tree, layer_outputs = self._hoc_searcher.search( sample_input, label_idx, compiled_mask) except hoc.NoValidResultError: log.warning( f"No Hierarchical Occlusion result was found in sample#{sample_id} " f"label:{self._labels[label_idx]}, skipped.") continue if explain is None: explain = Explain() explain.sample_id = sample_id self._add_hoc_result_to_explain(label_idx, str_mask, edit_tree, layer_outputs, explain) if explain is not None: summary.add_value("explainer", "hoc", explain) summary.record(1) self._manifest['hierarchical_occlusion'] = True
def _run_inference(self, dataset, summary, threshod=0.5): """ Run inference for the dataset and write the inference related data into summary. Args: dataset (`ds`): the parsed dataset summary (`SummaryRecord`): the summary object to store the data threshold (float): the threshold for prediction. Returns: imageid_labels (dict): a dict that maps image_id and the union of its ground truth and predicted labels. """ imageid_labels = {} ds.config.set_seed(58) self._count = 0 for j, next_element in enumerate(dataset): now = time() inputs, labels, _ = self._unpack_next_element(next_element) prob = self._model(inputs).asnumpy() for idx, inp in enumerate(inputs): gt_labels = labels[idx] gt_probs = [float(prob[idx][i]) for i in gt_labels] data_np = _convert_image_format( np.expand_dims(inp.asnumpy(), 0), 'NCHW') _, _, _, image_string = _make_image(_normalize(data_np)) predicted_labels = [ int(i) for i in (prob[idx] > threshod).nonzero()[0] ] predicted_probs = [ float(prob[idx][i]) for i in predicted_labels ] union_labs = list(set(gt_labels + predicted_labels)) imageid_labels[str(self._count)] = union_labs explain = Explain() explain.image_id = str(self._count) explain.image_data = image_string summary.add_value("explainer", "image", explain) explain = Explain() explain.image_id = str(self._count) explain.ground_truth_label.extend(gt_labels) explain.inference.ground_truth_prob.extend(gt_probs) explain.inference.predicted_label.extend(predicted_labels) explain.inference.predicted_prob.extend(predicted_probs) summary.add_value("explainer", "inference", explain) summary.record(1) self._count += 1 print( "Finish running and writing {}-th batch inference data. Time elapsed: {}s" .format(j, time() - now)) return imageid_labels
def _add_exp_step_samples(self, explainer, sample_label_sets, batch_saliency_full, summary): """ Add explanation results of samples to summary record. Args: explainer (Attribution): The explainer to be run. sample_label_sets (list[list[int]]): The label sets of samples. batch_saliency_full (Tensor): The saliency output from explainer. summary (SummaryRecord): The summary record. """ saliency_dict_lst = [] has_saliency_rec = False for idx, label_set in enumerate(sample_label_sets): saliency_dict = {} explain = Explain() explain.sample_id = self._sample_index for k, lab in enumerate(label_set): saliency = batch_saliency_full[idx:idx + 1, k:k + 1] saliency_dict[lab] = saliency saliency_np = _normalize(saliency.asnumpy().squeeze()) saliency_image = _np_to_image(saliency_np, mode='L') heatmap_path = self._save_heatmap(explainer.__class__.__name__, lab, self._sample_index, saliency_image) explanation = explain.explanation.add() explanation.explain_method = explainer.__class__.__name__ explanation.heatmap_path = heatmap_path explanation.label = lab has_saliency_rec = True summary.add_value("explainer", "explanation", explain) summary.record(1) self._sample_index += 1 saliency_dict_lst.append(saliency_dict) return saliency_dict_lst, has_saliency_rec
def _run_inference(self, summary, threshold=0.5): """ Run inference for the dataset and write the inference related data into summary. Args: summary (SummaryRecord): The summary object to store the data threshold (float): The threshold for prediction. Returns: dict, The map of sample d to the union of its ground truth and predicted labels. """ sample_id_labels = {} self._sample_index = 0 ds.config.set_seed(self._DATASET_SEED) for j, next_element in enumerate(self._dataset): now = time() inputs, labels, _ = self._unpack_next_element(next_element) prob = self._full_network(inputs).asnumpy() for idx, inp in enumerate(inputs): gt_labels = labels[idx] gt_probs = [float(prob[idx][i]) for i in gt_labels] data_np = _convert_image_format(np.expand_dims(inp.asnumpy(), 0), 'NCHW') original_image = _np_to_image(_normalize(data_np), mode='RGB') original_image_path = self._save_original_image(self._sample_index, original_image) predicted_labels = [int(i) for i in (prob[idx] > threshold).nonzero()[0]] predicted_probs = [float(prob[idx][i]) for i in predicted_labels] union_labs = list(set(gt_labels + predicted_labels)) sample_id_labels[str(self._sample_index)] = union_labs explain = Explain() explain.sample_id = self._sample_index explain.image_path = original_image_path summary.add_value("explainer", "sample", explain) explain = Explain() explain.sample_id = self._sample_index explain.ground_truth_label.extend(gt_labels) explain.inference.ground_truth_prob.extend(gt_probs) explain.inference.predicted_label.extend(predicted_labels) explain.inference.predicted_prob.extend(predicted_probs) summary.add_value("explainer", "inference", explain) summary.record(1) self._sample_index += 1 self._spaced_print("Finish running and writing {}-th batch inference data." " Time elapsed: {:.3f} s".format(j, time() - now), end='') return sample_id_labels
def _save_metadata(self, summary): """Save metadata of the explain job to summary.""" print("Start writing metadata......") explain = Explain() explain.metadata.label.extend(self._labels) if self._is_saliency_registered: exp_names = [exp.__class__.__name__ for exp in self._explainers] explain.metadata.explain_method.extend(exp_names) if self._benchmarkers is not None: bench_names = [bench.__class__.__name__ for bench in self._benchmarkers] explain.metadata.benchmark_method.extend(bench_names) summary.add_value("explainer", "metadata", explain) summary.record(1) print("Finish writing metadata.")
def _run_inference(self, summary, threshold=0.5): """ Run inference for the dataset and write the inference related data into summary. Args: summary (SummaryRecord): The summary object to store the data. threshold (float): The threshold for prediction. Returns: dict, The map of sample d to the union of its ground truth and predicted labels. """ has_uncertainty_rec = False sample_id_labels = {} self._sample_index = 0 ds.config.set_seed(self._DATASET_SEED) for j, next_element in enumerate(self._dataset): now = time() inputs, labels, _ = self._unpack_next_element(next_element) prob = self._full_network(inputs).asnumpy() if self._uncertainty is not None: prob_var = self._uncertainty.eval_epistemic_uncertainty(inputs) else: prob_var = None for idx, inp in enumerate(inputs): gt_labels = labels[idx] gt_probs = [float(prob[idx][i]) for i in gt_labels] if prob_var is not None: gt_prob_vars = [float(prob_var[idx][i]) for i in gt_labels] gt_itl_lows, gt_itl_his, gt_prob_sds = \ self._calc_beta_intervals(gt_probs, gt_prob_vars) data_np = _convert_image_format(np.expand_dims(inp.asnumpy(), 0), 'NCHW') original_image = _np_to_image(_normalize(data_np), mode='RGB') original_image_path = self._save_original_image(self._sample_index, original_image) predicted_labels = [int(i) for i in (prob[idx] > threshold).nonzero()[0]] predicted_probs = [float(prob[idx][i]) for i in predicted_labels] if prob_var is not None: predicted_prob_vars = [float(prob_var[idx][i]) for i in predicted_labels] predicted_itl_lows, predicted_itl_his, predicted_prob_sds = \ self._calc_beta_intervals(predicted_probs, predicted_prob_vars) union_labs = list(set(gt_labels + predicted_labels)) sample_id_labels[str(self._sample_index)] = union_labs explain = Explain() explain.sample_id = self._sample_index explain.image_path = original_image_path summary.add_value("explainer", "sample", explain) explain = Explain() explain.sample_id = self._sample_index explain.ground_truth_label.extend(gt_labels) explain.inference.ground_truth_prob.extend(gt_probs) explain.inference.predicted_label.extend(predicted_labels) explain.inference.predicted_prob.extend(predicted_probs) if prob_var is not None: explain.inference.ground_truth_prob_sd.extend(gt_prob_sds) explain.inference.ground_truth_prob_itl95_low.extend(gt_itl_lows) explain.inference.ground_truth_prob_itl95_hi.extend(gt_itl_his) explain.inference.predicted_prob_sd.extend(predicted_prob_sds) explain.inference.predicted_prob_itl95_low.extend(predicted_itl_lows) explain.inference.predicted_prob_itl95_hi.extend(predicted_itl_his) has_uncertainty_rec = True summary.add_value("explainer", "inference", explain) summary.record(1) if self._is_hoc_registered: self._run_hoc(summary, self._sample_index, inputs[idx], prob[idx]) self._sample_index += 1 self._spaced_print("Finish running and writing {}-th batch inference data." " Time elapsed: {:.3f} s".format(j, time() - now)) if has_uncertainty_rec: self._manifest["uncertainty"] = True return sample_id_labels
def _run_inference(self, dataset, summary, threshold=0.5): """ Run inference for the dataset and write the inference related data into summary. Args: dataset (`ds`): the parsed dataset summary (`SummaryRecord`): the summary object to store the data threshold (float): the threshold for prediction. Returns: imageid_labels (dict): a dict that maps image_id and the union of its ground truth and predicted labels. """ spacer = '{:120}\r' imageid_labels = {} ds.config.set_seed(_SEED) self._count = 0 for j, next_element in enumerate(dataset): now = time() inputs, labels, _ = self._unpack_next_element(next_element) prob = self._model(inputs).asnumpy() if self._uncertainty is not None: prob_var = self._uncertainty.eval_epistemic_uncertainty(inputs) prob_sd = np.sqrt(prob_var) else: prob_var = prob_sd = None for idx, inp in enumerate(inputs): gt_labels = labels[idx] gt_probs = [float(prob[idx][i]) for i in gt_labels] data_np = _convert_image_format( np.expand_dims(inp.asnumpy(), 0), 'NCHW') original_image = _np_to_image(_normalize(data_np), mode='RGB') original_image_path = self._save_original_image( self._count, original_image) predicted_labels = [ int(i) for i in (prob[idx] > threshold).nonzero()[0] ] predicted_probs = [ float(prob[idx][i]) for i in predicted_labels ] has_uncertainty = False gt_prob_sds = gt_prob_itl95_lows = gt_prob_itl95_his = None predicted_prob_sds = predicted_prob_itl95_lows = predicted_prob_itl95_his = None if prob_var is not None: gt_prob_sds = [float(prob_sd[idx][i]) for i in gt_labels] predicted_prob_sds = [ float(prob_sd[idx][i]) for i in predicted_labels ] try: gt_prob_itl95_lows, gt_prob_itl95_his = \ _calc_prob_interval(0.95, gt_probs, [float(prob_var[idx][i]) for i in gt_labels]) predicted_prob_itl95_lows, predicted_prob_itl95_his = \ _calc_prob_interval(0.95, predicted_probs, [float(prob_var[idx][i]) for i in predicted_labels]) has_uncertainty = True except ValueError: log.error(traceback.format_exc()) log.error("Error on calculating uncertainty") union_labs = list(set(gt_labels + predicted_labels)) imageid_labels[str(self._count)] = union_labs explain = Explain() explain.sample_id = self._count explain.image_path = original_image_path summary.add_value("explainer", "sample", explain) explain = Explain() explain.sample_id = self._count explain.ground_truth_label.extend(gt_labels) explain.inference.ground_truth_prob.extend(gt_probs) explain.inference.predicted_label.extend(predicted_labels) explain.inference.predicted_prob.extend(predicted_probs) if has_uncertainty: explain.inference.ground_truth_prob_sd.extend(gt_prob_sds) explain.inference.ground_truth_prob_itl95_low.extend( gt_prob_itl95_lows) explain.inference.ground_truth_prob_itl95_hi.extend( gt_prob_itl95_his) explain.inference.predicted_prob_sd.extend( predicted_prob_sds) explain.inference.predicted_prob_itl95_low.extend( predicted_prob_itl95_lows) explain.inference.predicted_prob_itl95_hi.extend( predicted_prob_itl95_his) summary.add_value("explainer", "inference", explain) summary.record(1) self._count += 1 print(spacer.format( "Finish running and writing {}-th batch inference data." " Time elapsed: {:.3f} s".format(j, time() - now)), end='') return imageid_labels
def run(self, dataset: Tuple, explainers: List, benchmarkers: Optional[List] = None, uncertainty: Optional[UncertaintyEvaluation] = None, activation_fn: Optional[Cell] = Softmax()): """ Genereates results and writes results into the summary files in `summary_dir` specified during the object initialization. Args: dataset (tuple): A tuple that contains `mindspore.dataset` object for iteration and its labels. - dataset[0]: A `mindspore.dataset` object to provide data to explain. - dataset[1]: A list of string that specifies the label names of the dataset. explainers (list[Explanation]): A list of explanation objects to generate attribution results. Explanation object is an instance initialized with the explanation methods in module `mindspore.explainer.explanation`. benchmarkers (list[Benchmark], optional): A list of benchmark objects to generate evaluation results. Default: None uncertainty (UncertaintyEvaluation, optional): An uncertainty evaluation object to evaluate the inference uncertainty of samples. activation_fn (Cell, optional): The activation layer that transforms the output of the network to label probability distribution :math:`P(y|x)`. Default: Softmax(). Examples: >>> from mindspore.explainer import ExplainRunner >>> from mindspore.explainer.explanation import GuidedBackprop, Gradient >>> from mindspore.nn import Softmax >>> from mindspore.train.serialization import load_checkpoint, load_param_into_net >>> # Prepare the dataset for explaining and evaluation, e.g., Cifar10 >>> dataset = get_dataset('/path/to/Cifar10_dataset') >>> classes = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'turck'] >>> # load checkpoint to a network, e.g. checkpoint of resnet50 trained on Cifar10 >>> param_dict = load_checkpoint("checkpoint.ckpt") >>> net = resnet50(len(classes)) >>> load_param_into_net(net, param_dict) >>> gbp = GuidedBackprop(net) >>> gradient = Gradient(net) >>> explainers = [gbp, gradient] >>> # runner is an ExplainRunner object >>> runner.run((dataset, classes), explainers, activation_fn=Softmax()) """ check_value_type("dataset", dataset, tuple) if len(dataset) != 2: raise ValueError( "Argument `dataset` should be a tuple with length = 2.") dataset, classes = dataset if benchmarkers is None: benchmarkers = [] self._verify_data_form(dataset, benchmarkers) self._classes = classes check_value_type("explainers", explainers, list) if not explainers: raise ValueError("Argument `explainers` must be a non-empty list") for exp in explainers: if not isinstance(exp, Attribution): raise TypeError( "Argument `explainers` should be a list of objects of classes in " "`mindspore.explainer.explanation`.") if benchmarkers: check_value_type("benchmarkers", benchmarkers, list) for bench in benchmarkers: if not isinstance(bench, AttributionMetric): raise TypeError( "Argument `benchmarkers` should be a list of objects of classes in explanation" "`mindspore.explainer.benchmark`.") check_value_type("activation_fn", activation_fn, Cell) self._model = ms.nn.SequentialCell( [explainers[0].model, activation_fn]) next_element = next(dataset.create_tuple_iterator()) inputs, _, _ = self._unpack_next_element(next_element) prop_test = self._model(inputs) check_value_type("output of model im explainer", prop_test, ms.Tensor) if prop_test.shape[1] != len(self._classes): raise ValueError( "The dimension of model output does not match the length of dataset classes. Please " "check dataset classes or the black-box model in the explainer again." ) if uncertainty is not None: check_value_type("uncertainty", uncertainty, UncertaintyEvaluation) prop_var_test = uncertainty.eval_epistemic_uncertainty(inputs) check_value_type("output of uncertainty", prop_var_test, np.ndarray) if prop_var_test.shape[1] != len(self._classes): raise ValueError( "The dimension of uncertainty output does not match the length of dataset classes" "classes. Please check dataset classes or the black-box model in the explainer again." ) self._uncertainty = uncertainty else: self._uncertainty = None with SummaryRecord(self._summary_dir) as summary: spacer = '{:120}\r' print("Start running and writing......") begin = time() print("Start writing metadata......") self._summary_timestamp = _extract_timestamp( summary.event_file_name) if self._summary_timestamp is None: raise RuntimeError( "Cannot extract timestamp from summary filename!" " It should contains a timestamp of 10 digits.") explain = Explain() explain.metadata.label.extend(classes) exp_names = [exp.__class__.__name__ for exp in explainers] explain.metadata.explain_method.extend(exp_names) if benchmarkers: bench_names = [ bench.__class__.__name__ for bench in benchmarkers ] explain.metadata.benchmark_method.extend(bench_names) summary.add_value("explainer", "metadata", explain) summary.record(1) print("Finish writing metadata.") now = time() print("Start running and writing inference data.....") imageid_labels = self._run_inference(dataset, summary) print( spacer.format("Finish running and writing inference data. " "Time elapsed: {:.3f} s".format(time() - now))) if not benchmarkers: for exp in explainers: start = time() print( "Start running and writing explanation data for {}......" .format(exp.__class__.__name__)) self._count = 0 ds.config.set_seed(_SEED) for idx, next_element in enumerate(dataset): now = time() self._run_exp_step(next_element, exp, imageid_labels, summary) print(spacer.format( "Finish writing {}-th explanation data for {}. Time elapsed: " "{:.3f} s".format(idx, exp.__class__.__name__, time() - now)), end='') print( spacer.format( "Finish running and writing explanation data for {}. Time elapsed: {:.3f} s" .format(exp.__class__.__name__, time() - start))) else: for exp in explainers: explain = Explain() for bench in benchmarkers: bench.reset() print(f"Start running and writing explanation and " f"benchmark data for {exp.__class__.__name__}......") self._count = 0 start = time() ds.config.set_seed(_SEED) for idx, next_element in enumerate(dataset): now = time() saliency_dict_lst = self._run_exp_step( next_element, exp, imageid_labels, summary) print(spacer.format( "Finish writing {}-th batch explanation data for {}. Time elapsed: {:.3f} s" .format(idx, exp.__class__.__name__, time() - now)), end='') for bench in benchmarkers: now = time() self._run_exp_benchmark_step( next_element, exp, bench, saliency_dict_lst) print(spacer.format( "Finish running {}-th batch {} data for {}. Time elapsed: {:.3f} s" .format(idx, bench.__class__.__name__, exp.__class__.__name__, time() - now)), end='') for bench in benchmarkers: benchmark = explain.benchmark.add() benchmark.explain_method = exp.__class__.__name__ benchmark.benchmark_method = bench.__class__.__name__ benchmark.total_score = bench.performance if isinstance(bench, LabelSensitiveMetric): benchmark.label_score.extend( bench.class_performances) print( spacer.format( "Finish running and writing explanation and benchmark data for {}. " "Time elapsed: {:.3f} s".format( exp.__class__.__name__, time() - start))) summary.add_value('explainer', 'benchmark', explain) summary.record(1) print("Finish running and writing. Total time elapsed: {:.3f} s". format(time() - begin))
def run(self, dataset: Tuple, explainers: List, benchmarkers: Optional[List] = None): """ Genereate results and write results into the summary files in `self.summary_dir`. Args: dataset (tuple): A tuple that contains `mindspore.dataset` object for iteration and its labels. - dataset[0], a `mindspore.dataset` object to provide data to explain. - dataset[1], a list of string that specifies the label names of the dataset. explainers (list): A list of explanation objects to generate _attribution results. benchmarkers (list): A list of benchmark objects to generate evaluation results. Default: None Examples: >>> from mindspore.explainer.explanation import GuidedBackprop, Gradient >>> # obtain dataset object >>> dataset = get_dataset() >>> classes = ["cat", "dog", ...] >>> # load checkpoint to a network, e.g. resnet50 >>> param_dict = load_checkpoint("checkpoint.ckpt") >>> net = resnet50(len(classes)) >>> load_parama_into_net(net, param_dict) >>> # bind net with its output activation >>> model = nn.SequentialCell([net, nn.Sigmoid()]) >>> gbp = GuidedBackprop(model) >>> gradient = Gradient(model) >>> runner = ExplainRunner("./") >>> explainers = [gbp, gradient] >>> runner.run((dataset, classes), explainers) """ if not isinstance(dataset, tuple): raise TypeError("Argument `dataset` must be a tuple.") if len(dataset) != 2: raise ValueError("Argument `dataset` should be a tuple with length = 2.") dataset, classes = dataset self._verify_data_form(dataset, benchmarkers) self._classes = classes if explainers is None or not explainers: raise ValueError("Argument `explainers` can neither be None nor empty.") for exp in explainers: if not isinstance(exp, Attribution) or not isinstance(explainers, list): raise TypeError("Argument explainers should be a list of objects of classes in " "`mindspore.explainer.explanation._attribution`.") if benchmarkers is not None: for bench in benchmarkers: if not isinstance(bench, AttributionMetric) or not isinstance(explainers, list): raise TypeError("Argument benchmarkers should be a list of objects of classes in explanation" "`mindspore.explainer.benchmark._attribution`.") self._model = explainers[0].model with SummaryRecord(self._summary_dir) as summary: print("Start running and writing......") begin = time() print("Start writing metadata.") explain = Explain() explain.metadata.label.extend(classes) exp_names = [exp.__class__.__name__ for exp in explainers] explain.metadata.explain_method.extend(exp_names) if benchmarkers is not None: bench_names = [bench.__class__.__name__ for bench in benchmarkers] explain.metadata.benchmark_method.extend(bench_names) summary.add_value("explainer", "metadata", explain) summary.record(1) print("Finish writing metadata.") now = time() print("Start running and writing inference data......") imageid_labels = self._run_inference(dataset, summary) print("Finish running and writing inference data. Time elapsed: {}s".format(time() - now)) if benchmarkers is None: for exp in explainers: start = time() print("Start running and writing explanation data for {}......".format(exp.__class__.__name__)) self._count = 0 ds.config.set_seed(58) for idx, next_element in enumerate(dataset): now = time() self._run_exp_step(next_element, exp, imageid_labels, summary) print("Finish writing {}-th explanation data. Time elapsed: {}".format( idx, time() - now)) print("Finish running and writing explanation data for {}. Time elapsed: {}".format( exp.__class__.__name__, time() - start)) else: for exp in explainers: explain = Explain() for bench in benchmarkers: bench.reset() print(f"Start running and writing explanation and benchmark data for {exp.__class__.__name__}.") self._count = 0 start = time() ds.config.set_seed(58) for idx, next_element in enumerate(dataset): now = time() saliency_dict_lst = self._run_exp_step(next_element, exp, imageid_labels, summary) print("Finish writing {}-th batch explanation data. Time elapsed: {}s".format( idx, time() - now)) for bench in benchmarkers: now = time() self._run_exp_benchmark_step(next_element, exp, bench, saliency_dict_lst) print("Finish running {}-th batch benchmark data for {}. Time elapsed: {}s".format( idx, bench.__class__.__name__, time() - now)) for bench in benchmarkers: benchmark = explain.benchmark.add() benchmark.explain_method = exp.__class__.__name__ benchmark.benchmark_method = bench.__class__.__name__ benchmark.total_score = bench.performance benchmark.label_score.extend(bench.class_performances) print("Finish running and writing explanation and benchmark data for {}. " "Time elapsed: {}s".format(exp.__class__.__name__, time() - start)) summary.add_value('explainer', 'benchmark', explain) summary.record(1) print("Finish running and writing. Total time elapsed: {}s".format(time() - begin))
def _run_sample(self, summary, next_element, sample_id_labels, threshold): """ Run inference for a sample. Args: summary (SummaryRecord): The summary object to store the data. next_element (tuple): The next dataset sample. sample_id_labels (dict): The sample id to labels dictionary. threshold (float): The threshold for prediction. """ inputs, labels, _ = self._unpack_next_element(next_element) prob = self._full_network(inputs).asnumpy() if self._uncertainty is not None: prob_var = self._uncertainty.eval_epistemic_uncertainty(inputs) else: prob_var = None for idx, inp in enumerate(inputs): gt_labels = labels[idx] gt_probs = [float(prob[idx][i]) for i in gt_labels] if prob_var is not None: gt_prob_vars = [float(prob_var[idx][i]) for i in gt_labels] gt_itl_lows, gt_itl_his, gt_prob_sds = \ self._calc_beta_intervals(gt_probs, gt_prob_vars) data_np = _convert_image_format(np.expand_dims(inp.asnumpy(), 0), 'NCHW') original_image = _np_to_image(_normalize(data_np), mode='RGB') original_image_path = self._save_original_image( self._sample_index, original_image) predicted_labels = [ int(i) for i in (prob[idx] > threshold).nonzero()[0] ] predicted_probs = [float(prob[idx][i]) for i in predicted_labels] if prob_var is not None: predicted_prob_vars = [ float(prob_var[idx][i]) for i in predicted_labels ] predicted_itl_lows, predicted_itl_his, predicted_prob_sds = \ self._calc_beta_intervals(predicted_probs, predicted_prob_vars) union_labs = list(set(gt_labels + predicted_labels)) sample_id_labels[str(self._sample_index)] = union_labs explain = Explain() explain.sample_id = self._sample_index explain.image_path = original_image_path summary.add_value("explainer", "sample", explain) explain = Explain() explain.sample_id = self._sample_index explain.ground_truth_label.extend(gt_labels) explain.inference.ground_truth_prob.extend(gt_probs) explain.inference.predicted_label.extend(predicted_labels) explain.inference.predicted_prob.extend(predicted_probs) if prob_var is not None: explain.inference.ground_truth_prob_sd.extend(gt_prob_sds) explain.inference.ground_truth_prob_itl95_low.extend( gt_itl_lows) explain.inference.ground_truth_prob_itl95_hi.extend(gt_itl_his) explain.inference.predicted_prob_sd.extend(predicted_prob_sds) explain.inference.predicted_prob_itl95_low.extend( predicted_itl_lows) explain.inference.predicted_prob_itl95_hi.extend( predicted_itl_his) self._manifest["uncertainty"] = True summary.add_value("explainer", "inference", explain) summary.record(1) if self._is_hoc_registered: self._run_hoc(summary, self._sample_index, inputs[idx], prob[idx])