def test_scalar_summary_with_ge_2():
""" test_scalar_summary_with_ge_2 """
log.debug("begin test_scalar_summary_with_ge_2")
# step 0: create the thread
with SummaryRecord(SUMMARY_DIR, file_suffix="_MS_SCALAR") as test_writer:
# step 1: create the network for summary
x = Tensor(np.array([1.1]).astype(np.float32))
y = Tensor(np.array([1.2]).astype(np.float32))
net = SummaryDemo()
net.set_train()
# step 2: create the Event
steps = 100
for i in range(1, steps):
x = Tensor(np.array([1.1]).astype(np.float32))
y = Tensor(np.array([1.2]).astype(np.float32))
net(x, y)
test_writer.record(i)
log.debug("finished test_scalar_summary_with_ge_2")
def test_histogram_summary_nan_inf():
"""Test histogram summary, input tensor has nan."""
with tempfile.TemporaryDirectory() as tmp_dir:
with SummaryRecord(tmp_dir,
file_suffix="_MS_HISTOGRAM") as test_writer:
dim1 = 100
dim2 = 100
arr = np.ones([dim1, dim2])
arr[0][0] = np.nan
arr[0][1] = np.inf
arr[0][2] = -np.inf
test_data = _wrap_test_data(Tensor(arr))
_cache_summary_tensor_data(test_data)
test_writer.record(step=1)
file_name = os.path.join(tmp_dir, test_writer.event_file_name)
reader = SummaryReader(file_name)
event = reader.read_event()
LOG.debug(event)
assert event.summary.value[0].histogram.nan_count == 1
def test_image_summary_train():
""" test_image_summary_train """
dataset = get_dataset()
log.debug("begin test_image_summary_sample")
# step 0: create the thread
test_writer = SummaryRecord(SUMMARY_DIR, file_suffix="_MS_IMAGE")
# step 1: create the test data for summary
# step 2: create the Event
model = get_model()
fn = ImageSummaryCallback(test_writer)
summary_recode = SummaryStep(fn, 1)
model.train(2, dataset, callbacks=summary_recode)
# step 3: send the event to mq
# step 4: accept the event and write the file
test_writer.close()
log.debug("finished test_image_summary_sample")
def test_histogram_multi_summary():
"""Test histogram multiple step."""
with tempfile.TemporaryDirectory() as tmp_dir:
test_writer = SummaryRecord(tmp_dir, file_suffix="_MS_HISTOGRAM")
rng = np.random.RandomState(10)
size = 50
num_step = 5
for i in range(num_step):
arr = rng.normal(size=size)
test_data = _wrap_test_data(Tensor(arr))
_cache_summary_tensor_data(test_data)
test_writer.record(step=i)
test_writer.close()
file_name = os.path.join(tmp_dir, test_writer.event_file_name)
reader = SummaryReader(file_name)
for _ in range(num_step):
event = reader.read_event()
assert event.summary.value[0].histogram.count == size
def test_graph_summary_sample():
""" test_graph_summary_sample """
log.debug("begin test_graph_summary_sample")
dataset = get_dataset()
net = Net()
loss = nn.SoftmaxCrossEntropyWithLogits()
optim = Momentum(net.trainable_params(), 0.1, 0.9)
context.set_context(mode=context.GRAPH_MODE)
model = Model(net, loss_fn=loss, optimizer=optim, metrics=None)
test_writer = SummaryRecord(SUMMARY_DIR,
file_suffix="_MS_GRAPH",
network=model._train_network)
model.train(2, dataset)
# step 2: create the Event
for i in range(1, 5):
test_writer.record(i)
# step 3: send the event to mq
# step 4: accept the event and write the file
test_writer.close()
log.debug("finished test_graph_summary_sample")
def test_image_summary_data():
""" test_image_summary_data """
dataset = get_dataset()
test_data_list = []
i = 1
for next_element in dataset:
tag = "image_" + str(i) + "[:Image]"
dct = {}
dct["name"] = tag
dct["data"] = Tensor(next_element[0])
test_data_list.append(dct)
i += 1
log.debug("begin test_image_summary_sample")
# step 0: create the thread
with SummaryRecord(SUMMARY_DIR, file_suffix="_MS_IMAGE") as test_writer:
# step 1: create the test data for summary
# step 2: create the Event
_cache_summary_tensor_data(test_data_list)
test_writer.record(1)
log.debug("finished test_image_summary_sample")
def __enter__(self):
self._first_step = True
self._dataset_sink_mode = True
self._record = SummaryRecord(log_dir=self._summary_dir)
return self
def __enter__(self):
self._record = SummaryRecord(log_dir=self._summary_dir,
max_file_size=self._max_file_size)
self._first_step, self._dataset_sink_mode = True, True
return self
def test_step_of_record_with_type_error(self, step):
summary_dir = tempfile.mkdtemp(dir=self.base_summary_dir)
with pytest.raises(TypeError):
with SummaryRecord(summary_dir) as sr:
sr.record(step)
def test_log_dir_with_type_error(self, log_dir):
with pytest.raises(TypeError):
with SummaryRecord(log_dir):
pass
def __enter__(self):
self._record = SummaryRecord(log_dir=self._summary_dir)
return self
def run(self,
dataset: Tuple,
explainers: List,
benchmarkers: Optional[List] = None,
uncertainty: Optional[UncertaintyEvaluation] = None,
activation_fn: Optional[Cell] = Softmax()):
"""
Genereates results and writes results into the summary files in `summary_dir` specified during the object
initialization.
Args:
dataset (tuple): A tuple that contains `mindspore.dataset` object for iteration and its labels.
- dataset[0]: A `mindspore.dataset` object to provide data to explain.
- dataset[1]: A list of string that specifies the label names of the dataset.
explainers (list[Explanation]): A list of explanation objects to generate attribution results. Explanation
object is an instance initialized with the explanation methods in module
`mindspore.explainer.explanation`.
benchmarkers (list[Benchmark], optional): A list of benchmark objects to generate evaluation results.
Default: None
uncertainty (UncertaintyEvaluation, optional): An uncertainty evaluation object to evaluate the inference
uncertainty of samples.
activation_fn (Cell, optional): The activation layer that transforms the output of the network to
label probability distribution :math:`P(y|x)`. Default: Softmax().
Examples:
>>> from mindspore.explainer import ExplainRunner
>>> from mindspore.explainer.explanation import GuidedBackprop, Gradient
>>> from mindspore.nn import Softmax
>>> from mindspore.train.serialization import load_checkpoint, load_param_into_net
>>> # Prepare the dataset for explaining and evaluation, e.g., Cifar10
>>> dataset = get_dataset('/path/to/Cifar10_dataset')
>>> classes = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'turck']
>>> # load checkpoint to a network, e.g. checkpoint of resnet50 trained on Cifar10
>>> param_dict = load_checkpoint("checkpoint.ckpt")
>>> net = resnet50(len(classes))
>>> load_param_into_net(net, param_dict)
>>> gbp = GuidedBackprop(net)
>>> gradient = Gradient(net)
>>> explainers = [gbp, gradient]
>>> # runner is an ExplainRunner object
>>> runner.run((dataset, classes), explainers, activation_fn=Softmax())
"""
check_value_type("dataset", dataset, tuple)
if len(dataset) != 2:
raise ValueError(
"Argument `dataset` should be a tuple with length = 2.")
dataset, classes = dataset
if benchmarkers is None:
benchmarkers = []
self._verify_data_form(dataset, benchmarkers)
self._classes = classes
check_value_type("explainers", explainers, list)
if not explainers:
raise ValueError("Argument `explainers` must be a non-empty list")
for exp in explainers:
if not isinstance(exp, Attribution):
raise TypeError(
"Argument `explainers` should be a list of objects of classes in "
"`mindspore.explainer.explanation`.")
if benchmarkers:
check_value_type("benchmarkers", benchmarkers, list)
for bench in benchmarkers:
if not isinstance(bench, AttributionMetric):
raise TypeError(
"Argument `benchmarkers` should be a list of objects of classes in explanation"
"`mindspore.explainer.benchmark`.")
check_value_type("activation_fn", activation_fn, Cell)
self._model = ms.nn.SequentialCell(
[explainers[0].model, activation_fn])
next_element = next(dataset.create_tuple_iterator())
inputs, _, _ = self._unpack_next_element(next_element)
prop_test = self._model(inputs)
check_value_type("output of model im explainer", prop_test, ms.Tensor)
if prop_test.shape[1] != len(self._classes):
raise ValueError(
"The dimension of model output does not match the length of dataset classes. Please "
"check dataset classes or the black-box model in the explainer again."
)
if uncertainty is not None:
check_value_type("uncertainty", uncertainty, UncertaintyEvaluation)
prop_var_test = uncertainty.eval_epistemic_uncertainty(inputs)
check_value_type("output of uncertainty", prop_var_test,
np.ndarray)
if prop_var_test.shape[1] != len(self._classes):
raise ValueError(
"The dimension of uncertainty output does not match the length of dataset classes"
"classes. Please check dataset classes or the black-box model in the explainer again."
)
self._uncertainty = uncertainty
else:
self._uncertainty = None
with SummaryRecord(self._summary_dir) as summary:
spacer = '{:120}\r'
print("Start running and writing......")
begin = time()
print("Start writing metadata......")
self._summary_timestamp = _extract_timestamp(
summary.event_file_name)
if self._summary_timestamp is None:
raise RuntimeError(
"Cannot extract timestamp from summary filename!"
" It should contains a timestamp of 10 digits.")
explain = Explain()
explain.metadata.label.extend(classes)
exp_names = [exp.__class__.__name__ for exp in explainers]
explain.metadata.explain_method.extend(exp_names)
if benchmarkers:
bench_names = [
bench.__class__.__name__ for bench in benchmarkers
]
explain.metadata.benchmark_method.extend(bench_names)
summary.add_value("explainer", "metadata", explain)
summary.record(1)
print("Finish writing metadata.")
now = time()
print("Start running and writing inference data.....")
imageid_labels = self._run_inference(dataset, summary)
print(
spacer.format("Finish running and writing inference data. "
"Time elapsed: {:.3f} s".format(time() - now)))
if not benchmarkers:
for exp in explainers:
start = time()
print(
"Start running and writing explanation data for {}......"
.format(exp.__class__.__name__))
self._count = 0
ds.config.set_seed(_SEED)
for idx, next_element in enumerate(dataset):
now = time()
self._run_exp_step(next_element, exp, imageid_labels,
summary)
print(spacer.format(
"Finish writing {}-th explanation data for {}. Time elapsed: "
"{:.3f} s".format(idx, exp.__class__.__name__,
time() - now)),
end='')
print(
spacer.format(
"Finish running and writing explanation data for {}. Time elapsed: {:.3f} s"
.format(exp.__class__.__name__,
time() - start)))
else:
for exp in explainers:
explain = Explain()
for bench in benchmarkers:
bench.reset()
print(f"Start running and writing explanation and "
f"benchmark data for {exp.__class__.__name__}......")
self._count = 0
start = time()
ds.config.set_seed(_SEED)
for idx, next_element in enumerate(dataset):
now = time()
saliency_dict_lst = self._run_exp_step(
next_element, exp, imageid_labels, summary)
print(spacer.format(
"Finish writing {}-th batch explanation data for {}. Time elapsed: {:.3f} s"
.format(idx, exp.__class__.__name__,
time() - now)),
end='')
for bench in benchmarkers:
now = time()
self._run_exp_benchmark_step(
next_element, exp, bench, saliency_dict_lst)
print(spacer.format(
"Finish running {}-th batch {} data for {}. Time elapsed: {:.3f} s"
.format(idx, bench.__class__.__name__,
exp.__class__.__name__,
time() - now)),
end='')
for bench in benchmarkers:
benchmark = explain.benchmark.add()
benchmark.explain_method = exp.__class__.__name__
benchmark.benchmark_method = bench.__class__.__name__
benchmark.total_score = bench.performance
if isinstance(bench, LabelSensitiveMetric):
benchmark.label_score.extend(
bench.class_performances)
print(
spacer.format(
"Finish running and writing explanation and benchmark data for {}. "
"Time elapsed: {:.3f} s".format(
exp.__class__.__name__,
time() - start)))
summary.add_value('explainer', 'benchmark', explain)
summary.record(1)
print("Finish running and writing. Total time elapsed: {:.3f} s".
format(time() - begin))
def test_summary_step10_summaryrecord1(self):
"""Test record 10 step summary."""
with SummaryRecord(self.summary_dir) as test_writer:
train_summary_record(test_writer, steps=10)
def run(self,
dataset: Tuple,
explainers: List,
benchmarkers: Optional[List] = None):
"""
Genereate results and write results into the summary files in `self.summary_dir`.
Args:
dataset (tuple): A tuple that contains `mindspore.dataset` object for iteration and its labels.
- dataset[0], a `mindspore.dataset` object to provide data to explain.
- dataset[1], a list of string that specifies the label names of the dataset.
explainers (list): A list of explanation objects to generate _attribution results.
benchmarkers (list): A list of benchmark objects to generate evaluation results. Default: None
Examples:
>>> from mindspore.explainer.explanation import GuidedBackprop, Gradient
>>> # obtain dataset object
>>> dataset = get_dataset()
>>> classes = ["cat", "dog", ...]
>>> # load checkpoint to a network, e.g. resnet50
>>> param_dict = load_checkpoint("checkpoint.ckpt")
>>> net = resnet50(len(classes))
>>> load_parama_into_net(net, param_dict)
>>> # bind net with its output activation
>>> model = nn.SequentialCell([net, nn.Sigmoid()])
>>> gbp = GuidedBackprop(model)
>>> gradient = Gradient(model)
>>> runner = ExplainRunner("./")
>>> explainers = [gbp, gradient]
>>> runner.run((dataset, classes), explainers)
"""
if not isinstance(dataset, tuple):
raise TypeError("Argument `dataset` must be a tuple.")
if len(dataset) != 2:
raise ValueError("Argument `dataset` should be a tuple with length = 2.")
dataset, classes = dataset
self._verify_data_form(dataset, benchmarkers)
self._classes = classes
if explainers is None or not explainers:
raise ValueError("Argument `explainers` can neither be None nor empty.")
for exp in explainers:
if not isinstance(exp, Attribution) or not isinstance(explainers, list):
raise TypeError("Argument explainers should be a list of objects of classes in "
"`mindspore.explainer.explanation._attribution`.")
if benchmarkers is not None:
for bench in benchmarkers:
if not isinstance(bench, AttributionMetric) or not isinstance(explainers, list):
raise TypeError("Argument benchmarkers should be a list of objects of classes in explanation"
"`mindspore.explainer.benchmark._attribution`.")
self._model = explainers[0].model
with SummaryRecord(self._summary_dir) as summary:
print("Start running and writing......")
begin = time()
print("Start writing metadata.")
explain = Explain()
explain.metadata.label.extend(classes)
exp_names = [exp.__class__.__name__ for exp in explainers]
explain.metadata.explain_method.extend(exp_names)
if benchmarkers is not None:
bench_names = [bench.__class__.__name__ for bench in benchmarkers]
explain.metadata.benchmark_method.extend(bench_names)
summary.add_value("explainer", "metadata", explain)
summary.record(1)
print("Finish writing metadata.")
now = time()
print("Start running and writing inference data......")
imageid_labels = self._run_inference(dataset, summary)
print("Finish running and writing inference data. Time elapsed: {}s".format(time() - now))
if benchmarkers is None:
for exp in explainers:
start = time()
print("Start running and writing explanation data for {}......".format(exp.__class__.__name__))
self._count = 0
ds.config.set_seed(58)
for idx, next_element in enumerate(dataset):
now = time()
self._run_exp_step(next_element, exp, imageid_labels, summary)
print("Finish writing {}-th explanation data. Time elapsed: {}".format(
idx, time() - now))
print("Finish running and writing explanation data for {}. Time elapsed: {}".format(
exp.__class__.__name__, time() - start))
else:
for exp in explainers:
explain = Explain()
for bench in benchmarkers:
bench.reset()
print(f"Start running and writing explanation and benchmark data for {exp.__class__.__name__}.")
self._count = 0
start = time()
ds.config.set_seed(58)
for idx, next_element in enumerate(dataset):
now = time()
saliency_dict_lst = self._run_exp_step(next_element, exp, imageid_labels, summary)
print("Finish writing {}-th batch explanation data. Time elapsed: {}s".format(
idx, time() - now))
for bench in benchmarkers:
now = time()
self._run_exp_benchmark_step(next_element, exp, bench, saliency_dict_lst)
print("Finish running {}-th batch benchmark data for {}. Time elapsed: {}s".format(
idx, bench.__class__.__name__, time() - now))
for bench in benchmarkers:
benchmark = explain.benchmark.add()
benchmark.explain_method = exp.__class__.__name__
benchmark.benchmark_method = bench.__class__.__name__
benchmark.total_score = bench.performance
benchmark.label_score.extend(bench.class_performances)
print("Finish running and writing explanation and benchmark data for {}. "
"Time elapsed: {}s".format(exp.__class__.__name__, time() - start))
summary.add_value('explainer', 'benchmark', explain)
summary.record(1)
print("Finish running and writing. Total time elapsed: {}s".format(time() - begin))