def test_get_tensor_stats_success(self):
"""Get tensor stats success."""
test_tag_name = self._complete_tag_name
processor = TensorProcessor(self._mock_data_manager)
results = processor.get_tensors([self._train_id], [test_tag_name],
None,
None,
detail='stats')
recv_metadata = results.get('tensors')[0].get("values")
for recv_values, expected_values in zip(recv_metadata, self._tensors):
assert recv_values.get('wall_time') == expected_values.get(
'wall_time')
assert recv_values.get('step') == expected_values.get('step')
expected_data = expected_values.get('value').get("float_data")
expected_statistic_instance = TensorUtils.get_statistics_from_tensor(
expected_data)
expected_statistic = TensorUtils.get_statistics_dict(
stats=expected_statistic_instance,
overall_stats=expected_statistic_instance)
recv_statistic = recv_values.get('value').get("statistics")
assert recv_statistic.get("max") - expected_statistic.get(
"max") < 1e-6
assert recv_statistic.get("min") - expected_statistic.get(
"min") < 1e-6
assert recv_statistic.get("avg") - expected_statistic.get(
"avg") < 1e-6
assert recv_statistic.get("count") - expected_statistic.get(
"count") < 1e-6
def get_tensors(self, train_ids, tags, step, dims, detail):
"""
Get tensor data for given train_ids, tags, step, dims and detail.
Args:
train_ids (list): Specify list of train job ID.
tags (list): Specify list of tag.
step (int): Specify step of tag, it's necessary when detail is equal to 'data'.
dims (str): Specify dims of step, it's necessary when detail is equal to 'data'.
detail (str): Specify which data to query, available values: 'stats', 'histogram' and 'data'.
Returns:
dict, a dict including the `tensors`.
Raises:
UrlDecodeError, If unquote train id error with strict mode.
"""
Validation.check_param_empty(train_id=train_ids, tag=tags)
TensorUtils.validate_dims_format(dims)
for index, train_id in enumerate(train_ids):
try:
train_id = unquote(train_id, errors='strict')
except UnicodeDecodeError:
raise UrlDecodeError('Unquote train id error with strict mode')
else:
train_ids[index] = train_id
tensors = []
for train_id in train_ids:
tensors += self._get_train_tensors(train_id, tags, step, dims,
detail)
return {"tensors": tensors}
def get_tensors_diff(self, tensor_name, shape, tolerance=0):
"""
Get tensor comparisons data for given name, detail, shape and tolerance.
Args:
tensor_name (str): The name of tensor for cache.
shape (tuple): Specify concrete dimensions of shape.
tolerance (str): Specify tolerance of difference between current step tensor and previous
step tensor. Default value is 0. Its is a percentage. The boundary value is equal to
max(abs(min),abs(max)) * tolerance. The function of min and max is being used to
calculate the min value and max value of the result of the current step tensor subtract
the previous step tensor. If the absolute value of result is less than or equal to
boundary value, the result will set to be zero.
Raises:
DebuggerParamValueError, If get current step node and previous step node failed or
the type of tensor value is not numpy.ndarray."
Returns:
dict, the retrieved data.
"""
curr_tensor = self.get_valid_tensor_by_name(tensor_name)
prev_tensor = self.get_valid_tensor_by_name(tensor_name, prev=True)
if not (curr_tensor and prev_tensor):
log.error("Get current step and previous step for this tensor name %s failed.", tensor_name)
raise DebuggerParamValueError(f"Get current step and previous step for this tensor name "
f"{tensor_name} failed.")
curr_tensor_slice = curr_tensor.get_tensor_value_by_shape(shape)
prev_tensor_slice = prev_tensor.get_tensor_value_by_shape(shape)
# get tensor comparison basic info
tensor_info = curr_tensor.get_basic_info()
tensor_info.pop('has_prev_step')
tensor_info.pop('value')
# calculate tensor comparision object
tensor_comparison = curr_tensor.tensor_comparison
if not tensor_comparison or tensor_comparison.tolerance != tolerance:
if curr_tensor.value.shape != prev_tensor.value.shape:
raise DebuggerParamValueError("The shape of these two step tensors is not the same.")
tensor_diff = TensorUtils.calc_diff_between_two_tensor(curr_tensor.value, prev_tensor.value, tolerance)
stats = TensorUtils.get_statistics_from_tensor(tensor_diff)
tensor_comparison = TensorComparison(tolerance, stats, tensor_diff)
curr_tensor.update_tensor_comparisons(tensor_comparison)
# calculate diff value
# the type of curr_tensor_slice is one of np.ndarray or str
if isinstance(curr_tensor_slice, np.ndarray) and isinstance(prev_tensor_slice, np.ndarray):
if not shape:
tensor_diff_slice = tensor_comparison.value
else:
tensor_diff_slice = tensor_comparison.value[shape]
result = np.stack([prev_tensor_slice, curr_tensor_slice, tensor_diff_slice], axis=-1)
tensor_info['diff'] = result.tolist()
elif isinstance(curr_tensor_slice, str):
tensor_info['diff'] = curr_tensor_slice
# add comparision statistics
tensor_info.update(self._get_comparison_statistics(curr_tensor, prev_tensor))
reply = {'tensor_value': tensor_info}
return reply
def _get_comparison_statistics(curr_tensor, prev_tensor):
"""Get comparison statistics."""
stats_info = {}
diff_tensor_stats = curr_tensor.tensor_comparison.stats
curr_tensor_stats = TensorUtils.get_statistics_from_tensor(curr_tensor.value)
prev_tensor_stats = TensorUtils.get_statistics_from_tensor(prev_tensor.value)
stats_info['curr_step_statistics'] = TensorUtils.get_overall_statistic_dict(overall_stats=curr_tensor_stats)
stats_info['prev_step_statistics'] = TensorUtils.get_overall_statistic_dict(overall_stats=prev_tensor_stats)
stats_info['statistics'] = TensorUtils.get_overall_statistic_dict(overall_stats=diff_tensor_stats)
return stats_info
def get_tensor_statistics(self):
"""
Get Tensor statistics.
Returns:
dict, overall statistics.
"""
if self.empty or self.dtype == self._STRING_TYPE:
return {}
stats = TensorUtils.get_statistics_from_tensor(self.value)
statistics = TensorUtils.get_overall_statistic_dict(stats)
return statistics
def get_tensor_statistics(self):
"""
Get Tensor statistics.
Returns:
dict, overall statistics.
"""
if self.empty:
return {}
if not self.stats:
self.stats = TensorUtils.get_statistics_from_tensor(self.value)
statistics = TensorUtils.get_overall_statistic_dict(self.stats)
return statistics
def get_tensors_diff(self, tensor_name, shape, tolerance=0):
"""
Get tensor comparisons data for given name, detail, shape and tolerance.
Args:
tensor_name (str): The name of tensor for cache.
shape (tuple): Specify concrete dimensions of shape.
tolerance (str): Specify tolerance of difference between current step tensor and previous
step tensor. Default value is 0. Its is a percentage. The boundary value is equal to
max(abs(min),abs(max)) * tolerance. The function of min and max is being used to
calculate the min value and max value of the result of the current step tensor subtract
the previous step tensor. If the absolute value of result is less than or equal to
boundary value, the result will set to be zero.
Raises:
DebuggerParamValueError, If get current step node and previous step node failed.
Returns:
dict, the retrieved data.
"""
curr_tensor = self.get_tensor_value_by_name(tensor_name)
prev_tensor = self.get_tensor_value_by_name(tensor_name, prev=True)
if not (curr_tensor and prev_tensor):
log.error(
"Get current step and previous step for this tensor name %s failed.",
tensor_name)
raise DebuggerParamValueError(
f"Get current step and previous step for this tensor name "
f"{tensor_name} failed.")
curr_tensor_slice = curr_tensor.get_tensor_value_by_shape(shape)
prev_tensor_slice = prev_tensor.get_tensor_value_by_shape(shape)
tensor_info = curr_tensor.get_basic_info()
if isinstance(tensor_info, dict):
del tensor_info['has_prev_step']
del tensor_info['value']
# the type of curr_tensor_slice is one of None, np.ndarray or str
if isinstance(curr_tensor_slice, np.ndarray) and isinstance(
prev_tensor_slice, np.ndarray):
diff_tensor = TensorUtils.calc_diff_between_two_tensor(
curr_tensor_slice, prev_tensor_slice, tolerance)
result = np.stack(
[prev_tensor_slice, curr_tensor_slice, diff_tensor], axis=-1)
tensor_info['diff'] = result.tolist()
stats = TensorUtils.get_statistics_from_tensor(diff_tensor)
tensor_info['statistics'] = TensorUtils.get_statistics_dict(stats)
elif isinstance(curr_tensor_slice, str):
tensor_info['diff'] = curr_tensor_slice
reply = {'tensor_value': tensor_info}
return reply
def test_get_tensor_histogram_success(self):
"""Get tensor histogram success."""
test_tag_name = self._complete_tag_name
processor = TensorProcessor(self._mock_data_manager)
results = processor.get_tensors([self._train_id], [test_tag_name],
None,
None,
detail='histogram')
recv_metadata = results.get('tensors')[0].get("values")
for recv_values, expected_values in zip(recv_metadata, self._tensors):
assert recv_values.get('wall_time') == expected_values.get(
'wall_time')
assert recv_values.get('step') == expected_values.get('step')
expected_data = expected_values.get('value').get("float_data")
expected_statistic = TensorUtils.get_statistics_from_tensor(
expected_data)
expected_buckets = calc_original_buckets(expected_data,
expected_statistic)
recv_buckets = recv_values.get('value').get("histogram_buckets")
for recv_bucket, expected_bucket in zip(recv_buckets,
expected_buckets):
assert recv_bucket[0] - expected_bucket.left < 1e-6
assert recv_bucket[1] - expected_bucket.width < 1e-6
assert recv_bucket[2] - expected_bucket.count <= 1
def retrieve_tensor_value(self,
name,
detail,
shape,
graph_name=None,
prev=False):
"""Retrieve the tensor value."""
log.info("Retrieve tensor value: name: %s, detail: %s, shape: %s",
name, detail, shape)
self.validate_tensor_param(name, detail)
# Limit to query max two dimensions for tensor in table view.
parsed_shape = TensorUtils.parse_shape(shape,
limit=MAX_DIMENSIONS_FOR_TENSOR)
node_type, tensor_name = self._get_tensor_name_and_type_by_ui_name(
name, graph_name)
reply = self.cache_store.get_stream_handler(Streams.TENSOR).get({
'name':
tensor_name,
'node_type':
node_type,
'shape':
parsed_shape,
'prev':
prev
})
reply['tensor_value']['name'] = name
return reply
def test_get_tensor_data_success(self):
"""Get tensor data success."""
test_tag_name = self._complete_tag_name
processor = TensorProcessor(self._mock_data_manager)
results = processor.get_tensors([self._train_id], [test_tag_name],
step='1',
dims='[0,0,:-1,:]',
detail='data')
recv_metadata = results.get('tensors')[0].get("values")
for recv_values, expected_values in zip(recv_metadata, self._tensors):
assert recv_values.get('wall_time') == expected_values.get(
'wall_time')
assert recv_values.get('step') == expected_values.get('step')
dims = expected_values.get('value').get("dims")
expected_data = np.array(
expected_values.get('value').get("float_data")).reshape(dims)
recv_tensor = np.array(recv_values.get('value').get("data"))
expected_tensor = TensorUtils.get_specific_dims_data(
expected_data, (0, 0, slice(None, -1, None), slice(None)))
# Compare tensor shape when recv_tensor shape is not empty.
if recv_tensor.shape != (0, ):
assert recv_tensor.shape == expected_tensor.shape
assert np.allclose(recv_tensor, expected_tensor, rtol=1e-6)
def test_get_statistics_from_tensor(self):
"""Tests get statistics from tensor."""
ndarray = np.array(
[1, 2, 3, 4, 5,
float('-INF'),
float('INF'),
float('NAN')]).reshape([2, 2, 2])
statistics = TensorUtils.get_statistics_from_tensor(ndarray)
assert (statistics.max, statistics.min, statistics.avg, statistics.count,
statistics.nan_count, statistics.neg_inf_count, statistics.pos_inf_count) == \
(5, 1, 3, 8,
1, 1, 1)
def __init__(self, tensor_message):
# Original dims can not be pickled to transfer to other process, so tuple is used.
self._dims = tuple(tensor_message.dims)
self._data_type = tensor_message.data_type
self._np_array = self.get_ndarray(tensor_message.float_data)
self._stats = TensorUtils.get_statistics_from_tensor(self._np_array)
original_buckets = calc_original_buckets(self._np_array, self._stats)
self._count = sum(bucket.count for bucket in original_buckets)
self._max = self._stats.max
self._min = self._stats.min
self._histogram = Histogram(tuple(original_buckets), self._max,
self._min, self._count)
def get_tensor_serializable_value_by_shape(self, shape=None):
"""
Get tensor value info by shape.
Args:
shape (tuple): The specified range of tensor value.
Returns:
dict, the specified tensor value and value statistics.
"""
tensor_value = self.get_tensor_value_by_shape(shape)
res = {}
# the type of tensor_value is one of None, np.ndarray or str
if isinstance(tensor_value, np.ndarray):
statistics = TensorUtils.get_statistics_from_tensor(tensor_value)
res['statistics'] = TensorUtils.get_statistics_dict(statistics)
res['value'] = tensor_value.tolist()
elif isinstance(tensor_value, str):
res['value'] = tensor_value
return res
def _get_tensors_histogram(self, tensors):
"""
Builds a JSON-serializable object with information about tensor histogram data.
Args:
tensors (list): The list of _Tensor data.
Returns:
dict, a dict including the `wall_time`, `step`, and `value' for each tensor.
{
"wall_time": 0,
"step": 0,
"value": {
"dims": [1],
"data_type": "DT_FLOAT32",
"histogram_buckets": [[0.1, 0.2, 3]]
"statistics": {
"max": 0,
"min": 0,
"avg": 0,
"count": 1,
"nan_count": 0,
"neg_inf_count": 0,
"pos_inf_count": 0
}
}
}
"""
values = []
for tensor in tensors:
# This value is an instance of TensorContainer
value = tensor.value
if value.error_code is not None:
raise TensorTooLargeError("Step: {}".format(tensor.step))
buckets = value.buckets()
values.append({
"wall_time": tensor.wall_time,
"step": tensor.step,
"value": {
"dims":
value.dims,
"data_type":
anf_ir_pb2.DataType.Name(value.data_type),
"histogram_buckets":
buckets,
"statistics":
TensorUtils.get_statistics_dict(stats=value.stats,
overall_stats=value.stats)
}
})
return values
def tensor_comparisons(self,
name,
shape,
detail='data',
tolerance='0',
rank_id=0,
graph_name=None):
"""
Get tensor comparisons data for given name, detail, shape and tolerance.
Args:
name (str): The name of tensor for ui.
shape (str): Specify concrete dimensions of shape.
detail (str): Specify which data to query. Current available value is 'data' which means
concrete tensor data. Histogram or unique count can be supported in the future.
rank_id (int): The id of rank. Default: 0.
tolerance (str): Specify tolerance of difference between current step tensor and previous
step tensor. Default value is 0.
graph_name (str): The graph name. Default: None.
Returns:
dict, the retrieved data.
"""
if self.cache_store.get_stream_handler(
Streams.METADATA).state != ServerStatus.WAITING.value:
log.error(
"Failed to compare tensors as the MindSpore is not in waiting state."
)
raise DebuggerCompareTensorError(
"Failed to compare tensors as the MindSpore is not in waiting state."
)
self.validate_tensor_param(name, detail)
# Limit to query max two dimensions for tensor in table view.
parsed_shape = TensorUtils.parse_shape(shape,
limit=MAX_DIMENSIONS_FOR_TENSOR)
node_type, tensor_name, graph_name = self._get_tensor_name_and_type_by_ui_name(
name, graph_name, rank_id)
tolerance = to_float(tolerance, 'tolerance')
tensor_stream = self.cache_store.get_stream_handler(
Streams.TENSOR).get_tensor_handler_by_rank_id(rank_id)
cur_step = self.cache_store.get_stream_handler(Streams.METADATA).step
if node_type == NodeTypeEnum.PARAMETER.value:
reply = tensor_stream.get_tensors_diff(tensor_name, parsed_shape,
tolerance, cur_step)
else:
raise DebuggerParamValueError(
"The node type must be parameter, but got {}.".format(
node_type))
if reply.pop('view_cmd', False):
self._send_view_cmd(name, graph_name, rank_id, tensor_name,
node_type)
return reply
def test_calc_original_buckets(self):
"""Tests calculate original buckets."""
ndarray = np.array(
[1, 2, 3, 4, 5,
float('-INF'),
float('INF'),
float('NAN')]).reshape([2, 2, 2])
statistics = TensorUtils.get_statistics_from_tensor(ndarray)
buckets = tensor.calc_original_buckets(ndarray, statistics)
assert (buckets[0].left, buckets[0].width, buckets[0].count) == (1, 2,
2)
assert (buckets[1].left, buckets[1].width, buckets[1].count) == (3, 2,
3)
def _get_train_tensors(self, train_id, tags, step, dims, detail):
"""
Get tensor data for given train_id, tags, step, dims and detail.
Args:
train_id (str): Specify list of train job ID.
tags (list): Specify list of tag.
step (int): Specify step of tensor, it's necessary when detail is set to 'data'.
dims (str): Specify dims of tensor, it's necessary when detail is set to 'data'.
detail (str): Specify which data to query, available values: 'stats', 'histogram' and 'data'.
Returns:
list[dict], a list of dictionaries containing the `train_id`, `tag`, `values`.
Raises:
TensorNotExistError, If tensor with specific train_id and tag is not exist in cache.
ParamValueError, If the value of detail is not within available values:
'stats', 'histogram' and 'data'.
"""
tensors_response = []
for tag in tags:
try:
tensors = self._data_manager.list_tensors(train_id, tag)
except ParamValueError as err:
raise TensorNotExistError(err.message)
if tensors and not isinstance(tensors[0].value, TensorContainer):
raise TensorNotExistError(
"there is no tensor data in this tag: {}".format(tag))
if detail is None or detail == 'stats':
values = self._get_tensors_summary(detail, tensors)
elif detail == 'data':
Validation.check_param_empty(step=step, dims=dims)
# Limit to query max two dimensions for tensor in table view.
dims = TensorUtils.parse_shape(dims,
limit=MAX_DIMENSIONS_FOR_TENSOR)
step = to_int(step, "step")
values = self._get_tensors_data(step, dims, tensors)
elif detail == 'histogram':
values = self._get_tensors_histogram(tensors)
else:
raise ParamValueError(
'Can not support this value: {} of detail.'.format(detail))
tensor = {"train_id": train_id, "tag": tag, "values": values}
tensors_response.append(tensor)
return tensors_response
def _get_tensors_summary(self, detail, tensors):
"""
Builds a JSON-serializable object with information about tensor summary.
Args:
detail (str): Specify which data to query, detail value is None or 'stats' at this method.
tensors (list): The list of _Tensor data.
Returns:
dict, a dict including the `wall_time`, `step`, and `value' for each tensor.
{
"wall_time": 0,
"step": 0,
"value": {
"dims": [1],
"data_type": "DT_FLOAT32"
"statistics": {
"max": 0,
"min": 0,
"avg": 0,
"count": 1,
"nan_count": 0,
"neg_inf_count": 0,
"pos_inf_count": 0
} This dict is being set when detail is equal to stats.
}
}
"""
values = []
for tensor in tensors:
# This value is an instance of TensorContainer
value = tensor.value
value_dict = {
"dims": value.dims,
"data_type": anf_ir_pb2.DataType.Name(value.data_type)
}
if detail and detail == 'stats':
stats = None
if value.error_code is None:
stats = TensorUtils.get_statistics_dict(
stats=value.stats, overall_stats=value.stats)
value_dict.update({"statistics": stats})
values.append({
"wall_time": tensor.wall_time,
"step": tensor.step,
"value": value_dict
})
return values
def test_get_tensor_data_success(self):
"""Get tensor data success."""
test_tag_name = self._complete_tag_name
processor = TensorProcessor(self._mock_data_manager)
results = processor.get_tensors([self._train_id], [test_tag_name], step='1', dims='[0,0,:,:]', detail='data')
recv_metadata = results.get('tensors')[0].get("values")
for recv_values, expected_values in zip(recv_metadata, self._tensors):
assert recv_values.get('wall_time') == expected_values.get('wall_time')
assert recv_values.get('step') == expected_values.get('step')
dims = expected_values.get('value').get("dims")
expected_data = np.array(expected_values.get('value').get("float_data")).reshape(dims)
recv_tensor = np.array(recv_values.get('value').get("data"))
expected_tensor = TensorUtils.get_specific_dims_data(expected_data, [0, 0, None, None], dims)
assert np.sum(np.isclose(recv_tensor, expected_tensor, rtol=1e-6) == 0) == 0
def _get_tensors_data(self, step, dims, tensors):
"""
Builds a JSON-serializable object with information about tensor dims data.
Args:
step (int): Specify step of tensor.
dims (str): Specify dims of tensor.
tensors (list): The list of _Tensor data.
Returns:
dict, a dict including the `wall_time`, `step`, and `value' for each tensor.
{
"wall_time": 0,
"step": 0,
"value": {
"dims": [1],
"data_type": "DT_FLOAT32",
"data": [[0.1]]
"statistics": {
"max": 0,
"min": 0,
"avg": 0,
"count": 1,
"nan_count": 0,
"neg_inf_count": 0,
"pos_inf_count": 0
}
}
}
Raises:
ResponseDataExceedMaxValueError, If the size of response data exceed max value.
StepTensorDataNotInCacheError, If query step is not in cache.
"""
values = []
step_in_cache = False
dims = TensorUtils.convert_array_from_str_dims(dims, limit=2)
for tensor in tensors:
# This value is an instance of TensorContainer
value = tensor.value
if step != tensor.step:
continue
step_in_cache = True
res_data = TensorUtils.get_specific_dims_data(
value.ndarray, dims, list(value.dims))
flatten_data = res_data.flatten().tolist()
if len(flatten_data) > MAX_TENSOR_RESPONSE_DATA_SIZE:
raise ResponseDataExceedMaxValueError(
"the size of response data: {} exceed max value: {}.".
format(len(flatten_data), MAX_TENSOR_RESPONSE_DATA_SIZE))
def transfer(array):
if not isinstance(array, np.ndarray):
# The list is used here so that len function can be used
# when the value of array is `NAN`、｀-INF｀ or ｀INF｀.
array = [array]
transfer_data = [None] * len(array)
for index, data in enumerate(array):
if isinstance(data, np.ndarray):
transfer_data[index] = transfer(data)
else:
if np.isnan(data):
transfer_data[index] = 'NAN'
elif np.isneginf(data):
transfer_data[index] = '-INF'
elif np.isposinf(data):
transfer_data[index] = 'INF'
else:
transfer_data[index] = float(data)
return transfer_data
stats = TensorUtils.get_statistics_from_tensor(res_data)
if stats.nan_count + stats.neg_inf_count + stats.pos_inf_count > 0:
tensor_data = transfer(res_data)
else:
tensor_data = res_data.tolist()
values.append({
"wall_time": tensor.wall_time,
"step": tensor.step,
"value": {
"dims": value.dims,
"data_type": anf_ir_pb2.DataType.Name(value.data_type),
"data": tensor_data,
"statistics": TensorUtils.get_statistics_dict(stats)
}
})
break
if not step_in_cache:
raise StepTensorDataNotInCacheError(
"this step: {} data may has been dropped.".format(step))
return values
