def __init__(self, tensor_message):
# Original dims can not be pickled to transfer to other process, so tuple is used.
self._dims = tuple(tensor_message.dims)
self._data_type = tensor_message.data_type
self._np_array = self.get_ndarray(tensor_message.float_data)
self._stats = TensorUtils.get_statistics_from_tensor(self._np_array)
original_buckets = calc_original_buckets(self._np_array, self._stats)
self._count = sum(bucket.count for bucket in original_buckets)
self._max = self._stats.max
self._min = self._stats.min
self._histogram = Histogram(tuple(original_buckets), self._max,
self._min, self._count)
def __init__(self, histogram_message: Summary.Histogram):
original_buckets = [
Bucket(bucket.left, bucket.width, bucket.count)
for bucket in histogram_message.buckets
]
# Ensure buckets are sorted from min to max.
original_buckets.sort(key=lambda bucket: bucket.left)
self._count = sum(bucket.count for bucket in original_buckets)
self._max = mask_invalid_number(histogram_message.max)
self._min = mask_invalid_number(histogram_message.min)
self._histogram = Histogram(tuple(original_buckets), self._max,
self._min, self._count)
def __init__(self, tensor_message):
self._lock = threading.Lock
# Original dims can not be pickled to transfer to other process, so tuple is used.
self._dims = tuple(tensor_message.dims)
self._data_type = tensor_message.data_type
self._np_array = None
self._data = _get_data_from_tensor(tensor_message)
self._stats = get_statistics_from_tensor(self.get_or_calc_ndarray())
original_buckets = calc_original_buckets(self.get_or_calc_ndarray(),
self._stats)
self._count = sum(bucket.count for bucket in original_buckets)
self._max = self._stats.max
self._min = self._stats.min
self._histogram = Histogram(tuple(original_buckets), self._max,
self._min, self._count)
class HistogramContainer:
"""
Histogram data container.
Args:
histogram_message (Summary.Histogram): Histogram message in summary file.
"""
def __init__(self, histogram_message: Summary.Histogram):
original_buckets = [
Bucket(bucket.left, bucket.width, bucket.count)
for bucket in histogram_message.buckets
]
# Ensure buckets are sorted from min to max.
original_buckets.sort(key=lambda bucket: bucket.left)
self._count = sum(bucket.count for bucket in original_buckets)
self._max = mask_invalid_number(histogram_message.max)
self._min = mask_invalid_number(histogram_message.min)
self._histogram = Histogram(tuple(original_buckets), self._max,
self._min, self._count)
@property
def max(self):
"""Gets max value of the tensor."""
return self._max
@property
def min(self):
"""Gets min value of the tensor."""
return self._min
@property
def count(self):
"""Gets valid number count of the tensor."""
return self._count
@property
def histogram(self):
"""Gets histogram data"""
return self._histogram
def buckets(self):
"""Gets histogram buckets"""
return self._histogram.buckets()
class TensorContainer:
"""
Tensor data container.
Args:
tensor_message (Summary.TensorProto): Tensor message in summary file.
"""
def __init__(self, tensor_message):
# Original dims can not be pickled to transfer to other process, so tuple is used.
self._dims = tuple(tensor_message.dims)
self._data_type = tensor_message.data_type
self._np_array = self.get_ndarray(tensor_message.float_data)
self._stats = TensorUtils.get_statistics_from_tensor(self._np_array)
original_buckets = calc_original_buckets(self._np_array, self._stats)
self._count = sum(bucket.count for bucket in original_buckets)
self._max = self._stats.max
self._min = self._stats.min
self._histogram = Histogram(tuple(original_buckets), self._max,
self._min, self._count)
@property
def size(self):
"""Get size of tensor."""
return self._np_array.size
@property
def dims(self):
"""Get dims of tensor."""
return self._dims
@property
def data_type(self):
"""Get data type of tensor."""
return self._data_type
@property
def ndarray(self):
"""Get ndarray of tensor."""
return self._np_array
@property
def max(self):
"""Get max value of tensor."""
return self._max
@property
def min(self):
"""Get min value of tensor."""
return self._min
@property
def stats(self):
"""Get statistics data of tensor."""
return self._stats
@property
def count(self):
"""Get count value of tensor."""
return self._count
@property
def histogram(self):
"""Get histogram data."""
return self._histogram
def buckets(self):
"""Get histogram buckets."""
return self._histogram.buckets()
def get_ndarray(self, tensor):
"""
Get ndarray of tensor.
Args:
tensor (mindinsight_anf_ir.proto.DataType): tensor data.
Returns:
numpy.ndarray, ndarray of tensor.
"""
return np.array(tuple(tensor)).reshape(self.dims)
class TensorContainer:
"""
Tensor data container.
Args:
tensor_message (Summary.TensorProto): Tensor message in summary file.
"""
def __init__(self, tensor_message):
self._lock = threading.Lock
# Original dims can not be pickled to transfer to other process, so tuple is used.
self._dims = tuple(tensor_message.dims)
self._data_type = tensor_message.data_type
self._np_array = None
self._data = _get_data_from_tensor(tensor_message)
self._stats = get_statistics_from_tensor(self.get_or_calc_ndarray())
original_buckets = calc_original_buckets(self.get_or_calc_ndarray(),
self._stats)
self._count = sum(bucket.count for bucket in original_buckets)
self._max = self._stats.max
self._min = self._stats.min
self._histogram = Histogram(tuple(original_buckets), self._max,
self._min, self._count)
@property
def dims(self):
"""Get dims of tensor."""
return self._dims
@property
def data_type(self):
"""Get data type of tensor."""
return self._data_type
@property
def max(self):
"""Get max value of tensor."""
return self._max
@property
def min(self):
"""Get min value of tensor."""
return self._min
@property
def stats(self):
"""Get statistics data of tensor."""
return self._stats
@property
def count(self):
"""Get count value of tensor."""
return self._count
@property
def histogram(self):
"""Get histogram data."""
return self._histogram
def buckets(self):
"""Get histogram buckets."""
return self._histogram.buckets()
def get_or_calc_ndarray(self):
"""Get or calculate ndarray."""
with self._lock():
if self._np_array is None:
self._convert_to_numpy_array()
return self._np_array
def _convert_to_numpy_array(self):
"""Convert a list data to numpy array."""
try:
ndarray = np.array(self._data).reshape(self._dims)
except ValueError as ex:
logger.error("Reshape array fail, detail: %r", str(ex))
return
self._np_array = ndarray