def test_get_tensor_stats_success(self): """Get tensor stats success.""" test_tag_name = self._complete_tag_name processor = TensorProcessor(self._mock_data_manager) results = processor.get_tensors([self._train_id], [test_tag_name], None, None, detail='stats') recv_metadata = results.get('tensors')[0].get("values") for recv_values, expected_values in zip(recv_metadata, self._tensors): assert recv_values.get('wall_time') == expected_values.get( 'wall_time') assert recv_values.get('step') == expected_values.get('step') expected_data = expected_values.get('value').get("float_data") expected_statistic_instance = TensorUtils.get_statistics_from_tensor( expected_data) expected_statistic = TensorUtils.get_statistics_dict( stats=expected_statistic_instance, overall_stats=expected_statistic_instance) recv_statistic = recv_values.get('value').get("statistics") assert recv_statistic.get("max") - expected_statistic.get( "max") < 1e-6 assert recv_statistic.get("min") - expected_statistic.get( "min") < 1e-6 assert recv_statistic.get("avg") - expected_statistic.get( "avg") < 1e-6 assert recv_statistic.get("count") - expected_statistic.get( "count") < 1e-6
def _get_tensors_histogram(self, tensors): """ Builds a JSON-serializable object with information about tensor histogram data. Args: tensors (list): The list of _Tensor data. Returns: dict, a dict including the `wall_time`, `step`, and `value' for each tensor. { "wall_time": 0, "step": 0, "value": { "dims": [1], "data_type": "DT_FLOAT32", "histogram_buckets": [[0.1, 0.2, 3]] "statistics": { "max": 0, "min": 0, "avg": 0, "count": 1, "nan_count": 0, "neg_inf_count": 0, "pos_inf_count": 0 } } } """ values = [] for tensor in tensors: # This value is an instance of TensorContainer value = tensor.value if value.error_code is not None: raise TensorTooLargeError("Step: {}".format(tensor.step)) buckets = value.buckets() values.append({ "wall_time": tensor.wall_time, "step": tensor.step, "value": { "dims": value.dims, "data_type": anf_ir_pb2.DataType.Name(value.data_type), "histogram_buckets": buckets, "statistics": TensorUtils.get_statistics_dict(stats=value.stats, overall_stats=value.stats) } }) return values
def _get_tensors_summary(self, detail, tensors): """ Builds a JSON-serializable object with information about tensor summary. Args: detail (str): Specify which data to query, detail value is None or 'stats' at this method. tensors (list): The list of _Tensor data. Returns: dict, a dict including the `wall_time`, `step`, and `value' for each tensor. { "wall_time": 0, "step": 0, "value": { "dims": [1], "data_type": "DT_FLOAT32" "statistics": { "max": 0, "min": 0, "avg": 0, "count": 1, "nan_count": 0, "neg_inf_count": 0, "pos_inf_count": 0 } This dict is being set when detail is equal to stats. } } """ values = [] for tensor in tensors: # This value is an instance of TensorContainer value = tensor.value value_dict = { "dims": value.dims, "data_type": anf_ir_pb2.DataType.Name(value.data_type) } if detail and detail == 'stats': stats = None if value.error_code is None: stats = TensorUtils.get_statistics_dict( stats=value.stats, overall_stats=value.stats) value_dict.update({"statistics": stats}) values.append({ "wall_time": tensor.wall_time, "step": tensor.step, "value": value_dict }) return values
def get_tensors_diff(self, tensor_name, shape, tolerance=0): """ Get tensor comparisons data for given name, detail, shape and tolerance. Args: tensor_name (str): The name of tensor for cache. shape (tuple): Specify concrete dimensions of shape. tolerance (str): Specify tolerance of difference between current step tensor and previous step tensor. Default value is 0. Its is a percentage. The boundary value is equal to max(abs(min),abs(max)) * tolerance. The function of min and max is being used to calculate the min value and max value of the result of the current step tensor subtract the previous step tensor. If the absolute value of result is less than or equal to boundary value, the result will set to be zero. Raises: DebuggerParamValueError, If get current step node and previous step node failed. Returns: dict, the retrieved data. """ curr_tensor = self.get_tensor_value_by_name(tensor_name) prev_tensor = self.get_tensor_value_by_name(tensor_name, prev=True) if not (curr_tensor and prev_tensor): log.error( "Get current step and previous step for this tensor name %s failed.", tensor_name) raise DebuggerParamValueError( f"Get current step and previous step for this tensor name " f"{tensor_name} failed.") curr_tensor_slice = curr_tensor.get_tensor_value_by_shape(shape) prev_tensor_slice = prev_tensor.get_tensor_value_by_shape(shape) tensor_info = curr_tensor.get_basic_info() if isinstance(tensor_info, dict): del tensor_info['has_prev_step'] del tensor_info['value'] # the type of curr_tensor_slice is one of None, np.ndarray or str if isinstance(curr_tensor_slice, np.ndarray) and isinstance( prev_tensor_slice, np.ndarray): diff_tensor = TensorUtils.calc_diff_between_two_tensor( curr_tensor_slice, prev_tensor_slice, tolerance) result = np.stack( [prev_tensor_slice, curr_tensor_slice, diff_tensor], axis=-1) tensor_info['diff'] = result.tolist() stats = TensorUtils.get_statistics_from_tensor(diff_tensor) tensor_info['statistics'] = TensorUtils.get_statistics_dict(stats) elif isinstance(curr_tensor_slice, str): tensor_info['diff'] = curr_tensor_slice reply = {'tensor_value': tensor_info} return reply
def get_tensor_serializable_value_by_shape(self, shape=None): """ Get tensor value info by shape. Args: shape (tuple): The specified range of tensor value. Returns: dict, the specified tensor value and value statistics. """ tensor_value = self.get_tensor_value_by_shape(shape) res = {} # the type of tensor_value is one of None, np.ndarray or str if isinstance(tensor_value, np.ndarray): statistics = TensorUtils.get_statistics_from_tensor(tensor_value) res['statistics'] = TensorUtils.get_statistics_dict(statistics) res['value'] = tensor_value.tolist() elif isinstance(tensor_value, str): res['value'] = tensor_value return res
def _get_tensors_data(self, step, dims, tensors): """ Builds a JSON-serializable object with information about tensor dims data. Args: step (int): Specify step of tensor. dims (str): Specify dims of tensor. tensors (list): The list of _Tensor data. Returns: dict, a dict including the `wall_time`, `step`, and `value' for each tensor. { "wall_time": 0, "step": 0, "value": { "dims": [1], "data_type": "DT_FLOAT32", "data": [[0.1]] "statistics": { "max": 0, "min": 0, "avg": 0, "count": 1, "nan_count": 0, "neg_inf_count": 0, "pos_inf_count": 0 } } } Raises: ResponseDataExceedMaxValueError, If the size of response data exceed max value. StepTensorDataNotInCacheError, If query step is not in cache. """ values = [] step_in_cache = False dims = TensorUtils.convert_array_from_str_dims(dims, limit=2) for tensor in tensors: # This value is an instance of TensorContainer value = tensor.value if step != tensor.step: continue step_in_cache = True res_data = TensorUtils.get_specific_dims_data( value.ndarray, dims, list(value.dims)) flatten_data = res_data.flatten().tolist() if len(flatten_data) > MAX_TENSOR_RESPONSE_DATA_SIZE: raise ResponseDataExceedMaxValueError( "the size of response data: {} exceed max value: {}.". format(len(flatten_data), MAX_TENSOR_RESPONSE_DATA_SIZE)) def transfer(array): if not isinstance(array, np.ndarray): # The list is used here so that len function can be used # when the value of array is `NAN`、`-INF` or `INF`. array = [array] transfer_data = [None] * len(array) for index, data in enumerate(array): if isinstance(data, np.ndarray): transfer_data[index] = transfer(data) else: if np.isnan(data): transfer_data[index] = 'NAN' elif np.isneginf(data): transfer_data[index] = '-INF' elif np.isposinf(data): transfer_data[index] = 'INF' else: transfer_data[index] = float(data) return transfer_data stats = TensorUtils.get_statistics_from_tensor(res_data) if stats.nan_count + stats.neg_inf_count + stats.pos_inf_count > 0: tensor_data = transfer(res_data) else: tensor_data = res_data.tolist() values.append({ "wall_time": tensor.wall_time, "step": tensor.step, "value": { "dims": value.dims, "data_type": anf_ir_pb2.DataType.Name(value.data_type), "data": tensor_data, "statistics": TensorUtils.get_statistics_dict(stats) } }) break if not step_in_cache: raise StepTensorDataNotInCacheError( "this step: {} data may has been dropped.".format(step)) return values