def testMergeEstimators(self): estimator1 = size_estimator.SizeEstimator(size_threshold=10, size_fn=len) self.assertEqual(estimator1.get_estimate(), 0) estimator2 = size_estimator.SizeEstimator(size_threshold=10, size_fn=len) self.assertEqual(estimator2.get_estimate(), 0) a = b'plmjh' b, c = a, a expected_size_estimate = (len(a) / (sys.getrefcount(a) - 1)) * 4 estimator1.update(a) estimator1.update(b) estimator2.update(c) estimator2.update(a) estimator1 += estimator2 self.assertEqual(estimator1.get_estimate(), expected_size_estimate)
def __init__(self,
input_counts: List[int],
metric_counts: List[int],
size_estimator_fn: Callable[[Any], int],
desired_batch_size: Optional[int] = None):
"""Initializes accumulator using a list of metric counts per output.
Args:
input_counts: Number of inputs associated with each output index.
metric_counts: Number of metrics associated with each output index.
size_estimator_fn: Function to use for estimating the size of the inputs.
desired_batch_size: FOR TESTING ONLY.
"""
# Inputs have shape (num_outputs, num_metrics, num_accumulated_inputs)
self._inputs = []
# Weights have shape (num_outputs, num_metrics)
self._weights = [] # type: List[List[Optional[np.ndarray]]]
for input_count in input_counts:
self._inputs.append(tuple([] for _ in range(input_count)))
for output_metric_count in metric_counts:
self._weights.append([None] * output_metric_count)
self._size_estimator = size_estimator.SizeEstimator(
size_threshold=self._TOTAL_INPUT_BYTE_SIZE_THRESHOLD,
size_fn=size_estimator_fn)
if desired_batch_size and desired_batch_size > 0:
self._desired_batch_size = desired_batch_size
else:
self._desired_batch_size = self._DEFAULT_DESIRED_BATCH_SIZE
def testFlush(self): estimator = size_estimator.SizeEstimator(size_threshold=10, size_fn=len) self.assertEqual(estimator.get_estimate(), 0) estimator.update(b'plmjh') estimator.update(b'plmjhghytfghsggssss') self.assertTrue(estimator.should_flush()) estimator.clear() self.assertEqual(estimator.get_estimate(), 0)
def __init__(self, desired_batch_size: Optional[int] = None):
self.metric_variables = None # type: Optional[types.MetricVariablesType]
self.inputs = [] # type: List[bytes]
self.size_estimator = size_estimator.SizeEstimator(
size_threshold=self._TOTAL_INPUT_BYTE_SIZE_THRESHOLD, size_fn=len)
if desired_batch_size and desired_batch_size > 0:
self._desired_batch_size = desired_batch_size
else:
self._desired_batch_size = self._DEFAULT_DESIRED_BATCH_SIZE
def testRefCountAmortization(self): estimator = size_estimator.SizeEstimator(size_threshold=10, size_fn=len) self.assertEqual(estimator.get_estimate(), 0) a = b'plmjh' b, c = a, a expected_size_estimate = (len(a) / (sys.getrefcount(a) - 1)) * 4 estimator.update(a) estimator.update(b) estimator.update(c) estimator.update(a) self.assertEqual(estimator.get_estimate(), expected_size_estimate) self.assertFalse(estimator.should_flush())
def __init__(self,
metric_counts: List[int],
desired_batch_size: Optional[int] = None):
"""Initializes accumulator using a list of metric counts per output."""
# Inputs have shape (num_outputs, num_metrics, num_accumulated_inputs)
self._inputs = [
] # type: List[Tuple[List[np.ndarray], List[np.ndarray], List[np.ndarray]]]
# Weights have shape (num_outputs, num_metrics)
self._weights = [] # type: List[List[Optional[np.ndarray]]]
for output_metric_count in metric_counts:
self._inputs.append(([], [], []))
self._weights.append([None] * output_metric_count)
self._pad = False
self._last_dim = 0
self._size_estimator = size_estimator.SizeEstimator(
size_threshold=self._TOTAL_INPUT_BYTE_SIZE_THRESHOLD,
size_fn=_numpy_array_size_fn)
if desired_batch_size and desired_batch_size > 0:
self._desired_batch_size = desired_batch_size
else:
self._desired_batch_size = self._DEFAULT_DESIRED_BATCH_SIZE
