class STDLayerMeasure(EventDrivenLayerMeasure):
def __init__(self, layer_index: int, layer_name: str):
super(STDLayerMeasure).__init__(layer_index, layer_name)
self.running_mean = RunningMeanWelford()
def update_layer(self, activations):
layer_measure = activations.std(axis=0)
self.running_mean.update(layer_measure)
def get_final_result(self):
return self.running_mean.mean()
def eval(self,q:Queue,inner_q:Queue):
running_mean = RunningMeanWelford()
# activation_sum=0
n=0
for transformation in self.queue_as_generator(q):
for activations in self.queue_as_generator(inner_q):
if self.sign != 1:
activations *= self.sign
activated = (activations > self.threshold) * 1
running_mean.update_all(activated)
self.l=running_mean.mean()
def eval(self, q: Queue, inner_q: Queue):
m = RunningMeanWelford()
for iteration_info in self.queue_as_generator(q):
inner_m = RunningMeanAndVarianceWelford()
for activations in self.queue_as_generator(inner_q):
activations = self.conv_aggregation.apply(activations)
inner_m.update_all(activations)
# i += 1
inner_result = self.measure_function.apply_running(inner_m)
m.update(inner_result)
return m.mean()
def eval(self,
activations_iterator: ActivationsIterator,
verbose=False) -> MeasureResult:
running_means = [
RunningMeanWelford() for i in activations_iterator.layer_names()
]
for x, transformation_activations in tqdm(
activations_iterator.samples_first(), disable=not verbose):
for x_transformed, activations in transformation_activations:
for i, layer_activations in enumerate(activations):
if self.sign != 1:
layer_activations *= self.sign
activated: np.ndarray = (layer_activations >
self.thresholds[i]) * 1.0
# print(activated.shape,activated.min(),activated.max(),activated.dtype)
if np.any(activated < 0):
print(activated)
running_means[i].update_all(activated)
layers_l = [m.mean() for m in running_means]
return MeasureResult(layers_l, activations_iterator.layer_names(),
self)
def eval_means_per_transformation(
self, activations_iterator: ActivationsIterator
) -> ([ActivationsByLayer], [int]):
'''
For all activations, calculates the mean activation value for each transformation
:param activations_iterator:
:return: A list of mean activation values for each activation in each layer
The list of samples per transformation
'''
n_layers = len(activations_iterator.layer_names())
means_per_transformation = []
samples_per_transformation = []
for transformation, transformation_activations in activations_iterator.transformations_first(
):
samples_variances_running = [
RunningMeanWelford() for i in range(n_layers)
]
# calculate the variance of all samples for this transformation
n_samples = 0
for x, batch_activations in transformation_activations:
n_samples += x.shape[0]
for j, layer_activations in enumerate(batch_activations):
for i in range(layer_activations.shape[0]):
samples_variances_running[j].update(
layer_activations[i, ])
samples_per_transformation.append(n_samples)
means_per_transformation.append(
[rm.mean() for rm in samples_variances_running])
return means_per_transformation, samples_per_transformation
def distance(self,batch:np.ndarray,batch_inverted:np.ndarray,mean_running:RunningMeanWelford):
n_shape=len(batch.shape)
assert n_shape>=2
n,f=batch.shape[0],batch.shape[1]
if n_shape>2 and self.normalize:
# normalize all extra dimensions
for i in range(n):
for j in range(f):
batch[i,j,:]/=np.linalg.norm(batch[i,j,:])
batch_inverted[i,j,:]/=np.linalg.norm(batch_inverted[i,j,:])
# ssd of all values
distances = (batch-batch_inverted)**2
n_shape=len(batch.shape)
if n_shape>2:
# aggregate extra dims to keep only the feature dim
distances= distances.mean(axis=tuple(range(2,n_shape)))
distances = np.sqrt(distances)
assert len(distances.shape)==2
mean_running.update_all(distances)
def eval(self,activations_iterator:ActivationsIterator,verbose=False)->MeasureResult:
activations_iterator = activations_iterator.get_both_iterator()
mean_running=None
for x, transformation_activations_iterator in activations_iterator.samples_first():
# transformation_activations_iterator can iterate over all transforms
for x_transformed, activations,inverted_activations in transformation_activations_iterator:
if mean_running is None:
# do this after the first iteration since we dont know the number
# of layers until the first iteration of the activations_iterator
mean_running = [RunningMeanWelford() for i in range(len(activations))]
for j, (layer_activations,inverted_layer_activations) in enumerate(zip(activations,inverted_activations)):
self.distance_function.distance(layer_activations,inverted_layer_activations,mean_running[j])
# calculate the final mean over all samples (and layers)
means = [b.mean() for b in mean_running]
return MeasureResult(means,activations_iterator.layer_names(),self)
def eval_global_means(self, means_per_layer_and_transformation: [
ActivationsByLayer
], n_layers: int) -> ActivationsByLayer:
'''
:param means_per_layer_and_transformation:
:param n_layers:
:return: The global means for each layer, averaging out the transformations
'''
# n_transformations = len(means_per_layer_and_transformation)
global_means_running = [RunningMeanWelford() for i in range(n_layers)]
for transformation_means in means_per_layer_and_transformation:
# means_per_layer has the means for a given transformation
for i, layer_means in enumerate(transformation_means):
global_means_running[i].update(layer_means)
return [rm.mean() for rm in global_means_running]
def eval(self,
activations_iterator: ActivationsIterator,
verbose=False) -> MeasureResult:
layer_names = activations_iterator.layer_names()
n_layers = len(layer_names)
mean_running = [RunningMeanWelford() for i in range(n_layers)]
for transformation, transformation_activations in activations_iterator.transformations_first(
):
# calculate the variance of all samples for this transformation
for x, batch_activations in transformation_activations:
for j, layer_activations in enumerate(batch_activations):
layer_measure = self.distance_aggregation.apply(
layer_activations)
mean_running[j].update(layer_measure)
# calculate the final mean over all transformations (and layers)
mean_variances = [b.mean() for b in mean_running]
return MeasureResult(mean_variances, layer_names, self)
def eval(self,
activations_iterator: ActivationsIterator,
verbose=False) -> MeasureResult:
layer_names = activations_iterator.layer_names()
n_intermediates = len(layer_names)
mean_running = [RunningMeanWelford() for i in range(n_intermediates)]
for x, transformation_activations_iterator in activations_iterator.samples_first(
):
# transformation_activations_iterator can iterate over all transforms
for x_transformed, activations in transformation_activations_iterator:
for j, layer_activations in enumerate(activations):
# calculate the distance aggregation only for this batch
layer_measure = self.distance_aggregation.apply(
layer_activations)
# update the mean over all transformation
mean_running[j].update(layer_measure)
# calculate the final mean over all samples (and layers)
mean_variances = [b.mean() for b in mean_running]
return MeasureResult(mean_variances, layer_names, self)
def eval(self,
activations_iterator: ActivationsIterator,
verbose=False) -> MeasureResult:
activations_iterator: ActivationsIterator = activations_iterator.get_inverted_activations_iterator(
)
ts = list(map(str, (activations_iterator.get_transformations())))
mean_variances_running = None
for transformation, samples_activations_iterator in activations_iterator.transformations_first(
):
samples_variances_running = None
# calculate the variance of all samples for this transformation
for x, batch_activations in samples_activations_iterator:
if mean_variances_running is None:
n_layers = len(batch_activations)
mean_variances_running = [
RunningMeanWelford() for i in range(n_layers)
]
if samples_variances_running is None:
n_layers = len(batch_activations)
samples_variances_running = [
RunningMeanAndVarianceWelford()
for i in range(n_layers)
]
for j, layer_activations in enumerate(batch_activations):
samples_variances_running[j].update_all(layer_activations)
# update the mean over all transformation (and layers)
for layer_mean_variances_running, layer_samples_variance_running in zip(
mean_variances_running, samples_variances_running):
layer_mean_variances_running.update(
layer_samples_variance_running.std())
# calculate the final mean over all transformations (and layers)
mean_variances = [b.mean() for b in mean_variances_running]
return MeasureResult(mean_variances,
activations_iterator.layer_names(), self)
def eval(self,
activations_iterator: ActivationsIterator,
verbose=False) -> MeasureResult:
activations_iterator = activations_iterator.get_inverted_activations_iterator(
)
mean_running = None
for x, transformation_activations_iterator in activations_iterator.samples_first(
):
# transformation_activations_iterator can iterate over all transforms
for x_transformed, activations in transformation_activations_iterator:
if mean_running is None:
mean_running = [
RunningMeanWelford() for i in range(len(activations))
]
for j, layer_activations in enumerate(activations):
layer_measure = self.distance_aggregation.apply(
layer_activations)
# update the mean over all transformation
mean_running[j].update(layer_measure)
# calculate the final mean over all samples (and layers)
means = [b.mean() for b in mean_running]
return MeasureResult(means, activations_iterator.layer_names(), self)
def eval(self,
activations_iterator: ActivationsIterator,
verbose=False) -> MeasureResult:
activations_iterator = activations_iterator.get_inverted_activations_iterator(
)
mean_running = None
for x, transformation_activations in activations_iterator.samples_first(
):
transformation_variances_running = None
#calculate the running mean/variance/std over all transformations of x
for x_transformed, activations in transformation_activations:
if mean_running is None:
n_layers = len(activations)
mean_running = [
RunningMeanWelford() for i in range(n_layers)
]
if transformation_variances_running is None:
n_layers = len(activations)
transformation_variances_running = [
RunningMeanAndVarianceWelford()
for i in range(n_layers)
]
for i, layer_activations in enumerate(activations):
# apply function to conv layers
# update the mean over all transformations for this sample
transformation_variances_running[i].update_all(
layer_activations)
# update the mean with the numpy sample of all transformations of x
for i, layer_variance in enumerate(
transformation_variances_running):
mean_running[i].update(layer_variance.std())
# calculate the final mean over all samples (for each layer)
mean_variances = [b.mean() for b in mean_running]
return MeasureResult(mean_variances,
activations_iterator.layer_names(), self)
def test_mean_welford(x: np.ndarray):
compare_means_batched(RunningMeanWelford(), x)
def __init__(self, layer_index: int, layer_name: str):
super(STDLayerMeasure).__init__(layer_index, layer_name)
self.running_mean = RunningMeanWelford()