def test(self, test_inputs, with_implicit_reward=False):
outs = get_layer_outs_new(self.model,
test_inputs,
skip=self.skip_layers)
for layer_index, layer_out in enumerate(
outs): # layer_out is output of layer for all inputs
for out_for_input in layer_out: # out_for_input is output of layer for single input
for neuron_index in range(out_for_input.shape[-1]):
neuron_out = np.mean(out_for_input[..., neuron_index])
global_neuron_index = (layer_index, neuron_index)
self.neuron_set.add(global_neuron_index)
neuron_low = self.major_func_regions[layer_index][0][
neuron_index]
neuron_high = self.major_func_regions[layer_index][1][
neuron_index]
section_length = (neuron_high - neuron_low) / self.k
section_index = floor(
(neuron_out - neuron_low) /
section_length) if section_length > 0 else 0
self.activation_table_by_section[(global_neuron_index,
section_index)] = True
if neuron_out < neuron_low:
self.lower_activation_table[global_neuron_index] = True
elif neuron_out > neuron_high:
self.upper_activation_table[global_neuron_index] = True
multisection_activated = len(self.activation_table_by_section.keys())
lower_activated = len(self.lower_activation_table.keys())
upper_activated = len(self.upper_activation_table.keys())
total = len(self.neuron_set)
return (
percent_str(multisection_activated, self.k * total), # kmn
multisection_activated,
percent_str(upper_activated + lower_activated, 2 * total), # nbc
percent_str(upper_activated, total), # snac
lower_activated,
upper_activated,
total,
multisection_activated,
upper_activated,
lower_activated,
total,
outs)
def measure_neuron_cov(model, test_inputs, scaler, threshold=0, skip_layers=None, outs=None):
if outs is None:
outs = get_layer_outs_new(model, test_inputs, skip_layers)
activation_table = defaultdict(bool)
for layer_index, layer_out in enumerate(outs): # layer_out is output of layer for all inputs
for out_for_input in layer_out: # out_for_input is output of layer for single input
out_for_input = scaler(out_for_input)
for neuron_index in range(out_for_input.shape[-1]):
activation_table[(layer_index, neuron_index)] = activation_table[(layer_index, neuron_index)] or\
np.mean(out_for_input[..., neuron_index]) > threshold
covered = len([1 for c in activation_table.values() if c])
total = len(activation_table.keys())
return percent_str(covered, total), covered, total, outs
def measure_tkn_with_pattern(model, test_inputs, k, skip=None, outs=None):
if skip is None:
skip = []
if outs is None:
outs = get_layer_outs(model, test_inputs, skip=skip)[:-1]
activation_table = {}
neuron_count_by_layer = {}
pattern_set = set()
layer_count = len(outs)
print(layer_count)
for input_index in range(len(
test_inputs)): # out_for_input is output of layer for single input
pattern = []
for layer_index in range(
layer_count): # layer_out is output of layer for all inputs
out_for_input = outs[layer_index][0][input_index]
neuron_outs = np.zeros((out_for_input.shape[-1], ))
neuron_count_by_layer[layer_index] = len(neuron_outs)
for i in range(out_for_input.shape[-1]):
neuron_outs[i] = np.mean(out_for_input[..., i])
top_k_neuron_indexes = (np.argsort(neuron_outs,
axis=None)[-k:len(neuron_outs)])
pattern.append(tuple(top_k_neuron_indexes))
for neuron_index in top_k_neuron_indexes:
activation_table[(layer_index, neuron_index)] = True
if layer_index + 1 == layer_count:
pattern_set.add(tuple(pattern))
neuron_count = sum(neuron_count_by_layer.values())
covered = len(activation_table.keys())
print(neuron_count)
print(covered)
return percent_str(
covered, neuron_count), covered, neuron_count, len(pattern_set), outs
def test(self, test_inputs):
outs = get_layer_outs_new(self.model, test_inputs, self.skip_layers)
neuron_count_by_layer = {}
layer_count = len(outs)
for input_index in range(
len(test_inputs
)): # out_for_input is output of layer for single input
pattern = []
for layer_index in range(
layer_count
): # layer_out is output of layer for all inputs
out_for_input = outs[layer_index][input_index]
neuron_outs = np.zeros((out_for_input.shape[-1], ))
neuron_count_by_layer[layer_index] = len(neuron_outs)
for i in range(out_for_input.shape[-1]):
neuron_outs[i] = np.mean(out_for_input[..., i])
top_k_neuron_indexes = (np.argsort(
neuron_outs, axis=None)[-self.k:len(neuron_outs)])
pattern.append(tuple(top_k_neuron_indexes))
for neuron_index in top_k_neuron_indexes:
self.activation_table[(layer_index, neuron_index)] = True
if layer_index + 1 == layer_count:
self.pattern_set.add(tuple(pattern))
neuron_count = sum(neuron_count_by_layer.values())
covered = len(self.activation_table.keys())
return (
percent_str(covered, neuron_count), # tknc
covered,
neuron_count,
len(self.pattern_set), # tknp
outs)
def test(self, test_inputs, with_implicit_reward=False):
outs = get_layer_outs_new(self.model, test_inputs, self.skip_layers)
for layer_index, layer_out in enumerate(outs): # layer_out is output of layer for all inputs
for out_for_input in layer_out: # out_for_input is output of layer for single input
out_for_input = self.scaler(out_for_input)
for neuron_index in range(out_for_input.shape[-1]):
if self.activation_table[(layer_index, neuron_index)] == 1:
pass
elif np.mean(out_for_input[..., neuron_index]) > self.threshold:
self.activation_table[(layer_index, neuron_index)] = 1
elif self.calc_implicit_reward_neuron:
p1 = np.mean(out_for_input[..., neuron_index])
p2 = self.threshold
r = self.calc_implicit_reward_neuron(p1, p2)
self.activation_table[(layer_index, neuron_index)] = r
reward, covered, implicit_reward = self.calc_reward(self.activation_table, with_implicit_reward=with_implicit_reward)
total = len(self.activation_table.keys())
return percent_str(reward, total), reward, total, outs
def measure_k_multisection_cov(model,
test_inputs,
k,
train_inputs=None,
major_func_regions=None,
skip=None,
outs=None):
if skip is None:
skip = []
if outs is None:
outs = get_layer_outs(model, test_inputs, skip=skip)
if major_func_regions is None:
if train_inputs is None:
raise ValueError(
"Training inputs must be provided when major function regions are not given"
)
major_func_regions = calc_major_func_regions(model, train_inputs, skip)
activation_table_by_section, upper_activation_table, lower_activation_table = {}, {}, {}
neuron_set = set()
for layer_index, layer_out in enumerate(
outs): # layer_out is output of layer for all inputs
print(layer_index)
for out_for_input in layer_out[
0]: # out_for_input is output of layer for single input
for neuron_index in range(out_for_input.shape[-1]):
neuron_out = np.mean(out_for_input[..., neuron_index])
global_neuron_index = (layer_index, neuron_index)
neuron_set.add(global_neuron_index)
neuron_low = major_func_regions[layer_index][0][neuron_index]
neuron_high = major_func_regions[layer_index][1][neuron_index]
section_length = (neuron_high - neuron_low) / k
section_index = floor(
(neuron_out - neuron_low) /
section_length) if section_length > 0 else 0
activation_table_by_section[(global_neuron_index,
section_index)] = True
if neuron_out < neuron_low:
lower_activation_table[global_neuron_index] = True
elif neuron_out > neuron_high:
upper_activation_table[global_neuron_index] = True
multisection_activated = len(activation_table_by_section.keys())
lower_activated = len(lower_activation_table.keys())
upper_activated = len(upper_activation_table.keys())
total = len(neuron_set)
return (
percent_str(multisection_activated, k * total), # kmn
multisection_activated,
percent_str(upper_activated + lower_activated, 2 * total), # nbc
percent_str(upper_activated, total), # snac
lower_activated,
upper_activated,
total,
multisection_activated,
upper_activated,
lower_activated,
total,
outs)