def calculate_read_head_influence(read_vector, weights_dict,
controller_output):
read_heads = read_vector.shape[1]
width = read_vector.shape[2]
read_head_influence = np.empty([read_vector.shape[0], read_heads])
for t in range(controller_output.shape[0]):
mu_output = np.reshape(read_vector[t, :, :], [read_heads * width])
co_output = np.concatenate([mu_output, controller_output[t, :]],
axis=-1)
matmul = np.matmul(co_output, weights_dict['output_layer/weights_concat:0']['var']) + \
weights_dict['output_layer/bias_merge:0']['var']
pred_full = softmax(matmul)
for r in range(read_heads):
zero_head = np.ones([read_heads, width])
zero_head[r, :] = 0
memory_unit_output = np.reshape(
read_vector[t, :, :] * zero_head, [read_heads * width])
co_output = np.concatenate(
[memory_unit_output, controller_output[t, :]], axis=-1)
matmul_muh = np.matmul(co_output, weights_dict['output_layer/weights_concat:0']['var']) + \
weights_dict['output_layer/bias_merge:0']['var']
pred_head = softmax(matmul_muh)
read_head_influence[t, r] = np.abs(pred_full - pred_head).sum()
read_head_influence = softmax(read_head_influence)
return read_head_influence
def estimate_memory_usage(variables):
analyse_values, prediction, decoded_predictions, data_sample, weights_dict = variables
data, target, mask, x_word = data_sample
analyse_outputs, analyse_signals, analyse_states = analyse_values
controller_states, memory_states = analyse_states
if memory_states.__len__() == 6:
memory, usage_vector, write_weightings, precedence_weighting, link_matrix, read_weightings = memory_states
else:
memory, usage_vector, write_weightings, read_weightings = memory_states
read_head = read_weightings.shape[2]
memory_width = memory.shape[-1]
time_len = memory.shape[0]
memory_unit_mask = np.concatenate([np.ones([time_len, read_head * memory_width]), np.zeros(
[time_len, analyse_outputs.shape[-1] - (read_head * memory_width)])], axis=-1)
controller_mask = np.concatenate([np.zeros([time_len, read_head * memory_width]),
np.ones([time_len, analyse_outputs.shape[-1] - (read_head * memory_width)])],
axis=-1)
controller_influence = []
memory_unit_influence = []
for b in range(mask.shape[1]):
matmul = np.matmul(analyse_outputs[:, b, :], weights_dict['output_layer/weights_concat:0']['var']) + \
weights_dict['output_layer/bias_merge:0']['var']
pred_both = softmax(matmul)
matmul = np.matmul(analyse_outputs[:, b, :] * controller_mask,
weights_dict['output_layer/weights_concat:0']['var']) + \
weights_dict['output_layer/bias_merge:0']['var']
pred_c = softmax(matmul)
matmul = np.matmul(analyse_outputs[:, b, :] * memory_unit_mask,
weights_dict['output_layer/weights_concat:0']['var']) + \
weights_dict['output_layer/bias_merge:0']['var']
pred_mu = softmax(matmul)
co_inf = (np.abs(pred_both - pred_mu) * np.expand_dims(mask[:, b], 1)).sum() / mask[:, b].sum()
me_inf = (np.abs(pred_both - pred_c) * np.expand_dims(mask[:, b], 1)).sum() / mask[:, b].sum()
co_inf = (1 / (co_inf + me_inf)) * co_inf
me_inf = (1 / (co_inf + me_inf)) * me_inf
controller_influence.append(co_inf)
memory_unit_influence.append(me_inf)
controller_influence = np.mean(controller_influence)
memory_unit_influence = np.mean(memory_unit_influence)
return controller_influence, memory_unit_influence
def get_memory_influence(self, batch=-1):
memory_unit_mask = np.concatenate([
np.ones([self.seq_len, self.max_read_head * self.memory_width]),
np.zeros([
self.seq_len, self.output_size -
(self.max_read_head * self.memory_width)
])
],
axis=-1)
controller_mask = np.concatenate([
np.zeros([self.seq_len, self.max_read_head * self.memory_width]),
np.ones([
self.seq_len, self.output_size -
(self.max_read_head * self.memory_width)
])
],
axis=-1)
controller_influence = []
memory_unit_influence = []
if batch == -1:
batch_size = self.mask.shape[1]
else:
batch_size = 1
for b in range(batch_size):
if batch != -1:
b = batch
matmul = np.matmul(self.analyse_outputs[:, b, :],
self.weights_dict['output_layer/weights_concat:0']['var']) + \
self.weights_dict['output_layer/bias_merge:0']['var']
pred_both = softmax(matmul)
matmul = np.matmul(self.analyse_outputs[:, b, :] * controller_mask,
self.weights_dict['output_layer/weights_concat:0']['var']) + \
self.weights_dict['output_layer/bias_merge:0']['var']
pred_c = softmax(matmul)
matmul = np.matmul(self.analyse_outputs[:, b, :] * memory_unit_mask,
self.weights_dict['output_layer/weights_concat:0']['var']) + \
self.weights_dict['output_layer/bias_merge:0']['var']
pred_mu = softmax(matmul)
co_inf = np.abs(pred_both - pred_mu).sum(axis=-1)
me_inf = np.abs(pred_both - pred_c).sum(axis=-1)
co_inf = (1 / (co_inf + me_inf + 1e-8)) * co_inf
me_inf = (1 / (co_inf + me_inf + 1e-8)) * me_inf
controller_influence.append(co_inf)
memory_unit_influence.append(me_inf)
if controller_influence.__len__() > 1:
controller_influence = np.mean(controller_influence, axis=1)
memory_unit_influence = np.mean(memory_unit_influence, axis=1)
else:
controller_influence = controller_influence[0]
memory_unit_influence = memory_unit_influence[0]
return controller_influence, memory_unit_influence