def sqnet_test():
N = [32] * 3
L = len(N) - 1
f, df, af = [np.tanh] * L, [mu.tanh_df] * L, [np.arctanh] * L
f = {k: np.tanh for k in range(1, L + 1)}
df = {k: mu.tanh_df for k in range(1, L + 1)}
af = {k: np.arctanh for k in range(1, L + 1)}
sqn = BackPropSeqNet(N=N, f=f, df=df, af=af)
memory_size = 40
sqn.pretrain(memory_size, eta=0.01, num_epochs=10000, term=0.1, verbose=1)
# mem check
k = sqn.first()
keys = np.empty((sqn.layer_size(), memory_size))
keys[:, [0]] = k
clobbered = np.zeros(memory_size, dtype=bool)
for m in range(1, memory_size):
k = sqn.next(k)
keys[:, [m]] = k
clobbered[:m] |= (keys[:, :m] == keys[:, [m]]).all(axis=0)
seq_mem_check = (not clobbered.any())
print('Final error: %f' % sqn.error_history[-1])
print('mem check passed: %s' % str(seq_mem_check))
print(clobbered)
print(mu.patterns_to_ints(keys))
print(
mu.patterns_to_ints(keys)[:, np.newaxis] == mu.patterns_to_ints(keys)[
np.newaxis, :])
# learning curve
plt.plot(sqn.error_history, 'r.')
plt.xlabel('Training iteration')
plt.ylabel('L_inf error on key sequence')
plt.title('sequence pretraining learning curve')
plt.show()
def run_array_write_trial(mnh, values, write_epochs, params, verbose=0):
N, M = values.shape
kv_accuracy, seq_accuracy = np.empty(M), np.empty(M)
keys = np.empty((N, M))
key = mnh.first()
for m in range(M):
keys[:,[m]] = key
mnh.write(key, values[:,[m]], num_epochs=write_epochs)
key = mnh.next(key)
kv_accuracy[m] = mnh.key_value_accuracy()
seq_accuracy[m] = mnh.sequence_accuracy()
# sanity check
net_keys = np.empty(keys.shape)
net_values = np.empty(values.shape)
for m in range(M):
net_keys[:,[m]] = mnh.next(keys[:,[m]])
net_values[:,[m]] = mnh.read(keys[:,[m]])
clobbered = np.zeros(keys.shape[1],dtype=bool)
for m in range(M):
clobbered[:m] |= (keys[:,:m]==keys[:,[m]]).all(axis=0)
seq_mem_check = (not clobbered.any()) and (net_keys[:,:-1]==keys[:,1:]).all()
kv_mem_check = (values[:,~clobbered]==net_values[:,~clobbered]).all()
if verbose > 0:
print(params)
if verbose > 1:
print('***kv trial***')
print('clobbered:')
print(clobbered)
print('seq keys:')
print(mu.patterns_to_ints(keys[:,1:]))
print(mu.patterns_to_ints(net_keys[:,:-1]))
print('kv values:')
print(mu.patterns_to_ints(keys))
print(mu.patterns_to_ints(values))
print(mu.patterns_to_ints(net_values))
print('seq, kv mem check:')
print(seq_mem_check, kv_mem_check)
print('seq, kv acc:')
print(seq_accuracy)
print(kv_accuracy)
print(seq_accuracy[-1], kv_accuracy[-1])
print('%d failed writes'%mnh.net.failed_write_count)
# h = [None,None]
# for c in range(mnh.net.cheat_error_history.shape[0]):
# h[0] = plt.plot(mnh.net.cheat_error_history[c,:],'r.')[0]
# h[1] = plt.plot(mnh.net.write_error_history,'b.')[0]
# plt.xlabel('Training iteration')
# plt.ylabel('L_inf error on previous and current memories')
# plt.title('learning curves during write(k,v)')
# plt.legend(h,['||read(k_prev)-sign(read(k_prev))||','||read(k)-v||'])
# plt.show()
return seq_accuracy, kv_accuracy, seq_mem_check, kv_mem_check
def passive_ticks(self, num_ticks, eta=0.01, batch=True, verbose=0):
for t in range(num_ticks):
x = mu.forward_pass(self.current_key, self.W_kv, self.f_kv)
value = np.sign(x[self.L_kv])
# Progress update:
e = x[self.L_kv] - value
# Weight update:
y = mu.backward_pass(x, e, self.W_kv, self.df_kv)
G = mu.error_gradient(x, y)
for k in self.W_kv:
if batch:
self.dW_kv[k] += -eta * G[k]
if (self.current_key == self.last_key).all():
self.W_kv[k] += self.dW_kv[k]
self.dW_kv[k] *= 0
else:
self.W_kv[k] += -eta * G[k]
# passive advance
if verbose > 0:
print('passive tick %d: k->v %s (|>| %f)' %
(t,
mu.patterns_to_ints(
np.concatenate((self.current_key, value), axis=1)),
np.fabs(x[self.L_kv]).min()))
if (self.current_key == self.last_key).all():
self.current_key = self.first()
else:
self.current_key = self.next(self.current_key)
def run_array_rotate_trial(mnh, values):
M = values.shape[1]
kv_accuracy = []
# write values to array
k = mnh.first()
for m in range(M):
mnh.write(k, values[:,[m]])
kv_accuracy.append(mnh.key_value_accuracy())
k = mnh.next(k)
# rotate array
k = mnh.first()
v_prev = mnh.read(k)
for m in range(1,M):
k = mnh.next(k)
v = mnh.read(k)
mnh.write(k, v_prev)
kv_accuracy.append(mnh.key_value_accuracy())
v_prev = v
mnh.write(mnh.first(), v_prev)
kv_accuracy.append(mnh.key_value_accuracy())
# sanity check
net_values = []
k = mnh.first()
for m in range(M):
net_values.append(mnh.read(k))
k = mnh.next(k)
net_values = np.concatenate(net_values,axis=1)
mem_check = (values.shape == net_values.shape) and (np.roll(values,1,axis=1)==net_values).all()
print('***array trial***')
print('array rotate:')
print(mu.patterns_to_ints(values))
print(mu.patterns_to_ints(net_values))
print('mem check:')
print(mem_check)
print('acc:')
print(kv_accuracy)
return kv_accuracy, mem_check
def memory_string(self, to_int=True):
def _val(k):
if mu.hash_pattern(k) in self.key_value_map:
return self.key_value_map[mu.hash_pattern(k)]
else: return np.nan*np.ones((self.net.layer_size(),1))
# key sequence
k = self.first()
keys = [k]
values = [_val(k)]
while mu.hash_pattern(k) in self.key_sequence:
k = self.key_sequence[mu.hash_pattern(k)]
keys.append(k)
values.append(_val(k))
if to_int:
string = 'keys/values:\n'
string += str(np.concatenate([
mu.patterns_to_ints(keys),
mu.patterns_to_ints(values)],axis=0))
else:
string = 'keys:\n'
string += str(np.concatenate(keys,axis=1))
string += '\nvalues:\n'
string += str(np.concatenate(values,axis=1))
return string
def next(self, key):
next_key = mu.int_to_pattern(self.layer_size(),
mu.patterns_to_ints(key)[0] + 1)
if (self.last_key == key).all(): self.last_key = next_key
return next_key
def next(self, k):
return self._noise(mu.int_to_pattern(self.N, mu.patterns_to_ints(k)[0]+1))