def test_digits(verbose=False):
epochs = 1000
pad = 0.0001
feedback = True
split_learn = True
biases = True
Ns = [256 for _ in range(3)]
tokens = [str(x) for x in range(100)]
net = CHL_Net(Ns, feedback, split_learn, biases)
# Input/output pattern pairs (0-99)
in_coder = Coder(tanh_activator(pad, (Ns[0])))
out_coder = Coder(tanh_activator(pad, Ns[-1]))
patterns = [(in_coder.encode(tok), out_coder.encode(tok))
for tok in tokens]
net.train(epochs, patterns, verbose=verbose)
def test_arith(verbose=False):
epochs = 100
feedback = False
split_learn = False
biases = True
pad = 0.0001
N = 256
Ns = [N * 3, N * 2, N * 2]
net = CHL_Net(Ns, feedback, split_learn, biases)
in_size = int(Ns[0] / 3)
out_size = int(Ns[-1] / 2)
in1_coder = Coder(tanh_activator(pad, in_size))
in2_coder = Coder(tanh_activator(pad, in_size))
in3_coder = Coder(tanh_activator(pad, in_size))
out1_coder = Coder(tanh_activator(pad, out_size))
out2_coder = Coder(tanh_activator(pad, out_size))
# (x,y,op) => op(x,y) pairs
patterns = []
for op in arith_ops:
for i in range(10):
for j in range(10):
in1 = in1_coder.encode(str(i))
in2 = in2_coder.encode(str(j))
in3 = in3_coder.encode(op)
try:
f0, f1 = arith_ops[op]
out1 = out1_coder.encode(f0(i, j))
out2 = out2_coder.encode(f1(i, j))
except:
out1 = out1_coder.encode("null")
out2 = out2_coder.encode("null")
patterns.append(
(np.append(np.append(in1, in2, axis=0), in3,
axis=0), np.append(out1, out2, axis=0)))
net.train(epochs, patterns, verbose=verbose)
def __init__(self, nvmnet, opdef, arg_regs, res_regs, op_reg):
self.opdef = opdef
self.op_name = opdef.op_name
self.operations = dict(opdef.operations)
self.in_ops = list(opdef.in_ops)
self.out_ops = list(opdef.out_ops)
self.tokens = list(opdef.tokens)
self.arg_registers = arg_regs
self.res_registers = res_regs
self.op_register = op_reg
self.hidden_name = "%s_gh" % self.op_name
self.gate_name = "%s_go" % self.op_name
# 1. OP->HID, 2. OP->OP, [3. RESX->RESX, 4. RESX->RESY for op in ops]
self.gate_map = GateMap([(self.hidden_name, self.op_register, "u"),
(self.op_register, self.op_register, "u")] +
[("res", "res", op)
for op in self.operations] +
[("res", "arg", op) for op in self.operations])
# Hidden gate layer
N = 16
self.hidden_size = N**2
hid_activator = tanh_activator(nvmnet.pad, self.hidden_size)
self.hidden_layer = Layer(self.hidden_name, (N, N), hid_activator,
Coder(hid_activator))
# Gate output layer
self.gate_size = self.gate_map.get_gate_count()
gate_activator = heaviside_activator(self.gate_size)
self.gate_layer = Layer(self.gate_name, (self.gate_size, 1),
gate_activator, Coder(gate_activator))
# Gate layer (detects operator)
hidden_gate_layer = {
"name": self.hidden_name,
"neural model": "nvm",
"rows": 1,
"columns": self.hidden_size,
}
gate_layer = {
"name": self.gate_name,
"neural model": "nvm_heaviside",
"rows": 1,
"columns": self.gate_size,
}
self.structure = {
"name": self.op_name,
"type": "parallel",
"layers": [hidden_gate_layer, gate_layer]
}
# Make gate connection
def build_gate(to_name, index, suffix=""):
return {
"name": get_conn_name(to_name, self.gate_name, suffix),
"from layer": self.gate_name,
"to layer": to_name,
"type": "subset",
"opcode": "add",
"subset config": {
"from row end": 1,
"from column start": index,
"from column end": index + 1,
"to row end": 1,
"to column end": 1,
},
"plastic": False,
"gate": True,
}
# Squash weights to cancel gain
def build_squash(to_name, suffix="", gated=True):
return {
"name": get_conn_name(to_name, to_name, suffix),
"from layer": "bias",
"to layer": to_name,
"type": "fully connected",
"opcode": "add",
"plastic": False,
"gated": gated,
}
# Make weight/bias connections
def build_conns(to_name, from_name, suffix="", gated=True):
return [{
"name": get_conn_name(to_name, from_name, suffix + "-w"),
"from layer": from_name,
"to layer": to_name,
"type": "fully connected",
"opcode": "add",
"plastic": False,
"gated": gated
}, {
"name": get_conn_name(to_name, from_name, suffix + "-b"),
"from layer": 'bias',
"to layer": to_name,
"type": "fully connected",
"opcode": "add",
"plastic": False,
"gated": gated
}]
self.connections = []
# Hidden gate input
self.connections.append(
build_gate(
self.hidden_name,
self.gate_map.get_gate_index(
(self.hidden_name, self.op_register, "u"))))
self.connections += build_conns(self.hidden_name,
self.op_register,
gated=True)
# Hidden gate recurrence
self.connections += build_conns(self.hidden_name,
self.hidden_name,
gated=False)
# Gate activation
self.connections += build_conns(self.gate_name,
self.hidden_name,
gated=False)
# Operation squash
self.connections.append(
build_gate(
self.op_register,
self.gate_map.get_gate_index(
(self.op_register, self.op_register, "u")), self.op_name))
self.connections.append(
build_squash(self.op_register, suffix=self.op_name + "-squash"))
for op in self.operations:
for to_name in self.res_registers:
# Recurrent connections
self.connections.append(
build_gate(
to_name,
self.gate_map.get_gate_index(("res", "res", op)),
op + "-1"))
self.connections += build_conns(to_name,
to_name,
suffix=op,
gated=True)
# Inter-layer connections
self.connections.append(
build_gate(
to_name,
self.gate_map.get_gate_index(("res", "arg", op)),
op + "-2"))
for from_name in self.arg_registers:
if to_name != from_name:
self.connections += build_conns(to_name,
from_name,
suffix=op,
gated=True)
self.layer_map = {
name: nvmnet.layers[name]
for name in self.arg_registers + self.res_registers +
[self.op_register]
}
self.layer_map[self.gate_name] = self.gate_layer
self.layer_map[self.hidden_name] = self.hidden_layer
self.conn_names = tuple(conn["name"] for conn in self.connections)
def test(N, pad, mask_frac, mappings, stabil=5):
fsm_states = mappings.keys() + list(
set(v for m in mappings.values() for k, v in m))
input_states = list(set(k for m in mappings.values() for k, v in m))
shape = (N, N)
size = N**2
act = tanh_activator(pad, size)
act_log = logistic_activator(pad, size)
input_layer, fsm_layer = (Layer(k, shape, act, Coder(act)) for k in "ab")
input_layer.encode_tokens(input_states, orthogonal=True)
fsm_layer.encode_tokens(fsm_states, orthogonal=True)
########### OLD METHOD ###################
# Learn recurrent weights
w_r = np.zeros((size, size))
b = np.zeros((size, 1))
X = fsm_layer.encode_tokens(fsm_states)
dw, db = rehebbian(w_r, b, X, X, act, act)
w_r = w_r + dw
# Learn inter-regional weights
w = np.zeros((size, size * 2))
b = np.zeros((size, 1))
for s, m in mappings.items():
X = input_layer.encode_tokens([k for k, v in m])
s = np.repeat(fsm_layer.coder.encode(s), X.shape[1], axis=1)
X = np.concatenate((X, s), axis=0)
Y = fsm_layer.encode_tokens([v for k, v in m])
dw, db = rehebbian(w, b, X, Y, act, act)
w = w + dw
# Test
correct = 0
weighted = 0.
total = 0
for start, m in mappings.items():
start = fsm_layer.coder.encode(start)
for inp, end in m:
x = np.concatenate((input_layer.coder.encode(inp), start), axis=0)
y = act.f(w.dot(x))
# Stabilize
for _ in range(stabil):
old_y = y
y = act.f(w_r.dot(y))
if np.array_equal(y, old_y):
break
out = fsm_layer.coder.decode(y)
if out == end:
correct += 1
weighted += 1.0
else:
weighted += float(
len(
np.where(
np.sign(y) == np.sign(fsm_layer.coder.encode(
end))))) / size
total += 1
old_acc = float(correct) / total
weighted_old_acc = weighted / total
########### NEW METHOD ###################
input_layer, fsm_layer = (Layer(k, shape, act, Coder(act)) for k in "ab")
input_layer.encode_tokens(input_states, orthogonal=False)
fsm_layer.encode_tokens(fsm_states, orthogonal=False)
# Create gating masks for each state
w_masks = {
s: (np.random.random((size, size)) < (1. / mask_frac)).astype(np.float)
for s in fsm_states
}
# Ensure nonzero masks
for mask in w_masks.values():
if np.sum(mask) == 0:
mask[randint(0, mask.shape[0] - 1),
randint(0, mask.shape[1] - 1)] = 1.
# Test learning of masks
w_m = np.zeros((size**2, size))
b = np.zeros((size**2, 1))
X = fsm_layer.encode_tokens(fsm_states)
Y = np.concatenate(tuple(w_masks[s].reshape(-1, 1) for s in fsm_states),
axis=1)
dw, db = rehebbian(w_m, b, X, Y, act, act)
w_m = w_m + dw
'''
for s in fsm_states:
x = fsm_layer.coder.encode(s)
y = act_log.f(w_m.dot(x))
print(np.sum((y.reshape(size,size) > 0.5) != (w_masks[s] > 0.5)))
'''
# Learn recurrent weights
w_r = np.zeros((size, size))
b = np.zeros((size, 1))
X = fsm_layer.encode_tokens(fsm_states)
dw, db = rehebbian(w_r, b, X, X, act, act)
w_r = w_r + dw
# Learn inter-regional weights
w = np.zeros((size, size))
b = np.zeros((size, 1))
for start, m in mappings.items():
# Start state mask, input_layer input
X = input_layer.encode_tokens([k for k, v in m])
Y = fsm_layer.encode_tokens([v for k, v in m])
w_mask = w_masks[start]
dw, db = rehebbian(np.multiply(w, w_mask), b, X, Y, act, act)
w = w + (np.multiply(dw, w_mask) * mask_frac)
# Test
total = 0
weighted = 0.
masked_weighted = 0.
correct = 0
masked_correct = 0
for start, m in mappings.items():
#w_masked = np.multiply(w_masks[start], w)
x = fsm_layer.coder.encode(start)
w_masked = np.multiply(w, act_log.f(w_m.dot(x)).reshape(size, size))
#w_masked = np.multiply(w, (w_m.dot(x) > 0).astype(np.int).reshape(size,size))
for inp, end in m:
x = input_layer.coder.encode(inp)
y = act.f(w_masked.dot(x))
# Stabilize
for _ in range(stabil + 1):
old_y = y
y = act.f(w_r.dot(y))
if np.array_equal(y, old_y):
break
out = fsm_layer.coder.decode(y)
# Check output
if out == end:
correct += 1
weighted += 1.0
else:
weighted += float(
len(
np.where(
np.sign(y) == np.sign(fsm_layer.coder.encode(
end))))) / size
total += 1
new_acc = float(correct) / total
weighted_new_acc = weighted / total
return {
"old_acc": old_acc,
"new_acc": new_acc,
"weighted_old_acc": weighted_old_acc,
"weighted_new_acc": weighted_new_acc
}
def __init__(self, N, mask_frac, conv=1., stabil=10):
N = int(N / conv)
self.stabil = stabil
self.mask_frac = mask_frac
self.size = N**2
self.mask_size = int(self.size / self.mask_frac)
pad = 0.0001
# Build mem/ptr/ctx unit
self.prefix = "test"
layer_configs, connections = build_unit(self.prefix, N, "graph_net",
conv, pad)
# Assign gate indices
self.gate_layer_name = "g"
self.gates = {}
for conn in connections:
if any(
conn.get(key, False)
for key in ["gate", "decay", "learning"]):
conn["from layer"] = self.gate_layer_name
conn["subset config"]["from column start"] = len(self.gates)
conn["subset config"]["from column end"] = len(self.gates) + 1
self.gates[conn["name"]] = len(self.gates)
# Build gate layer
layer_configs.append(
build_layer(self.gate_layer_name, "nvm_heaviside", 1,
len(self.gates), pad))
structure = {
"name": "graph_net",
"type": "parallel",
"layers": layer_configs
}
self.net = Network({
"structures": [structure],
"connections": connections
})
### Create activators and coders
self.act = tanh_activator(pad, self.size)
self.act_h = heaviside_activator(self.size)
self.layer_names = [
self.prefix + "m", self.prefix + "p", self.prefix + "c",
self.gate_layer_name
]
self.acts = {
self.prefix + "m": self.act,
self.prefix + "p": self.act,
self.prefix + "c": self.act_h,
self.gate_layer_name: self.act_h,
}
self.coders = {
self.prefix + "m": Coder(self.act),
self.prefix + "p": Coder(self.act),
self.prefix + "c": Coder(self.act_h),
self.gate_layer_name: Coder(self.act_h),
}