def __init__(self, nvmnet, auxiliary=[]):
self.auxiliary = auxiliary
aux_structures = [aux.structure for aux in auxiliary]
aux_conns = [c for aux in auxiliary for c in aux.connections]
structure, connections = make_syngen_network(nvmnet)
self.net = Network({
"structures": [structure] + aux_structures,
"connections": connections + aux_conns
})
# Initialize bias
self.get_output("bias")[0] = 1.0
self.initialize_weights(nvmnet)
self.initialize_activity(nvmnet)
def __init__(self, N, mask_frac, conv=1., stabil=10):
N = int(N / conv)
self.stabil = stabil
self.mask_frac = mask_frac
self.size = N**2
self.mask_size = int(self.size / self.mask_frac)
pad = 0.0001
# Build mem/ptr/ctx unit
self.prefix = "test"
layer_configs, connections = build_unit(self.prefix, N, "graph_net",
conv, pad)
# Assign gate indices
self.gate_layer_name = "g"
self.gates = {}
for conn in connections:
if any(
conn.get(key, False)
for key in ["gate", "decay", "learning"]):
conn["from layer"] = self.gate_layer_name
conn["subset config"]["from column start"] = len(self.gates)
conn["subset config"]["from column end"] = len(self.gates) + 1
self.gates[conn["name"]] = len(self.gates)
# Build gate layer
layer_configs.append(
build_layer(self.gate_layer_name, "nvm_heaviside", 1,
len(self.gates), pad))
structure = {
"name": "graph_net",
"type": "parallel",
"layers": layer_configs
}
self.net = Network({
"structures": [structure],
"connections": connections
})
### Create activators and coders
self.act = tanh_activator(pad, self.size)
self.act_h = heaviside_activator(self.size)
self.layer_names = [
self.prefix + "m", self.prefix + "p", self.prefix + "c",
self.gate_layer_name
]
self.acts = {
self.prefix + "m": self.act,
self.prefix + "p": self.act,
self.prefix + "c": self.act_h,
self.gate_layer_name: self.act_h,
}
self.coders = {
self.prefix + "m": Coder(self.act),
self.prefix + "p": Coder(self.act),
self.prefix + "c": Coder(self.act_h),
self.gate_layer_name: Coder(self.act_h),
}
class SyngenNVM:
def __init__(self, nvmnet, auxiliary=[]):
self.auxiliary = auxiliary
aux_structures = [aux.structure for aux in auxiliary]
aux_conns = [c for aux in auxiliary for c in aux.connections]
structure, connections = make_syngen_network(nvmnet)
self.net = Network({
"structures": [structure] + aux_structures,
"connections": connections + aux_conns
})
# Initialize bias
self.get_output("bias")[0] = 1.0
self.initialize_weights(nvmnet)
self.initialize_activity(nvmnet)
def initialize_weights(self, nvmnet):
init_syngen_nvm_weights(nvmnet, self.net)
for aux in self.auxiliary:
aux.initialize_weights(self, nvmnet)
def initialize_activity(self, nvmnet):
try:
init_syngen_nvm_activity(nvmnet.activity, self.net)
except AttributeError:
pass
for aux in self.auxiliary:
aux.initialize_activity(self, nvmnet)
def get_output(self, layer_name):
return self.net.get_neuron_data("nvm", layer_name,
"output").to_np_array()
def decode_output(self, layer_name, coder):
return coder.decode(self.get_output(layer_name))
def run(self, syn_env=None, args={}):
modules = [] if syn_env is None else syn_env.modules
default_args = {
"multithreaded": False,
"worker threads": 0,
"iterations": 0,
"refresh rate": 0,
"verbose": False,
"learning flag": False
}
for key, val in default_args.items():
if key not in args:
args[key] = val
env = Environment({"modules": modules})
report = self.net.run(env, args)
env.free()
return report
def free(self):
self.net.free()
self.net = None
class GraphNet:
def __init__(self, N, mask_frac, conv=1., stabil=10):
N = int(N / conv)
self.stabil = stabil
self.mask_frac = mask_frac
self.size = N**2
self.mask_size = int(self.size / self.mask_frac)
pad = 0.0001
# Build mem/ptr/ctx unit
self.prefix = "test"
layer_configs, connections = build_unit(self.prefix, N, "graph_net",
conv, pad)
# Assign gate indices
self.gate_layer_name = "g"
self.gates = {}
for conn in connections:
if any(
conn.get(key, False)
for key in ["gate", "decay", "learning"]):
conn["from layer"] = self.gate_layer_name
conn["subset config"]["from column start"] = len(self.gates)
conn["subset config"]["from column end"] = len(self.gates) + 1
self.gates[conn["name"]] = len(self.gates)
# Build gate layer
layer_configs.append(
build_layer(self.gate_layer_name, "nvm_heaviside", 1,
len(self.gates), pad))
structure = {
"name": "graph_net",
"type": "parallel",
"layers": layer_configs
}
self.net = Network({
"structures": [structure],
"connections": connections
})
### Create activators and coders
self.act = tanh_activator(pad, self.size)
self.act_h = heaviside_activator(self.size)
self.layer_names = [
self.prefix + "m", self.prefix + "p", self.prefix + "c",
self.gate_layer_name
]
self.acts = {
self.prefix + "m": self.act,
self.prefix + "p": self.act,
self.prefix + "c": self.act_h,
self.gate_layer_name: self.act_h,
}
self.coders = {
self.prefix + "m": Coder(self.act),
self.prefix + "p": Coder(self.act),
self.prefix + "c": Coder(self.act_h),
self.gate_layer_name: Coder(self.act_h),
}
def add_gate_pattern(self, name, on):
arr = np.zeros(len(self.gates))
for to_prefix, to_name, from_prefix, from_name, typ in on:
i = "%s%s<%s%s_%s" % (to_prefix, to_name, from_prefix, from_name,
typ)
arr[self.gates[i]] = 1.
self.coders[self.gate_layer_name].encode(name, arr)
def execute(self,
visualizer=False,
input_iters={},
output_iters={},
args={}):
test_state = {
k: {
# Accuracy of Hebbian mem recovery
"w_correct": 0.,
# Symbolic final correct
"correct": 0,
# Total tests
"total": 0,
}
for k, v in output_iters.items()
}
def input_callback(layer_name, data):
try:
inp = next(input_iters[layer_name])
if inp is not None:
np.copyto(
data, self.acts[layer_name].g(
self.coders[layer_name].encode(inp)).flat)
except StopIteration:
interrupt_engine()
except Exception as e:
print(e)
interrupt_engine()
# Collects and validates memory state
def output_callback(layer_name, data):
try:
tok = next(output_iters[layer_name])
if tok is not None:
out = self.coders[layer_name].decode(data)
# Check output
if out == tok:
test_state[layer_name]["w_correct"] += 1
test_state[layer_name]["correct"] += 1
else:
test_state[layer_name]["w_correct"] += (np.sum(
np.sign(data) == np.sign(self.acts[layer_name].g(
self.coders[layer_name].encode(tok))).flat) /
data.size)
test_state[layer_name]["total"] += 1
except StopIteration:
interrupt_engine()
except Exception as e:
print(e)
interrupt_engine()
modules = [
make_custom_input_module("graph_net", list(input_iters.keys()),
"icb", input_callback),
make_custom_output_module("graph_net", list(output_iters.keys()),
"ocb", output_callback)
]
if visualizer:
modules.append({
"type":
"visualizer",
"negative":
True,
"layers": [{
"structure": "graph_net",
"layer": layer_name
} for layer_name in self.layer_names]
})
default_args = {
"multithreaded": True,
"worker threads": 0,
#"devices" : get_cpu(),
"iterations": 0,
"refresh rate": 0,
#"verbose" : True,
"learning flag": False
}
for key, val in default_args.items():
if key not in args:
args[key] = val
# Clear prior activity
for layer_name in self.layer_names:
self.net.get_neuron_data("graph_net", layer_name,
"output").to_np_array().fill(0.)
env = Environment({"modules": modules})
report = self.net.run(env, args)
env.free()
return report, test_state
def learn(self, mappings):
kfts = [(k, start, v) for start, m in mappings.items() for k, v in m]
# Construct masks
for k, start, v in kfts:
mask = np.zeros((self.size, 1))
mask[np.random.choice(self.size, self.mask_size,
replace=False)] = 1.
self.coders[self.prefix + "c"].encode(k, mask)
# Set up I/O sequence
mems, ptrs, ctxs, gates = [], [], [], []
self.add_gate_pattern("learn1", [])
self.add_gate_pattern(
"learn2", [(self.prefix, "p", self.prefix, "m", "learning"),
(self.prefix, "m", self.prefix, "m", "decay")])
self.add_gate_pattern("learn3", [])
self.add_gate_pattern(
"learn4", [(self.prefix, "m", self.prefix, "m", "learning"),
(self.prefix, "m", self.prefix, "c", "gate"),
(self.prefix, "p", self.prefix, "c", "gate")])
self.add_gate_pattern(
"learn5", [(self.prefix, "m", self.prefix, "p", "learning"),
(self.prefix, "c", self.prefix, "c", "decay"),
(self.prefix, "m", self.prefix, "m", "decay"),
(self.prefix, "p", self.prefix, "p", "decay")])
# Learn from/to ptr/mem transitions
for key_sym, from_sym, to_sym in kfts:
# Load mem/ptr/ctx
mems.append(from_sym)
ptrs.append(from_sym)
ctxs.append(key_sym)
gates.append("learn1")
# Learn mem->ptr, close mem gain
# learn_pm, mem_decay
mems.append(None)
ptrs.append(None)
ctxs.append(None)
gates.append("learn2")
# Load mem
mems.append(to_sym)
ptrs.append(None)
ctxs.append(None)
gates.append("learn3")
# Learn mem->mem, open ctx->mem and ctx->ptr
# learn_mm, mem_ctx, ptr_ctx
mems.append(None)
ptrs.append(None)
ctxs.append(None)
gates.append("learn4")
# Learn ptr->mem, close all gain
# learn_mp, ctx_decay, mem_decay, ptr_decay
mems.append(None)
ptrs.append(None)
ctxs.append(None)
gates.append("learn5")
input_iters = {
self.prefix + "m": iter(mems),
self.prefix + "p": iter(ptrs),
self.prefix + "c": iter(ctxs),
self.gate_layer_name: iter(gates)
}
self.execute(input_iters=input_iters)
def test_recovery(self, mappings):
# Retrieve all memory states
mem_states = set(k for k in mappings.keys()).union(
set(v for m in mappings.values() for k, v in m))
# Set up I/O sequence
mems, ctxs, gates = [], [], []
outputs = []
self.add_gate_pattern("recov1",
[(self.prefix, "c", self.prefix, "c", "decay"),
(self.prefix, "p", self.prefix, "p", "decay"),
(self.prefix, "m", self.prefix, "c", "gate")])
self.add_gate_pattern("recov2",
[(self.prefix, "m", self.prefix, "m", "gate"),
(self.prefix, "c", self.prefix, "c", "decay"),
(self.prefix, "m", self.prefix, "m", "decay"),
(self.prefix, "p", self.prefix, "p", "decay")])
self.add_gate_pattern("recov3",
[(self.prefix, "c", self.prefix, "c", "decay"),
(self.prefix, "m", self.prefix, "m", "decay"),
(self.prefix, "p", self.prefix, "p", "decay")])
for tok in mem_states:
for key_sym in self.coders[self.prefix + "c"].list_tokens():
# Load input, open mem->mem, close all gain
# mem_mem, ctx_decay, mem_decay, ptr_decay
mems.append(tok)
ctxs.append(key_sym)
gates.append("recov1")
outputs.append(None)
# Stabilize
for _ in range(self.stabil):
mems.append(None)
ctxs.append(None)
gates.append("recov2")
outputs.append(None)
# Read output, close all gain
# ctx_decay, mem_decay, ptr_decay
mems.append(None)
ctxs.append(None)
gates.append("recov3")
outputs.append(tok)
input_iters = {
self.prefix + "m": iter(mems),
self.prefix + "c": iter(ctxs),
self.gate_layer_name: iter(gates)
}
output_iters = {
self.prefix + "m": iter(outputs),
}
report, test_state = self.execute(input_iters=input_iters,
output_iters=output_iters)
correct = test_state[self.prefix + "m"]["correct"]
w_correct = test_state[self.prefix + "m"]["w_correct"]
total = test_state[self.prefix + "m"]["total"]
return (float(correct) / total, w_correct / total)
def test_transit(self, mappings):
# Set up I/O sequence
mems, ctxs, gates = [], [], []
outputs = []
self.add_gate_pattern("trans1",
[(self.prefix, "p", self.prefix, "p", "decay")])
self.add_gate_pattern("trans2",
[(self.prefix, "p", self.prefix, "m", "gate"),
(self.prefix, "m", self.prefix, "m", "decay"),
(self.prefix, "p", self.prefix, "p", "decay"),
(self.prefix, "p", self.prefix, "c", "gate")])
self.add_gate_pattern("trans3",
[(self.prefix, "m", self.prefix, "p", "gate"),
(self.prefix, "m", self.prefix, "m", "decay"),
(self.prefix, "p", self.prefix, "p", "decay"),
(self.prefix, "m", self.prefix, "c", "gate")])
self.add_gate_pattern("trans4",
[(self.prefix, "m", self.prefix, "m", "gate"),
(self.prefix, "c", self.prefix, "c", "decay"),
(self.prefix, "m", self.prefix, "m", "decay"),
(self.prefix, "p", self.prefix, "p", "decay")])
self.add_gate_pattern("trans5",
[(self.prefix, "c", self.prefix, "c", "decay"),
(self.prefix, "m", self.prefix, "m", "decay"),
(self.prefix, "p", self.prefix, "p", "decay")])
for start, m in mappings.items():
for inp, end in m:
# Decay mem/ptr
mems.append(start)
ctxs.append(inp)
gates.append("trans1")
outputs.append(None)
# p<m, decay mem/ptr, p<c
mems.append(None)
ctxs.append(None)
gates.append("trans2")
outputs.append(None)
# m<p, decay mem/ptr, m<c
mems.append(None)
ctxs.append(None)
gates.append("trans3")
outputs.append(None)
for _ in range(self.stabil):
# m<m
mems.append(None)
ctxs.append(None)
gates.append("trans4")
outputs.append(None)
mems.append(None)
ctxs.append(None)
# Clear
gates.append("trans5")
outputs.append(end)
input_iters = {
self.prefix + "m": iter(mems),
self.prefix + "c": iter(ctxs),
self.gate_layer_name: iter(gates)
}
output_iters = {self.prefix + "m": iter(outputs)}
report, test_state = self.execute(
#visualizer=True, args={"refresh rate" : 10, "verbose" : True},
input_iters=input_iters,
output_iters=output_iters)
correct = test_state[self.prefix + "m"]["correct"]
w_correct = test_state[self.prefix + "m"]["w_correct"]
total = test_state[self.prefix + "m"]["total"]
return (float(correct) / total, w_correct / total)
def test_traversal(self, trajs):
# Set up I/O sequence
mems, ctxs, gates = [], [], []
outputs = []
self.add_gate_pattern("trav_clear",
[(self.prefix, "m", self.prefix, "m", "decay"),
(self.prefix, "p", self.prefix, "p", "decay"),
(self.prefix, "c", self.prefix, "c", "decay")])
self.add_gate_pattern("trav_load",
[(self.prefix, "p", self.prefix, "p", "decay"),
(self.prefix, "c", self.prefix, "c", "decay")])
self.add_gate_pattern("trav1",
[(self.prefix, "p", self.prefix, "p", "decay")])
self.add_gate_pattern("trav2",
[(self.prefix, "p", self.prefix, "m", "gate"),
(self.prefix, "m", self.prefix, "m", "decay"),
(self.prefix, "p", self.prefix, "p", "decay"),
(self.prefix, "p", self.prefix, "c", "gate")])
self.add_gate_pattern("trav3",
[(self.prefix, "m", self.prefix, "p", "gate"),
(self.prefix, "m", self.prefix, "m", "decay"),
(self.prefix, "p", self.prefix, "p", "decay"),
(self.prefix, "m", self.prefix, "c", "gate")])
self.add_gate_pattern("trav4",
[(self.prefix, "m", self.prefix, "m", "gate"),
(self.prefix, "c", self.prefix, "c", "decay"),
(self.prefix, "m", self.prefix, "m", "decay"),
(self.prefix, "p", self.prefix, "p", "decay")])
self.add_gate_pattern("trav5",
[(self.prefix, "c", self.prefix, "c", "decay"),
(self.prefix, "p", self.prefix, "p", "decay")])
for start, ctx_keys, traj in trajs:
mems.append(None)
ctxs.append(None)
gates.append("trav_clear")
outputs.append(None)
mems.append(start)
ctxs.append(None)
gates.append("trav_load")
outputs.append(None)
for key, end in zip(ctx_keys, traj):
# Decay mem/ptr
mems.append(None)
ctxs.append(key)
gates.append("trav1")
outputs.append(None)
# p<m, decay mem/ptr, p<c
mems.append(None)
ctxs.append(None)
gates.append("trav2")
outputs.append(None)
# m<p, decay mem/ptr, m<c
mems.append(None)
ctxs.append(None)
gates.append("trav3")
outputs.append(None)
for _ in range(self.stabil):
# m<m
mems.append(None)
ctxs.append(None)
gates.append("trav4")
outputs.append(None)
mems.append(None)
ctxs.append(None)
# Clear
gates.append("trav5")
outputs.append(end)
input_iters = {
self.prefix + "m": iter(mems),
self.prefix + "c": iter(ctxs),
self.gate_layer_name: iter(gates)
}
output_iters = {self.prefix + "m": iter(outputs)}
report, test_state = self.execute(
#visualizer=True, args={"refresh rate" : 10, "verbose" : True},
input_iters=input_iters,
output_iters=output_iters)
correct = test_state[self.prefix + "m"]["correct"]
w_correct = test_state[self.prefix + "m"]["w_correct"]
total = test_state[self.prefix + "m"]["total"]
return (float(correct) / total, w_correct / total)
def test(self, mappings):
return {
"trans_acc": self.test_transit(mappings),
"recall_acc": self.test_recovery(mappings)
}