def __init__(self, spinn, origin, termination, transform=None):
scale = [nn.scale for nn in termination.node.nodes]
if transform is None:
transform = termination.transform
if origin.node.neurons > spinn.max_fan_in:
w = optsparse.compute_sparse_weights(origin, termination.node,
transform, spinn.max_fan_in)
else:
w = MU.prod(termination.node.encoders,
MU.prod(transform, MU.transpose(origin.decoders)))
w = MU.prod(w, 1.0 / termination.node.radii[0])
for i in range(len(w)):
for j in range(len(w[i])):
w[i][j] *= scale[i] / termination.tau
w = MU.transpose(w)
self.weights = w
self.tau = int(round(termination.tau * 1000))
if self.tau not in spinn.populations[termination.node].taus:
spinn.populations[termination.node].taus.append(self.tau)
self.pre = spinn.populations[origin.node].name
self.post = spinn.populations[termination.node].name
def compute_sparse_weights(origin,
post,
transform,
fan_in,
noise=0.1,
num_samples=100):
encoder = post.encoders
radius = post.radii[0]
if hasattr(transform, 'tolist'): transform = transform.tolist()
approx = origin.node.getDecodingApproximator('AXON')
# create X matrix
X = approx.evalPoints
X = MU.transpose([f.multiMap(X) for f in origin.functions])
# create A matrix
A = approx.values
S = fan_in
N_A = len(A)
samples = len(A[0])
N_B = len(encoder)
w_sparse = np.zeros((N_B, N_A), 'f')
noise_sd = MU.max(A) * noise
decoder_list = [None for _ in range(num_samples)]
for i in range(num_samples):
indices = random.sample(range(N_A), S)
activity = [A[j] for j in indices]
n = [[random.gauss(0, noise_sd) for _ in range(samples)]
for j in range(S)]
activity = MU.sum(activity, n)
activityT = MU.transpose(activity)
gamma = MU.prod(activity, activityT)
upsilon = MU.prod(activity, X)
gamma_inv = pinv(gamma, noise_sd * noise_sd)
decoder_list[i] = MU.prod([[x for x in row] for row in gamma_inv],
upsilon)
for i in range(N_B):
ww = MU.prod(random.choice(decoder_list),
MU.prod(MU.transpose(transform), encoder[i]))
for j, k in enumerate(indices):
w_sparse[i, k] = float(ww[j]) / radius
return list(w_sparse)
def addPlasticTermination(self,
name,
matrix,
tauPSC,
decoder,
weight_func=None):
"""Create a new termination. A new termination is created on each
of the ensembles, which are then grouped together.
If decoders are not known at the time the termination is created,
then pass in an array of zeros of the appropriate size (i.e. however
many neurons will be in the population projecting to the termination,
by number of dimensions)."""
terminations = []
d = 0
dd = self._nodes[0].dimension
for n in self._nodes:
encoder = n.encoders
w = MU.prod(encoder, [
MU.prod(matrix, MU.transpose(decoder))[d + i]
for i in range(dd)
])
if weight_func is not None:
w = weight_func(w)
t = n.addPESTermination(name, w, tauPSC, False)
terminations.append(t)
d += dd
termination = EnsembleTermination(self, name, terminations)
self.exposeTermination(termination, name)
return self.getTermination(name)
def compute_weight_matrix(self, proj):
orig=proj.origin
term=proj.termination
post=term.node
transform=term.transform
while hasattr(orig,'getWrappedOrigin'): orig=orig.getWrappedOrigin()
decoder=orig.getDecoders()
encoder=term.node.getEncoders()
# scale by radius
encoder=MU.prod(encoder,1.0/post.getRadii()[0])
encoder=MU.prod(encoder, self.weight_scale)
# scale by gain
for i, n in enumerate(post.nodes):
for j in range(len(encoder[i])):
encoder[i][j]*=n.scale
#encoder=MU.prodElementwise(encoder, [n.scale for n in post.nodes])
w=MU.prod(encoder,MU.prod(transform,MU.transpose(decoder)))
return w
def __init__(self, spinn, origin, termination, transform = None):
scale = [nn.scale for nn in termination.node.nodes]
if transform is None:
transform = termination.transform
if origin.node.neurons>spinn.max_fan_in:
w = optsparse.compute_sparse_weights(origin, termination.node, transform, spinn.max_fan_in)
else:
w = MU.prod(termination.node.encoders,MU.prod(transform,MU.transpose(origin.decoders)))
w = MU.prod(w,1.0/termination.node.radii[0])
for i in range(len(w)):
for j in range(len(w[i])):
w[i][j] *= scale[i] / termination.tau
w = MU.transpose(w)
self.weights = w
self.tau = int(round(termination.tau*1000))
if self.tau not in spinn.populations[termination.node].taus:
spinn.populations[termination.node].taus.append(self.tau)
self.pre = spinn.populations[origin.node].name
self.post = spinn.populations[termination.node].name
def addPlasticTermination(self,name,matrix,tauPSC,decoder,weight_func=None):
"""Create a new termination. A new termination is created on each
of the ensembles, which are then grouped together.
If decoders are not known at the time the termination is created,
then pass in an array of zeros of the appropriate size (i.e. however
many neurons will be in the population projecting to the termination,
by number of dimensions)."""
terminations = []
d = 0
dd=self._nodes[0].dimension
for n in self._nodes:
encoder = n.encoders
w = MU.prod(encoder,[MU.prod(matrix,MU.transpose(decoder))[d+i] for i in range(dd)])
if weight_func is not None:
w = weight_func(w)
t = n.addPESTermination(name,w,tauPSC,False)
terminations.append(t)
d += dd
termination = EnsembleTermination(self,name,terminations)
self.exposeTermination(termination,name)
return self.getTermination(name)
def __init__(self, actions, Qradius=1, noiselevel=0.03):
"""Builds the BGNetwork.
:param actions: actions available to the system
:type actions: list of tuples (action_name,action_vector)
:param Qradius: expected radius of Q values
:param noiselevel: standard deviation of noise added to Q values for
exploration
"""
self.name = "BGNetwork"
net = nef.Network(self, seed=HRLutils.SEED, quick=False)
self.N = 50
self.d = len(actions)
self.mut_inhib = 1.0 # mutual inhibition between actions
self.tauPSC = 0.007
# make basal ganglia
netbg = nef.Network("bg")
bginput = netbg.make("bginput", 1, self.d, mode="direct")
bginput.fixMode()
bginput.addDecodedTermination("input",
MU.diag([1.0 / Qradius for _ in
range(self.d)]), 0.001, False)
# divide by Q radius to get values back into 0 -- 1 range
bgoutput = netbg.make("bgoutput", 1, self.d, mode="direct")
bgoutput.fixMode()
basalganglia.make_basal_ganglia(netbg, bginput, bgoutput,
dimensions=self.d, neurons=200)
bg = netbg.network
net.add(bg)
bg.fixMode([SimulationMode.DEFAULT, SimulationMode.RATE])
bg.exposeTermination(bginput.getTermination("input"), "input")
bg.exposeOrigin(bgoutput.getOrigin("X"), "X")
# insert noise (used to give some randomness to drive exploration)
noiselevel = net.make_input("noiselevel", [noiselevel])
noise = noisenode.NoiseNode(1, dimension=len(actions))
net.add(noise)
net.connect(noiselevel, noise.getTermination("scale"))
net.connect(noise.getOrigin("noise"), "bg.bginput", pstc=0.001)
# add bias to shift everything up to 0.5--1.5
biasinput = net.make_input("biasinput", [0.5])
net.connect(biasinput, "bg.bginput",
transform=[[1] for _ in range(self.d)], pstc=0.001)
# invert BG output (so the "selected" action will have a positive value
# and the rest zero)
invert = thalamus.make(net, name="invert", neurons=self.N,
dimensions=self.d, useQuick=False)
invert.fixMode([SimulationMode.DEFAULT, SimulationMode.RATE])
net.connect(bg, invert.getTermination("bg_input"))
# add mutual inhibition
net.connect(invert.getOrigin("xBiased"), invert, pstc=self.tauPSC,
transform=[[0 if i == j else -self.mut_inhib
for j in range(self.d)]
for i in range(self.d)])
# threshold output values so that you get a nice clean 0 for
# non-selected and 1 for selected
threshf = HRLutils.node_fac()
threshold = 0.1
threshf.setIntercept(IndicatorPDF(threshold, 1.0))
val_threshold = net.make_array("val_threshold", self.N * 2, self.d,
node_factory=threshf, encoders=[[1]])
val_threshold.addDecodedOrigin(
"output",
[PiecewiseConstantFunction([threshold], [0, 1])
for _ in range(self.d)], "AXON", True)
net.connect(invert.getOrigin("xBiased"), val_threshold,
pstc=self.tauPSC)
# output action (action vectors weighted by BG output)
weight_actions = net.make_array("weight_actions", 50,
len(actions[0][1]), intercept=(0, 1))
net.connect(val_threshold.getOrigin("output"), weight_actions,
transform=MU.transpose([actions[i][1]
for i in range(self.d)]),
pstc=0.007)
# save the BG output (selected action and selected action value)
save_relay = net.make("save_relay", 1, 1, mode="direct")
save_relay.fixMode()
save_relay.addDecodedTermination("input", [[1]], 0.001, False)
saved_action = memory.Memory("saved_action", self.N * 2,
len(actions[0][1]), inputscale=75)
net.add(saved_action)
net.connect(weight_actions, saved_action.getTermination("target"))
net.connect(save_relay, saved_action.getTermination("transfer"))
saved_vals = memory.Memory("saved_values", self.N * 2, self.d,
inputscale=75)
net.add(saved_vals)
net.connect(val_threshold.getOrigin("output"),
saved_vals.getTermination("target"))
net.connect(save_relay, saved_vals.getTermination("transfer"))
# put the saved values through a threshold (we want a nice clean
# zero for non-selected values)
nfac = HRLutils.node_fac()
nfac.setIntercept(IndicatorPDF(0.2, 1))
saved_vals_threshold = net.make_array("saved_vals_threshold", self.N,
self.d, node_factory=nfac,
encoders=[[1]])
saved_vals_threshold.addDecodedOrigin(
"output", [PiecewiseConstantFunction([0.3], [0, 1])
for _ in range(self.d)], "AXON", True)
net.connect(saved_vals, saved_vals_threshold, pstc=self.tauPSC)
self.exposeTermination(bg.getTermination("input"), "input")
self.exposeTermination(save_relay.getTermination("input"),
"save_output")
self.exposeOrigin(val_threshold.getOrigin("output"), "curr_vals")
self.exposeOrigin(weight_actions.getOrigin("X"), "curr_action")
self.exposeOrigin(saved_vals_threshold.getOrigin("output"),
"saved_vals")
self.exposeOrigin(saved_action.getOrigin("X"), "saved_action")
def __init__(self,
name,
N,
stateN,
actions,
learningrate,
Qradius=1.0,
init_decoders=None):
"""Build ActionValues network.
:param name: name of Network
:param N: base number of neurons
:param stateN: number of neurons in state population
:param actions: actions available to the system
:type actions: list of tuples (action_name,action_vector)
:param learningrate: learning rate for PES rule
:param Qradius: expected radius of Q values
:param init_decoders: if specified, will be used to initialize the
connection weights to whatever function is specified by decoders
"""
self.name = name
net = nef.Network(self, seed=HRLutils.SEED, quick=False)
self.N = N
self.learningrate = learningrate
self.supervision = 1.0 # don't use the unsupervised stuff at all
self.tauPSC = 0.007
modterms = []
learnterms = []
# relays
output = net.make("output", 1, len(actions), mode="direct")
output.fixMode()
for i, action in enumerate(actions):
# create one population corresponding to each action
act_pop = net.make("action_" + action[0],
self.N * 4,
1,
node_factory=HRLutils.node_fac())
act_pop.fixMode([SimulationMode.DEFAULT, SimulationMode.RATE])
# add error termination
modterm = act_pop.addDecodedTermination(
"error", [[0 if j != i else 1 for j in range(len(actions))]],
0.005, True)
# set modulatory transform so that it selects one dimension of
# the error signal
# create learning termination
if init_decoders is not None:
weights = MU.prod(act_pop.getEncoders(),
MU.transpose(init_decoders))
else:
weights = [[
random.uniform(-1e-3, 1e-3) for j in range(stateN)
] for i in range(act_pop.getNeurons())]
learningterm = act_pop.addHPESTermination("learning", weights,
0.005, False, None)
# initialize the learning rule
net.learn(act_pop,
learningterm,
modterm,
rate=self.learningrate,
supervisionRatio=self.supervision)
# connect each action back to output relay
net.connect(act_pop.getOrigin("X"),
output,
transform=[[0] if j != i else [Qradius]
for j in range(len(actions))],
pstc=0.001)
# note, we learn all the Q values with radius 1, then just
# multiply by the desired Q radius here
modterms += [modterm]
learnterms += [learningterm]
# use EnsembleTerminations to group the individual action terminations
# into one multi-dimensional termination
self.exposeTermination(EnsembleTermination(self, "state", learnterms),
"state")
self.exposeTermination(EnsembleTermination(self, "error", modterms),
"error")
self.exposeOrigin(output.getOrigin("X"), "X")
def __init__(self, name, N, stateN, actions, learningrate, Qradius=1.0, init_decoders=None):
"""Build ActionValues network.
:param name: name of Network
:param N: base number of neurons
:param stateN: number of neurons in state population
:param actions: actions available to the system
:type actions: list of tuples (action_name,action_vector)
:param learningrate: learning rate for PES rule
:param Qradius: expected radius of Q values
:param init_decoders: if specified, will be used to initialize the connection
weights to whatever function is specified by the decoders
"""
self.name = name
net = nef.Network(self, seed=HRLutils.SEED, quick=False)
self.N = N
self.learningrate = learningrate
self.supervision = 1.0 # don't use the unsupervised stuff at all
self.tauPSC = 0.007
modterms = []
learnterms = []
# relays
output = net.make("output", 1, len(actions), mode="direct")
output.fixMode()
for i, action in enumerate(actions):
# create one population corresponding to each action
act_pop = net.make("action_" + action[0], self.N * 4, 1, node_factory=HRLutils.node_fac())
act_pop.fixMode([SimulationMode.DEFAULT, SimulationMode.RATE])
# add error termination
modterm = act_pop.addDecodedTermination("error", [[0 if j != i else 1 for j in range(len(actions))]],
0.005, True)
# set modulatory transform so that it selects one dimension of the error signal
# create learning termination
if init_decoders != None:
weights = MU.prod(act_pop.getEncoders(), MU.transpose(init_decoders))
else:
weights = [[random.uniform(-1e-3, 1e-3) for j in range(stateN)] for i in range(act_pop.getNeurons())]
learningterm = act_pop.addHPESTermination("learning", weights, 0.005, False, None)
# initialize the learning rule
net.learn(act_pop, learningterm, modterm, rate=self.learningrate, supervisionRatio=self.supervision)
# connect each action back to output relay
net.connect(act_pop.getOrigin("X"), output, transform=[[0] if j != i else [Qradius] for j in range(len(actions))],
pstc=0.001)
# note, we learn all the Q values with radius 1, then just multiply by the desired Q radius here
modterms += [modterm]
learnterms += [learningterm]
# use EnsembleTerminations to group the individual action terminations into one multi-dimensional termination
self.exposeTermination(EnsembleTermination(self, "state", learnterms), "state")
self.exposeTermination(EnsembleTermination(self, "error", modterms), "error")
self.exposeOrigin(output.getOrigin("X"), "X")
def make(net, preName='pre', postName='post', rate=5e-4):
# get pre and post ensembles from their names
pre = net.network.getNode(preName)
post = net.network.getNode(postName)
dim_pre = pre.getDimension()
dim_post = post.getDimension()
t = [[0] * dim_pre for i in range(dim_post)]
index_pre = range(dim_pre)
index_post = range(dim_post)
for i in range(max(len(index_pre),len(index_post))):
ipre = index_pre[i % len(index_pre)]
ipost = index_post[i % len(index_post)]
t[ipost][ipre] = 1
decoder = pre.getOrigin('X').getDecoders()
encoder = post.getEncoders()
encoder = MU.prod(encoder, 1.0 / post.getRadii()[0])
weight = MU.prod(encoder, MU.prod(t, MU.transpose(decoder)))
# random weight matrix to initialize projection from pre to post
# def rand_weights(w):
# for i in range(len(w)):
# for j in range(len(w[0])):
# w[i][j] = random.uniform(-1e-3,1e-3)
# return w
# weight = rand_weights(numeric.zeros((post.neurons, pre.neurons)).tolist())
# non-decoded termination (to learn transformation)
count = 0
prename = pre.getName()
while '%s_%02d' % (prename, count) in [t.name for t in post.terminations]:
count = count + 1
prename = '%s_%02d' % (prename, count)
post.addBCMTermination(prename, weight, 0.005, False, None)
# Add projections
net.connect(pre.getOrigin('AXON'),post.getTermination(prename))
# Set learning rule on the non-decoded termination
net.learn(post,prename,None,rate=rate)
if net.network.getMetaData("bcmterm") == None:
net.network.setMetaData("bcmterm", HashMap())
bcmterms = net.network.getMetaData("bcmterm")
bcmterm = HashMap(4)
bcmterm.put("preName", preName)
bcmterm.put("postName", postName)
bcmterm.put("rate", rate)
bcmterms.put(prename, bcmterm)
if net.network.getMetaData("templates") == None:
net.network.setMetaData("templates", ArrayList())
templates = net.network.getMetaData("templates")
templates.add(prename)
if net.network.getMetaData("templateProjections") == None:
net.network.setMetaData("templateProjections", HashMap())
templateproj = net.network.getMetaData("templateProjections")
templateproj.put(preName, postName)
def make(net, preName='pre', postName='post', rate=5e-4):
# get pre and post ensembles from their names
pre = net.network.getNode(preName)
post = net.network.getNode(postName)
dim_pre = pre.getDimension()
dim_post = post.getDimension()
t = [[0] * dim_pre for i in range(dim_post)]
index_pre = range(dim_pre)
index_post = range(dim_post)
for i in range(max(len(index_pre), len(index_post))):
ipre = index_pre[i % len(index_pre)]
ipost = index_post[i % len(index_post)]
t[ipost][ipre] = 1
decoder = pre.getOrigin('X').getDecoders()
encoder = post.getEncoders()
encoder = MU.prod(encoder, 1.0 / post.getRadii()[0])
weight = MU.prod(encoder, MU.prod(t, MU.transpose(decoder)))
# random weight matrix to initialize projection from pre to post
# def rand_weights(w):
# for i in range(len(w)):
# for j in range(len(w[0])):
# w[i][j] = random.uniform(-1e-3,1e-3)
# return w
# weight = rand_weights(numeric.zeros((post.neurons, pre.neurons)).tolist())
# non-decoded termination (to learn transformation)
count = 0
prename = pre.getName()
while '%s_%02d' % (prename, count) in [t.name for t in post.terminations]:
count = count + 1
prename = '%s_%02d' % (prename, count)
post.addBCMTermination(prename, weight, 0.005, False, None)
# Add projections
net.connect(pre.getOrigin('AXON'), post.getTermination(prename))
# Set learning rule on the non-decoded termination
net.learn(post, prename, None, rate=rate)
if net.network.getMetaData("bcmterm") == None:
net.network.setMetaData("bcmterm", HashMap())
bcmterms = net.network.getMetaData("bcmterm")
bcmterm = HashMap(4)
bcmterm.put("preName", preName)
bcmterm.put("postName", postName)
bcmterm.put("rate", rate)
bcmterms.put(prename, bcmterm)
if net.network.getMetaData("templates") == None:
net.network.setMetaData("templates", ArrayList())
templates = net.network.getMetaData("templates")
templates.add(prename)
if net.network.getMetaData("templateProjections") == None:
net.network.setMetaData("templateProjections", HashMap())
templateproj = net.network.getMetaData("templateProjections")
templateproj.put(preName, postName)
