def __init__(self, spinn, origin, termination, transform=None): scale = [nn.scale for nn in termination.node.nodes] if transform is None: transform = termination.transform if origin.node.neurons > spinn.max_fan_in: w = optsparse.compute_sparse_weights(origin, termination.node, transform, spinn.max_fan_in) else: w = MU.prod(termination.node.encoders, MU.prod(transform, MU.transpose(origin.decoders))) w = MU.prod(w, 1.0 / termination.node.radii[0]) for i in range(len(w)): for j in range(len(w[i])): w[i][j] *= scale[i] / termination.tau w = MU.transpose(w) self.weights = w self.tau = int(round(termination.tau * 1000)) if self.tau not in spinn.populations[termination.node].taus: spinn.populations[termination.node].taus.append(self.tau) self.pre = spinn.populations[origin.node].name self.post = spinn.populations[termination.node].name
def compute_sparse_weights(origin, post, transform, fan_in, noise=0.1, num_samples=100): encoder = post.encoders radius = post.radii[0] if hasattr(transform, 'tolist'): transform = transform.tolist() approx = origin.node.getDecodingApproximator('AXON') # create X matrix X = approx.evalPoints X = MU.transpose([f.multiMap(X) for f in origin.functions]) # create A matrix A = approx.values S = fan_in N_A = len(A) samples = len(A[0]) N_B = len(encoder) w_sparse = np.zeros((N_B, N_A), 'f') noise_sd = MU.max(A) * noise decoder_list = [None for _ in range(num_samples)] for i in range(num_samples): indices = random.sample(range(N_A), S) activity = [A[j] for j in indices] n = [[random.gauss(0, noise_sd) for _ in range(samples)] for j in range(S)] activity = MU.sum(activity, n) activityT = MU.transpose(activity) gamma = MU.prod(activity, activityT) upsilon = MU.prod(activity, X) gamma_inv = pinv(gamma, noise_sd * noise_sd) decoder_list[i] = MU.prod([[x for x in row] for row in gamma_inv], upsilon) for i in range(N_B): ww = MU.prod(random.choice(decoder_list), MU.prod(MU.transpose(transform), encoder[i])) for j, k in enumerate(indices): w_sparse[i, k] = float(ww[j]) / radius return list(w_sparse)
def addPlasticTermination(self, name, matrix, tauPSC, decoder, weight_func=None): """Create a new termination. A new termination is created on each of the ensembles, which are then grouped together. If decoders are not known at the time the termination is created, then pass in an array of zeros of the appropriate size (i.e. however many neurons will be in the population projecting to the termination, by number of dimensions).""" terminations = [] d = 0 dd = self._nodes[0].dimension for n in self._nodes: encoder = n.encoders w = MU.prod(encoder, [ MU.prod(matrix, MU.transpose(decoder))[d + i] for i in range(dd) ]) if weight_func is not None: w = weight_func(w) t = n.addPESTermination(name, w, tauPSC, False) terminations.append(t) d += dd termination = EnsembleTermination(self, name, terminations) self.exposeTermination(termination, name) return self.getTermination(name)
def compute_weight_matrix(self, proj): orig=proj.origin term=proj.termination post=term.node transform=term.transform while hasattr(orig,'getWrappedOrigin'): orig=orig.getWrappedOrigin() decoder=orig.getDecoders() encoder=term.node.getEncoders() # scale by radius encoder=MU.prod(encoder,1.0/post.getRadii()[0]) encoder=MU.prod(encoder, self.weight_scale) # scale by gain for i, n in enumerate(post.nodes): for j in range(len(encoder[i])): encoder[i][j]*=n.scale #encoder=MU.prodElementwise(encoder, [n.scale for n in post.nodes]) w=MU.prod(encoder,MU.prod(transform,MU.transpose(decoder))) return w
def __init__(self, spinn, origin, termination, transform = None): scale = [nn.scale for nn in termination.node.nodes] if transform is None: transform = termination.transform if origin.node.neurons>spinn.max_fan_in: w = optsparse.compute_sparse_weights(origin, termination.node, transform, spinn.max_fan_in) else: w = MU.prod(termination.node.encoders,MU.prod(transform,MU.transpose(origin.decoders))) w = MU.prod(w,1.0/termination.node.radii[0]) for i in range(len(w)): for j in range(len(w[i])): w[i][j] *= scale[i] / termination.tau w = MU.transpose(w) self.weights = w self.tau = int(round(termination.tau*1000)) if self.tau not in spinn.populations[termination.node].taus: spinn.populations[termination.node].taus.append(self.tau) self.pre = spinn.populations[origin.node].name self.post = spinn.populations[termination.node].name
def addPlasticTermination(self,name,matrix,tauPSC,decoder,weight_func=None): """Create a new termination. A new termination is created on each of the ensembles, which are then grouped together. If decoders are not known at the time the termination is created, then pass in an array of zeros of the appropriate size (i.e. however many neurons will be in the population projecting to the termination, by number of dimensions).""" terminations = [] d = 0 dd=self._nodes[0].dimension for n in self._nodes: encoder = n.encoders w = MU.prod(encoder,[MU.prod(matrix,MU.transpose(decoder))[d+i] for i in range(dd)]) if weight_func is not None: w = weight_func(w) t = n.addPESTermination(name,w,tauPSC,False) terminations.append(t) d += dd termination = EnsembleTermination(self,name,terminations) self.exposeTermination(termination,name) return self.getTermination(name)
def __init__(self, actions, Qradius=1, noiselevel=0.03): """Builds the BGNetwork. :param actions: actions available to the system :type actions: list of tuples (action_name,action_vector) :param Qradius: expected radius of Q values :param noiselevel: standard deviation of noise added to Q values for exploration """ self.name = "BGNetwork" net = nef.Network(self, seed=HRLutils.SEED, quick=False) self.N = 50 self.d = len(actions) self.mut_inhib = 1.0 # mutual inhibition between actions self.tauPSC = 0.007 # make basal ganglia netbg = nef.Network("bg") bginput = netbg.make("bginput", 1, self.d, mode="direct") bginput.fixMode() bginput.addDecodedTermination("input", MU.diag([1.0 / Qradius for _ in range(self.d)]), 0.001, False) # divide by Q radius to get values back into 0 -- 1 range bgoutput = netbg.make("bgoutput", 1, self.d, mode="direct") bgoutput.fixMode() basalganglia.make_basal_ganglia(netbg, bginput, bgoutput, dimensions=self.d, neurons=200) bg = netbg.network net.add(bg) bg.fixMode([SimulationMode.DEFAULT, SimulationMode.RATE]) bg.exposeTermination(bginput.getTermination("input"), "input") bg.exposeOrigin(bgoutput.getOrigin("X"), "X") # insert noise (used to give some randomness to drive exploration) noiselevel = net.make_input("noiselevel", [noiselevel]) noise = noisenode.NoiseNode(1, dimension=len(actions)) net.add(noise) net.connect(noiselevel, noise.getTermination("scale")) net.connect(noise.getOrigin("noise"), "bg.bginput", pstc=0.001) # add bias to shift everything up to 0.5--1.5 biasinput = net.make_input("biasinput", [0.5]) net.connect(biasinput, "bg.bginput", transform=[[1] for _ in range(self.d)], pstc=0.001) # invert BG output (so the "selected" action will have a positive value # and the rest zero) invert = thalamus.make(net, name="invert", neurons=self.N, dimensions=self.d, useQuick=False) invert.fixMode([SimulationMode.DEFAULT, SimulationMode.RATE]) net.connect(bg, invert.getTermination("bg_input")) # add mutual inhibition net.connect(invert.getOrigin("xBiased"), invert, pstc=self.tauPSC, transform=[[0 if i == j else -self.mut_inhib for j in range(self.d)] for i in range(self.d)]) # threshold output values so that you get a nice clean 0 for # non-selected and 1 for selected threshf = HRLutils.node_fac() threshold = 0.1 threshf.setIntercept(IndicatorPDF(threshold, 1.0)) val_threshold = net.make_array("val_threshold", self.N * 2, self.d, node_factory=threshf, encoders=[[1]]) val_threshold.addDecodedOrigin( "output", [PiecewiseConstantFunction([threshold], [0, 1]) for _ in range(self.d)], "AXON", True) net.connect(invert.getOrigin("xBiased"), val_threshold, pstc=self.tauPSC) # output action (action vectors weighted by BG output) weight_actions = net.make_array("weight_actions", 50, len(actions[0][1]), intercept=(0, 1)) net.connect(val_threshold.getOrigin("output"), weight_actions, transform=MU.transpose([actions[i][1] for i in range(self.d)]), pstc=0.007) # save the BG output (selected action and selected action value) save_relay = net.make("save_relay", 1, 1, mode="direct") save_relay.fixMode() save_relay.addDecodedTermination("input", [[1]], 0.001, False) saved_action = memory.Memory("saved_action", self.N * 2, len(actions[0][1]), inputscale=75) net.add(saved_action) net.connect(weight_actions, saved_action.getTermination("target")) net.connect(save_relay, saved_action.getTermination("transfer")) saved_vals = memory.Memory("saved_values", self.N * 2, self.d, inputscale=75) net.add(saved_vals) net.connect(val_threshold.getOrigin("output"), saved_vals.getTermination("target")) net.connect(save_relay, saved_vals.getTermination("transfer")) # put the saved values through a threshold (we want a nice clean # zero for non-selected values) nfac = HRLutils.node_fac() nfac.setIntercept(IndicatorPDF(0.2, 1)) saved_vals_threshold = net.make_array("saved_vals_threshold", self.N, self.d, node_factory=nfac, encoders=[[1]]) saved_vals_threshold.addDecodedOrigin( "output", [PiecewiseConstantFunction([0.3], [0, 1]) for _ in range(self.d)], "AXON", True) net.connect(saved_vals, saved_vals_threshold, pstc=self.tauPSC) self.exposeTermination(bg.getTermination("input"), "input") self.exposeTermination(save_relay.getTermination("input"), "save_output") self.exposeOrigin(val_threshold.getOrigin("output"), "curr_vals") self.exposeOrigin(weight_actions.getOrigin("X"), "curr_action") self.exposeOrigin(saved_vals_threshold.getOrigin("output"), "saved_vals") self.exposeOrigin(saved_action.getOrigin("X"), "saved_action")
def __init__(self, name, N, stateN, actions, learningrate, Qradius=1.0, init_decoders=None): """Build ActionValues network. :param name: name of Network :param N: base number of neurons :param stateN: number of neurons in state population :param actions: actions available to the system :type actions: list of tuples (action_name,action_vector) :param learningrate: learning rate for PES rule :param Qradius: expected radius of Q values :param init_decoders: if specified, will be used to initialize the connection weights to whatever function is specified by decoders """ self.name = name net = nef.Network(self, seed=HRLutils.SEED, quick=False) self.N = N self.learningrate = learningrate self.supervision = 1.0 # don't use the unsupervised stuff at all self.tauPSC = 0.007 modterms = [] learnterms = [] # relays output = net.make("output", 1, len(actions), mode="direct") output.fixMode() for i, action in enumerate(actions): # create one population corresponding to each action act_pop = net.make("action_" + action[0], self.N * 4, 1, node_factory=HRLutils.node_fac()) act_pop.fixMode([SimulationMode.DEFAULT, SimulationMode.RATE]) # add error termination modterm = act_pop.addDecodedTermination( "error", [[0 if j != i else 1 for j in range(len(actions))]], 0.005, True) # set modulatory transform so that it selects one dimension of # the error signal # create learning termination if init_decoders is not None: weights = MU.prod(act_pop.getEncoders(), MU.transpose(init_decoders)) else: weights = [[ random.uniform(-1e-3, 1e-3) for j in range(stateN) ] for i in range(act_pop.getNeurons())] learningterm = act_pop.addHPESTermination("learning", weights, 0.005, False, None) # initialize the learning rule net.learn(act_pop, learningterm, modterm, rate=self.learningrate, supervisionRatio=self.supervision) # connect each action back to output relay net.connect(act_pop.getOrigin("X"), output, transform=[[0] if j != i else [Qradius] for j in range(len(actions))], pstc=0.001) # note, we learn all the Q values with radius 1, then just # multiply by the desired Q radius here modterms += [modterm] learnterms += [learningterm] # use EnsembleTerminations to group the individual action terminations # into one multi-dimensional termination self.exposeTermination(EnsembleTermination(self, "state", learnterms), "state") self.exposeTermination(EnsembleTermination(self, "error", modterms), "error") self.exposeOrigin(output.getOrigin("X"), "X")
def __init__(self, name, N, stateN, actions, learningrate, Qradius=1.0, init_decoders=None): """Build ActionValues network. :param name: name of Network :param N: base number of neurons :param stateN: number of neurons in state population :param actions: actions available to the system :type actions: list of tuples (action_name,action_vector) :param learningrate: learning rate for PES rule :param Qradius: expected radius of Q values :param init_decoders: if specified, will be used to initialize the connection weights to whatever function is specified by the decoders """ self.name = name net = nef.Network(self, seed=HRLutils.SEED, quick=False) self.N = N self.learningrate = learningrate self.supervision = 1.0 # don't use the unsupervised stuff at all self.tauPSC = 0.007 modterms = [] learnterms = [] # relays output = net.make("output", 1, len(actions), mode="direct") output.fixMode() for i, action in enumerate(actions): # create one population corresponding to each action act_pop = net.make("action_" + action[0], self.N * 4, 1, node_factory=HRLutils.node_fac()) act_pop.fixMode([SimulationMode.DEFAULT, SimulationMode.RATE]) # add error termination modterm = act_pop.addDecodedTermination("error", [[0 if j != i else 1 for j in range(len(actions))]], 0.005, True) # set modulatory transform so that it selects one dimension of the error signal # create learning termination if init_decoders != None: weights = MU.prod(act_pop.getEncoders(), MU.transpose(init_decoders)) else: weights = [[random.uniform(-1e-3, 1e-3) for j in range(stateN)] for i in range(act_pop.getNeurons())] learningterm = act_pop.addHPESTermination("learning", weights, 0.005, False, None) # initialize the learning rule net.learn(act_pop, learningterm, modterm, rate=self.learningrate, supervisionRatio=self.supervision) # connect each action back to output relay net.connect(act_pop.getOrigin("X"), output, transform=[[0] if j != i else [Qradius] for j in range(len(actions))], pstc=0.001) # note, we learn all the Q values with radius 1, then just multiply by the desired Q radius here modterms += [modterm] learnterms += [learningterm] # use EnsembleTerminations to group the individual action terminations into one multi-dimensional termination self.exposeTermination(EnsembleTermination(self, "state", learnterms), "state") self.exposeTermination(EnsembleTermination(self, "error", modterms), "error") self.exposeOrigin(output.getOrigin("X"), "X")
def make(net, preName='pre', postName='post', rate=5e-4): # get pre and post ensembles from their names pre = net.network.getNode(preName) post = net.network.getNode(postName) dim_pre = pre.getDimension() dim_post = post.getDimension() t = [[0] * dim_pre for i in range(dim_post)] index_pre = range(dim_pre) index_post = range(dim_post) for i in range(max(len(index_pre),len(index_post))): ipre = index_pre[i % len(index_pre)] ipost = index_post[i % len(index_post)] t[ipost][ipre] = 1 decoder = pre.getOrigin('X').getDecoders() encoder = post.getEncoders() encoder = MU.prod(encoder, 1.0 / post.getRadii()[0]) weight = MU.prod(encoder, MU.prod(t, MU.transpose(decoder))) # random weight matrix to initialize projection from pre to post # def rand_weights(w): # for i in range(len(w)): # for j in range(len(w[0])): # w[i][j] = random.uniform(-1e-3,1e-3) # return w # weight = rand_weights(numeric.zeros((post.neurons, pre.neurons)).tolist()) # non-decoded termination (to learn transformation) count = 0 prename = pre.getName() while '%s_%02d' % (prename, count) in [t.name for t in post.terminations]: count = count + 1 prename = '%s_%02d' % (prename, count) post.addBCMTermination(prename, weight, 0.005, False, None) # Add projections net.connect(pre.getOrigin('AXON'),post.getTermination(prename)) # Set learning rule on the non-decoded termination net.learn(post,prename,None,rate=rate) if net.network.getMetaData("bcmterm") == None: net.network.setMetaData("bcmterm", HashMap()) bcmterms = net.network.getMetaData("bcmterm") bcmterm = HashMap(4) bcmterm.put("preName", preName) bcmterm.put("postName", postName) bcmterm.put("rate", rate) bcmterms.put(prename, bcmterm) if net.network.getMetaData("templates") == None: net.network.setMetaData("templates", ArrayList()) templates = net.network.getMetaData("templates") templates.add(prename) if net.network.getMetaData("templateProjections") == None: net.network.setMetaData("templateProjections", HashMap()) templateproj = net.network.getMetaData("templateProjections") templateproj.put(preName, postName)
def make(net, preName='pre', postName='post', rate=5e-4): # get pre and post ensembles from their names pre = net.network.getNode(preName) post = net.network.getNode(postName) dim_pre = pre.getDimension() dim_post = post.getDimension() t = [[0] * dim_pre for i in range(dim_post)] index_pre = range(dim_pre) index_post = range(dim_post) for i in range(max(len(index_pre), len(index_post))): ipre = index_pre[i % len(index_pre)] ipost = index_post[i % len(index_post)] t[ipost][ipre] = 1 decoder = pre.getOrigin('X').getDecoders() encoder = post.getEncoders() encoder = MU.prod(encoder, 1.0 / post.getRadii()[0]) weight = MU.prod(encoder, MU.prod(t, MU.transpose(decoder))) # random weight matrix to initialize projection from pre to post # def rand_weights(w): # for i in range(len(w)): # for j in range(len(w[0])): # w[i][j] = random.uniform(-1e-3,1e-3) # return w # weight = rand_weights(numeric.zeros((post.neurons, pre.neurons)).tolist()) # non-decoded termination (to learn transformation) count = 0 prename = pre.getName() while '%s_%02d' % (prename, count) in [t.name for t in post.terminations]: count = count + 1 prename = '%s_%02d' % (prename, count) post.addBCMTermination(prename, weight, 0.005, False, None) # Add projections net.connect(pre.getOrigin('AXON'), post.getTermination(prename)) # Set learning rule on the non-decoded termination net.learn(post, prename, None, rate=rate) if net.network.getMetaData("bcmterm") == None: net.network.setMetaData("bcmterm", HashMap()) bcmterms = net.network.getMetaData("bcmterm") bcmterm = HashMap(4) bcmterm.put("preName", preName) bcmterm.put("postName", postName) bcmterm.put("rate", rate) bcmterms.put(prename, bcmterm) if net.network.getMetaData("templates") == None: net.network.setMetaData("templates", ArrayList()) templates = net.network.getMetaData("templates") templates.add(prename) if net.network.getMetaData("templateProjections") == None: net.network.setMetaData("templateProjections", HashMap()) templateproj = net.network.getMetaData("templateProjections") templateproj.put(preName, postName)