def simpleRecall(): network = Network(5, 4) memoryLayer = network.createLayer(4, name="memory") inputGateLayer = network.createLayer(4, name="inputGate", bias=True) forgetGateLayer = network.createLayer(4, name="forgetGate", bias=True) outputGateLayer = network.createLayer(4, name="outputGate", bias=True) network.connectLayers(network.inputLayer, memoryLayer, interconnected=True) network.connectLayers(network.inputLayer, inputGateLayer, interconnected=True) network.connectLayers(network.inputLayer, forgetGateLayer, interconnected=True) network.connectLayers(network.inputLayer, outputGateLayer, interconnected=True) network.connectLayers(memoryLayer, inputGateLayer, recurrent=True, interconnected=True) network.connectLayers(memoryLayer, forgetGateLayer, recurrent=True, interconnected=True) network.connectLayers(memoryLayer, outputGateLayer, recurrent=True, interconnected=True) network.connectLayers(memoryLayer, network.outputLayer, interconnected=False) network.addSelfRecurrence(memoryLayer, forgetGateLayer) network.gateIncomingConnections(network.outputLayer, outputGateLayer) network.gateIncomingConnections(memoryLayer, inputGateLayer) printLayer(network.inputLayer) printLayer(inputGateLayer) printLayer(memoryLayer) printLayer(outputGateLayer) printLayer(network.outputLayer) inputSet = [] # Create set of input sequences. for _ in xrange(200): length = 15 # Fill 10 blank vectors. inputSequence = [[0.0 for _ in xrange(5)] for __ in xrange(length)] # Add one relevant vector. relevant = [0.0, 0.0, 0.0, 0.0, 0.0] relevant[random.randint(0, 3)] = 1.0 inputSequence.append(relevant) random.shuffle(inputSequence) # Add prompt inputSequence.append([0.0, 0.0, 0.0, 0.0, 1.0]) # Create output sequence. outputSequence = [[0.0] * 4] * (length + 1) #outputSequence = [None] * 11 expected = [0.0, 0.0, 0.0, 0.0] expected[relevant.index(1.0)] = 1.0 outputSequence.append(expected) inputSet.append((inputSequence, outputSequence)) testSet = inputSet[:20] inputSet = inputSet[20:] def calcDiff(x,y): diff = 0.0 zipped = zip(x,y) for actual,expected in zipped[:-1]: for a in actual: if a > 0.5: diff += a actual,expected = zipped[-1] for a,e in zip(actual, expected): if e == 1.0 and a < 0.5: #print(a, " != 1") diff += 1.0 - a elif e == 0.0 and a > 0.5: #print(a, " != 0") diff += a return diff beforeDiff = calculateDiff(network, testSet, calcDiff, printValues=False) print("Before diff: %f" % beforeDiff) #for _ in xrange(100): iterations = 0 while True: print("Iteration %d" % iterations) iterations += 1 for inputSequence, outputSequence in inputSet: #print(".") output = network.learn(inputSequence, outputSequence) diff = calculateDiff(network, testSet, calcDiff, printValues=False) print(diff) if diff < 1: break afterDiff = calculateDiff(network, testSet, calcDiff, printValues=False) print("Before diff: %f" % beforeDiff) print("After diff: %f" % afterDiff)
def forgetTest(debug=False): """ Tests to see if a network can remember a value and then forget it. Input sequences contain one value and two signals. The first signal indicates that the value should be let into the memory cell and remembered. The second signal indicates that the previously remembered value should be forgotten. (output) | (memory) ) -- (forget) | | x ---- (gate) | | | | (input) (input2) (input3) """ network = Network(3,1) gateLayer = network.createLayer(1, name="gate", activationFunction=activation.TanH) forgetLayer = network.createLayer(1, name="forget", bias=True) memoryLayer = network.createLayer(1, name="memory", activationFunction=activation.Identity) network.connectLayers(network.inputLayer[:1], memoryLayer) network.connectLayers(network.inputLayer[1:2], gateLayer) network.connectLayers(network.inputLayer[2:3], forgetLayer) network.connectLayers(memoryLayer, network.outputLayer) network.gateIncomingConnections(memoryLayer, gateLayer) network.addSelfRecurrence(memoryLayer, forgetLayer) printLayer(network.inputLayer) printLayer(memoryLayer) printLayer(gateLayer) printLayer(forgetLayer) printLayer(network.outputLayer) def genData(): length = 50 start = random.randint(0, length-2) end = 0 while end <= start: end = random.randint(1, length-1) sequence = [[random.random() - 0.5, 1.0 if _ == start else 0.0, 1.0 if _ == end else 0.0] for _ in xrange(length)] return sequence def genExpected(sequence): expected = [] toRemember = 0.0 for val,remember,forget in sequence: if remember == 1.0: toRemember = val if forget == 1.0: toRemember = 0.0 expected.append([toRemember]) return expected def calcDiff(x,y): return sum([abs(a[0]-b[0]) for a,b in zip(x,y)]) runTest(network, genData, genExpected, calcDiff, 100, 10, 100, debug=debug) printLayer(network.inputLayer) printLayer(memoryLayer) printLayer(gateLayer) printLayer(forgetLayer) printLayer(network.outputLayer)
def memoryTest(debug=False): """ Tests to see if a network can remember a value. Input sequences contain one value and one signals. The signal indicates that the value should be let into the memory cell and remembered. (output) | (memory) ) | x ---- (gate) | | (input) (input2) """ network = Network(2,1) gateLayer = network.createLayer(1, name="gate", activationFunction=activation.TanH) memoryLayer = network.createLayer(1, name="memory", activationFunction=activation.Identity) network.connectLayers(network.inputLayer[:1], memoryLayer) network.connectLayers(network.inputLayer[1:2], gateLayer) network.connectLayers(memoryLayer, network.outputLayer) network.gateIncomingConnections(memoryLayer, gateLayer) network.addSelfRecurrence(memoryLayer) printLayer(network.inputLayer) printLayer(memoryLayer) printLayer(gateLayer) printLayer(network.outputLayer) def genData(): sequence = [[random.random() - 0.5, 0.0] for _ in xrange(20)] if random.random() < 0.8: sequence.append([random.random() - 0.5, 1.0]) else: sequence.append([random.random() - 0.5, 0.0]) random.shuffle(sequence) return sequence def genExpected(sequence): val = None index = None for i, s in enumerate(sequence): if s[1] == 1.0: val = s[0] index = i expected = [[0.0] for _ in xrange(len(sequence))] if val is not None: for i in xrange(len(sequence) - index): expected[i + index] = [val] return expected def calcDiff(x,y): return sum([abs(a[0]-b[0]) if b is not None else 0.0 for a,b in zip(x,y)]) runTest(network, genData, genExpected, calcDiff, 100, 10, 100, debug=debug) printLayer(network.inputLayer) printLayer(memoryLayer) printLayer(gateLayer) printLayer(network.outputLayer)
def distractedRecall(): network = Network(10, 4) memoryLayer = network.createLayer(8, name="memory") inputGateLayer = network.createLayer(8, name="inputGate", bias=True) forgetGateLayer = network.createLayer(8, name="forgetGate", bias=True) outputGateLayer = network.createLayer(8, name="outputGate", bias=True) network.connectLayers(network.inputLayer, memoryLayer, interconnected=True) network.connectLayers(network.inputLayer, inputGateLayer, interconnected=True) network.connectLayers(network.inputLayer, forgetGateLayer, interconnected=True) network.connectLayers(network.inputLayer, outputGateLayer, interconnected=True) network.connectLayers(memoryLayer, inputGateLayer, recurrent=True, interconnected=True) network.connectLayers(memoryLayer, forgetGateLayer, recurrent=True, interconnected=True) network.connectLayers(memoryLayer, outputGateLayer, recurrent=True, interconnected=True) network.connectLayers(memoryLayer, network.outputLayer, interconnected=True) network.addSelfRecurrence(memoryLayer, forgetGateLayer) network.gateOutgoingConnections(memoryLayer, outputGateLayer) network.gateIncomingConnections(memoryLayer, inputGateLayer) inputSet = [] # Create set of input sequences. for _ in xrange(100): length = 10 # Create input sequence. inputSequence = [] # Fill random distractors. for _ in xrange(length): inputVector = [0.0 for _ in xrange(10)] inputVector[random.randint(4,7)] = 1 inputSequence.append(inputVector) # Select two random vectors and inject random target symbols. firstIndex = random.randint(0,length-1) secondIndex = firstIndex while secondIndex == firstIndex: secondIndex = random.randint(0,length-1) if firstIndex > secondIndex: temp = firstIndex firstIndex = secondIndex secondIndex = temp firstTarget = random.randint(0,3) secondTarget = random.randint(0,3) #print(firstIndex, secondIndex) #print(firstTarget, secondTarget) inputSequence[firstIndex] = [1.0 if x == firstTarget else 0.0 for x in xrange(10)] inputSequence[secondIndex] = [1.0 if x == secondTarget else 0.0 for x in xrange(10)] # Add prompts inputSequence.append([1.0 if x == 8 else 0.0 for x in xrange(10)]) inputSequence.append([1.0 if x == 9 else 0.0 for x in xrange(10)]) outputSequence = [[0] * 4] * length #outputSequence = [None] * length outputSequence.append([1 if x == firstTarget else 0 for x in xrange(4)]) outputSequence.append([1 if x == secondTarget else 0 for x in xrange(4)]) #for line in inputSequence: print(line) #for line in outputSequence: print(line) #sys.exit() inputSet.append((inputSequence, outputSequence)) testSet = inputSet[:10] inputSet = inputSet[10:] def calcDiff(x,y): diff = 0.0 zipped = zip(x,y) for actual,expected in zipped[:-1]: for a in actual: if a > 0.5: diff += a actual,expected = zipped[-1] for a,e in zip(actual, expected): if e == 1.0 and a < 0.5: print(a, " != 1") diff += 1.0 - a elif e == 0 and a > 0.5: print(a, " != 0") diff += a return diff beforeDiff = calculateDiff(network, testSet, calcDiff, printValues=False) print("Before diff: %f" % beforeDiff) #for _ in xrange(10): iterations = 0 while True: print("Iteration %d" % iterations) iterations += 1 for inputSequence, outputSequence in inputSet: #print(".") output = network.learn(inputSequence, outputSequence) diff = calculateDiff(network, testSet, calcDiff, printValues=False) print(diff) if diff < 1: break afterDiff = calculateDiff(network, testSet, calcDiff, printValues=True) print("Before diff: %f" % beforeDiff) print("After diff: %f" % afterDiff)