def filterTest(debug=False):
"""
Tests to see if a network can filter negative numbers using a gating mechanism.
(output)
|
x - (gate)
| /
(in)
"""
network = Network(1,1)
gateLayer = network.createLayer(1, name="gate")
network.connectLayers(network.inputLayer, network.outputLayer, interconnected=True)
network.connectLayers(network.inputLayer, gateLayer, interconnected=True)
network.gateIncomingConnections(network.outputLayer, gateLayer)
printLayer(network.inputLayer)
printLayer(gateLayer)
printLayer(network.outputLayer)
def genData():
return [[2 * (random.random() - 0.5)] for _ in xrange(20)]
def genExpected(x):
return [val if val[0] > 0 else [0.0] for val in x]
def calcDiff(x,y):
return sum([abs(a[0]-b[0]) for a,b in zip(x,y)])
runTest(network, genData, genExpected, calcDiff, 100, 10, 200, debug=debug)
printLayer(network.inputLayer)
printLayer(gateLayer)
printLayer(network.outputLayer)
print("=" * 80)
def simpleRecall():
network = Network(5, 4)
memoryLayer = network.createLayer(4, name="memory")
inputGateLayer = network.createLayer(4, name="inputGate", bias=True)
forgetGateLayer = network.createLayer(4, name="forgetGate", bias=True)
outputGateLayer = network.createLayer(4, name="outputGate", bias=True)
network.connectLayers(network.inputLayer, memoryLayer, interconnected=True)
network.connectLayers(network.inputLayer, inputGateLayer, interconnected=True)
network.connectLayers(network.inputLayer, forgetGateLayer, interconnected=True)
network.connectLayers(network.inputLayer, outputGateLayer, interconnected=True)
network.connectLayers(memoryLayer, inputGateLayer, recurrent=True, interconnected=True)
network.connectLayers(memoryLayer, forgetGateLayer, recurrent=True, interconnected=True)
network.connectLayers(memoryLayer, outputGateLayer, recurrent=True, interconnected=True)
network.connectLayers(memoryLayer, network.outputLayer, interconnected=False)
network.addSelfRecurrence(memoryLayer, forgetGateLayer)
network.gateIncomingConnections(network.outputLayer, outputGateLayer)
network.gateIncomingConnections(memoryLayer, inputGateLayer)
printLayer(network.inputLayer)
printLayer(inputGateLayer)
printLayer(memoryLayer)
printLayer(outputGateLayer)
printLayer(network.outputLayer)
inputSet = []
# Create set of input sequences.
for _ in xrange(200):
length = 15
# Fill 10 blank vectors.
inputSequence = [[0.0 for _ in xrange(5)] for __ in xrange(length)]
# Add one relevant vector.
relevant = [0.0, 0.0, 0.0, 0.0, 0.0]
relevant[random.randint(0, 3)] = 1.0
inputSequence.append(relevant)
random.shuffle(inputSequence)
# Add prompt
inputSequence.append([0.0, 0.0, 0.0, 0.0, 1.0])
# Create output sequence.
outputSequence = [[0.0] * 4] * (length + 1)
#outputSequence = [None] * 11
expected = [0.0, 0.0, 0.0, 0.0]
expected[relevant.index(1.0)] = 1.0
outputSequence.append(expected)
inputSet.append((inputSequence, outputSequence))
testSet = inputSet[:20]
inputSet = inputSet[20:]
def calcDiff(x,y):
diff = 0.0
zipped = zip(x,y)
for actual,expected in zipped[:-1]:
for a in actual:
if a > 0.5: diff += a
actual,expected = zipped[-1]
for a,e in zip(actual, expected):
if e == 1.0 and a < 0.5:
#print(a, " != 1")
diff += 1.0 - a
elif e == 0.0 and a > 0.5:
#print(a, " != 0")
diff += a
return diff
beforeDiff = calculateDiff(network, testSet, calcDiff, printValues=False)
print("Before diff: %f" % beforeDiff)
#for _ in xrange(100):
iterations = 0
while True:
print("Iteration %d" % iterations)
iterations += 1
for inputSequence, outputSequence in inputSet:
#print(".")
output = network.learn(inputSequence, outputSequence)
diff = calculateDiff(network, testSet, calcDiff, printValues=False)
print(diff)
if diff < 1: break
afterDiff = calculateDiff(network, testSet, calcDiff, printValues=False)
print("Before diff: %f" % beforeDiff)
print("After diff: %f" % afterDiff)
def forgetTest(debug=False):
"""
Tests to see if a network can remember a value and then forget it.
Input sequences contain one value and two signals.
The first signal indicates that the value should be let into the
memory cell and remembered.
The second signal indicates that the previously remembered value
should be forgotten.
(output)
|
(memory) ) -- (forget)
| |
x ---- (gate) |
| | |
(input) (input2) (input3)
"""
network = Network(3,1)
gateLayer = network.createLayer(1, name="gate", activationFunction=activation.TanH)
forgetLayer = network.createLayer(1, name="forget", bias=True)
memoryLayer = network.createLayer(1, name="memory", activationFunction=activation.Identity)
network.connectLayers(network.inputLayer[:1], memoryLayer)
network.connectLayers(network.inputLayer[1:2], gateLayer)
network.connectLayers(network.inputLayer[2:3], forgetLayer)
network.connectLayers(memoryLayer, network.outputLayer)
network.gateIncomingConnections(memoryLayer, gateLayer)
network.addSelfRecurrence(memoryLayer, forgetLayer)
printLayer(network.inputLayer)
printLayer(memoryLayer)
printLayer(gateLayer)
printLayer(forgetLayer)
printLayer(network.outputLayer)
def genData():
length = 50
start = random.randint(0, length-2)
end = 0
while end <= start: end = random.randint(1, length-1)
sequence = [[random.random() - 0.5, 1.0 if _ == start else 0.0, 1.0 if _ == end else 0.0] for _ in xrange(length)]
return sequence
def genExpected(sequence):
expected = []
toRemember = 0.0
for val,remember,forget in sequence:
if remember == 1.0: toRemember = val
if forget == 1.0: toRemember = 0.0
expected.append([toRemember])
return expected
def calcDiff(x,y):
return sum([abs(a[0]-b[0]) for a,b in zip(x,y)])
runTest(network, genData, genExpected, calcDiff, 100, 10, 100, debug=debug)
printLayer(network.inputLayer)
printLayer(memoryLayer)
printLayer(gateLayer)
printLayer(forgetLayer)
printLayer(network.outputLayer)
def memoryTest(debug=False):
"""
Tests to see if a network can remember a value.
Input sequences contain one value and one signals.
The signal indicates that the value should be let into the
memory cell and remembered.
(output)
|
(memory) )
|
x ---- (gate)
| |
(input) (input2)
"""
network = Network(2,1)
gateLayer = network.createLayer(1, name="gate", activationFunction=activation.TanH)
memoryLayer = network.createLayer(1, name="memory", activationFunction=activation.Identity)
network.connectLayers(network.inputLayer[:1], memoryLayer)
network.connectLayers(network.inputLayer[1:2], gateLayer)
network.connectLayers(memoryLayer, network.outputLayer)
network.gateIncomingConnections(memoryLayer, gateLayer)
network.addSelfRecurrence(memoryLayer)
printLayer(network.inputLayer)
printLayer(memoryLayer)
printLayer(gateLayer)
printLayer(network.outputLayer)
def genData():
sequence = [[random.random() - 0.5, 0.0] for _ in xrange(20)]
if random.random() < 0.8:
sequence.append([random.random() - 0.5, 1.0])
else:
sequence.append([random.random() - 0.5, 0.0])
random.shuffle(sequence)
return sequence
def genExpected(sequence):
val = None
index = None
for i, s in enumerate(sequence):
if s[1] == 1.0:
val = s[0]
index = i
expected = [[0.0] for _ in xrange(len(sequence))]
if val is not None:
for i in xrange(len(sequence) - index):
expected[i + index] = [val]
return expected
def calcDiff(x,y):
return sum([abs(a[0]-b[0]) if b is not None else 0.0 for a,b in zip(x,y)])
runTest(network, genData, genExpected, calcDiff, 100, 10, 100, debug=debug)
printLayer(network.inputLayer)
printLayer(memoryLayer)
printLayer(gateLayer)
printLayer(network.outputLayer)
def distractedRecall():
network = Network(10, 4)
memoryLayer = network.createLayer(8, name="memory")
inputGateLayer = network.createLayer(8, name="inputGate", bias=True)
forgetGateLayer = network.createLayer(8, name="forgetGate", bias=True)
outputGateLayer = network.createLayer(8, name="outputGate", bias=True)
network.connectLayers(network.inputLayer, memoryLayer, interconnected=True)
network.connectLayers(network.inputLayer, inputGateLayer, interconnected=True)
network.connectLayers(network.inputLayer, forgetGateLayer, interconnected=True)
network.connectLayers(network.inputLayer, outputGateLayer, interconnected=True)
network.connectLayers(memoryLayer, inputGateLayer, recurrent=True, interconnected=True)
network.connectLayers(memoryLayer, forgetGateLayer, recurrent=True, interconnected=True)
network.connectLayers(memoryLayer, outputGateLayer, recurrent=True, interconnected=True)
network.connectLayers(memoryLayer, network.outputLayer, interconnected=True)
network.addSelfRecurrence(memoryLayer, forgetGateLayer)
network.gateOutgoingConnections(memoryLayer, outputGateLayer)
network.gateIncomingConnections(memoryLayer, inputGateLayer)
inputSet = []
# Create set of input sequences.
for _ in xrange(100):
length = 10
# Create input sequence.
inputSequence = []
# Fill random distractors.
for _ in xrange(length):
inputVector = [0.0 for _ in xrange(10)]
inputVector[random.randint(4,7)] = 1
inputSequence.append(inputVector)
# Select two random vectors and inject random target symbols.
firstIndex = random.randint(0,length-1)
secondIndex = firstIndex
while secondIndex == firstIndex: secondIndex = random.randint(0,length-1)
if firstIndex > secondIndex:
temp = firstIndex
firstIndex = secondIndex
secondIndex = temp
firstTarget = random.randint(0,3)
secondTarget = random.randint(0,3)
#print(firstIndex, secondIndex)
#print(firstTarget, secondTarget)
inputSequence[firstIndex] = [1.0 if x == firstTarget else 0.0 for x in xrange(10)]
inputSequence[secondIndex] = [1.0 if x == secondTarget else 0.0 for x in xrange(10)]
# Add prompts
inputSequence.append([1.0 if x == 8 else 0.0 for x in xrange(10)])
inputSequence.append([1.0 if x == 9 else 0.0 for x in xrange(10)])
outputSequence = [[0] * 4] * length
#outputSequence = [None] * length
outputSequence.append([1 if x == firstTarget else 0 for x in xrange(4)])
outputSequence.append([1 if x == secondTarget else 0 for x in xrange(4)])
#for line in inputSequence: print(line)
#for line in outputSequence: print(line)
#sys.exit()
inputSet.append((inputSequence, outputSequence))
testSet = inputSet[:10]
inputSet = inputSet[10:]
def calcDiff(x,y):
diff = 0.0
zipped = zip(x,y)
for actual,expected in zipped[:-1]:
for a in actual:
if a > 0.5: diff += a
actual,expected = zipped[-1]
for a,e in zip(actual, expected):
if e == 1.0 and a < 0.5:
print(a, " != 1")
diff += 1.0 - a
elif e == 0 and a > 0.5:
print(a, " != 0")
diff += a
return diff
beforeDiff = calculateDiff(network, testSet, calcDiff, printValues=False)
print("Before diff: %f" % beforeDiff)
#for _ in xrange(10):
iterations = 0
while True:
print("Iteration %d" % iterations)
iterations += 1
for inputSequence, outputSequence in inputSet:
#print(".")
output = network.learn(inputSequence, outputSequence)
diff = calculateDiff(network, testSet, calcDiff, printValues=False)
print(diff)
if diff < 1: break
afterDiff = calculateDiff(network, testSet, calcDiff, printValues=True)
print("Before diff: %f" % beforeDiff)
print("After diff: %f" % afterDiff)