def _buildStructure(self, inputdim, insize, inlayer, convSize, numFeatureMaps):
#build layers
outdim = insize - convSize + 1
hlayer = TanhLayer(outdim * outdim * numFeatureMaps, name='h')
self.addModule(hlayer)
outlayer = SigmoidLayer(outdim * outdim, name='out')
self.addOutputModule(outlayer)
# build shared weights
convConns = []
for i in range(convSize):
convConns.append(MotherConnection(convSize * numFeatureMaps * inputdim, name='conv' + str(i)))
outConn = MotherConnection(numFeatureMaps)
# establish the connections.
for i in range(outdim):
for j in range(outdim):
offset = i * outdim + j
outmod = ModuleSlice(hlayer, inSliceFrom=offset * numFeatureMaps, inSliceTo=(offset + 1) * numFeatureMaps,
outSliceFrom=offset * numFeatureMaps, outSliceTo=(offset + 1) * numFeatureMaps)
self.addConnection(SharedFullConnection(outConn, outmod, outlayer, outSliceFrom=offset, outSliceTo=offset + 1))
for k, mc in enumerate(convConns):
offset = insize * (i + k) + j
inmod = ModuleSlice(inlayer, outSliceFrom=offset * inputdim, outSliceTo=offset * inputdim + convSize * inputdim)
self.addConnection(SharedFullConnection(mc, inmod, outmod))
def _buildBorderStructure(self, inmesh, hiddenmesh, outmesh):
self._buildSwipingStructure(inmesh, hiddenmesh, outmesh)
self.addModule(BiasUnit(name='bias'))
# build the motherconnections for the borders
if self.simpleborders:
if not 'borderconn' in self.predefined:
self.predefined['borderconn'] = MotherConnection(
hiddenmesh.componentIndim, name='bconn')
else:
if not 'bordconns' in self.predefined:
self.predefined['bordconns'] = {}
for dim, maxval in enumerate(self.dims):
if dim > 0 and self.symmetricdimensions:
self.predefined['bordconns'][dim] = self.predefined[
'bordconns'][0]
elif dim not in self.predefined['bordconns']:
self.predefined['bordconns'][dim] = {}
tmp = self.predefined['bordconns'][dim].copy()
if len(self.dims) == 1 and () not in tmp:
tmp[()] = MotherConnection(hiddenmesh.componentIndim,
name='bconn')
for t in iterCombinations(tupleRemoveItem(self.dims, dim)):
tc = self._canonicForm(t, dim)
if t == tc and t not in tmp:
# the connections from the borders are symetrical,
# so we need seperate ones only up to the middle
tmp[t] = MotherConnection(hiddenmesh.componentIndim,
name='bconn' + str(dim) +
str(t))
if self.extrapolateBorderValues:
p = self._extrapolateBorderAt(
t, self.predefined['bordconns'][dim])
if p != None:
tmp[t].params[:] = p
self.predefined['bordconns'][dim] = tmp
# link the bordering units to the bias, using the correct connection
for dim, maxval in enumerate(self.dims):
for unit in self._iterateOverUnits():
if self.simpleborders:
bconn = self.predefined['borderconn']
else:
tc = self._canonicForm(tupleRemoveItem(unit, dim), dim)
bconn = self.predefined['bordconns'][dim][tc]
hunits = []
if unit[dim] == 0:
for swipe in range(self.swipes):
if (swipe / 2**dim) % 2 == 0:
hunits.append(tuple(list(unit) + [swipe]))
if unit[dim] == maxval - 1:
for swipe in range(self.swipes):
if (swipe / 2**dim) % 2 == 1:
hunits.append(tuple(list(unit) + [swipe]))
for hunit in hunits:
self.addConnection(
SharedFullConnection(bconn, self['bias'],
hiddenmesh[hunit]))
def _buildBorderStructure(self, inmesh, hiddenmesh, outmesh):
self._buildSwipingStructure(inmesh, hiddenmesh, outmesh)
self.addModule(BiasUnit(name = 'bias'))
# build the motherconnections for the borders
if self.simpleborders:
if not 'borderconn' in self.predefined:
self.predefined['borderconn'] = MotherConnection(hiddenmesh.componentIndim, name = 'bconn')
else:
if not 'bordconns' in self.predefined:
self.predefined['bordconns'] = {}
for dim, maxval in enumerate(self.dims):
if dim > 0 and self.symmetricdimensions:
self.predefined['bordconns'][dim] = self.predefined['bordconns'][0]
elif dim not in self.predefined['bordconns']:
self.predefined['bordconns'][dim] = {}
tmp = self.predefined['bordconns'][dim].copy()
if len(self.dims) == 1 and () not in tmp:
tmp[()] = MotherConnection(hiddenmesh.componentIndim, name = 'bconn')
for t in iterCombinations(tupleRemoveItem(self.dims, dim)):
tc = self._canonicForm(t, dim)
if t == tc and t not in tmp:
# the connections from the borders are symmetrical,
# so we need separate ones only up to the middle
tmp[t] = MotherConnection(hiddenmesh.componentIndim, name = 'bconn'+str(dim)+str(t))
if self.extrapolateBorderValues:
p = self._extrapolateBorderAt(t, self.predefined['bordconns'][dim])
if p != None:
tmp[t].params[:] = p
self.predefined['bordconns'][dim] = tmp
# link the bordering units to the bias, using the correct connection
for dim, maxval in enumerate(self.dims):
for unit in self._iterateOverUnits():
if self.simpleborders:
bconn = self.predefined['borderconn']
else:
tc = self._canonicForm(tupleRemoveItem(unit, dim), dim)
bconn = self.predefined['bordconns'][dim][tc]
hunits = []
if unit[dim] == 0:
for swipe in range(self.swipes):
if (swipe/2**dim) % 2 == 0:
hunits.append(tuple(list(unit)+[swipe]))
if unit[dim] == maxval-1:
for swipe in range(self.swipes):
if (swipe/2**dim) % 2 == 1:
hunits.append(tuple(list(unit)+[swipe]))
for hunit in hunits:
self.addConnection(SharedFullConnection(bconn, self['bias'], hiddenmesh[hunit]))
def __init__(self, predefined = None, **kwargs):
""" For the current implementation, the sequence length
needs to be fixed, and given at construction time. """
if predefined is not None:
self.predefined = predefined
else:
self.predefined = {}
FeedForwardNetwork.__init__(self, **kwargs)
assert self.seqlen is not None
# the input is a 1D-mesh (as a view on a flat input layer)
inmod = LinearLayer(self.inputsize * self.seqlen, name='input')
inmesh = ModuleMesh.viewOnFlatLayer(inmod, (self.seqlen,), 'inmesh')
# the output is also a 1D-mesh
outmod = self.outcomponentclass(self.outputsize * self.seqlen, name='output')
outmesh = ModuleMesh.viewOnFlatLayer(outmod, (self.seqlen,), 'outmesh')
# the hidden layers are places in a 2xseqlen mesh
hiddenmesh = ModuleMesh.constructWithLayers(self.componentclass, self.hiddensize,
(2, self.seqlen), 'hidden')
# add the modules
for c in inmesh:
self.addInputModule(c)
for c in outmesh:
self.addOutputModule(c)
for c in hiddenmesh:
self.addModule(c)
# set the connections weights to be shared
inconnf = MotherConnection(inmesh.componentOutdim * hiddenmesh.componentIndim, name='inconn')
outconnf = MotherConnection(outmesh.componentIndim * hiddenmesh.componentOutdim, name='outconn')
forwardconn = MotherConnection(hiddenmesh.componentIndim * hiddenmesh.componentOutdim, name='fconn')
if self.symmetric:
backwardconn = forwardconn
inconnb = inconnf
outconnb = outconnf
else:
backwardconn = MotherConnection(hiddenmesh.componentIndim * hiddenmesh.componentOutdim, name='bconn')
inconnb = MotherConnection(inmesh.componentOutdim * hiddenmesh.componentIndim, name='inconn')
outconnb = MotherConnection(outmesh.componentIndim * hiddenmesh.componentOutdim, name='outconn')
# build the connections
for i in range(self.seqlen):
# input to hidden
self.addConnection(SharedFullConnection(inconnf, inmesh[(i,)], hiddenmesh[(0, i)]))
self.addConnection(SharedFullConnection(inconnb, inmesh[(i,)], hiddenmesh[(1, i)]))
# hidden to output
self.addConnection(SharedFullConnection(outconnf, hiddenmesh[(0, i)], outmesh[(i,)]))
self.addConnection(SharedFullConnection(outconnb, hiddenmesh[(1, i)], outmesh[(i,)]))
if i > 0:
# forward in time
self.addConnection(SharedFullConnection(forwardconn, hiddenmesh[(0, i - 1)], hiddenmesh[(0, i)]))
if i < self.seqlen - 1:
# backward in time
self.addConnection(SharedFullConnection(backwardconn, hiddenmesh[(1, i + 1)], hiddenmesh[(1, i)]))
self.sortModules()
def __init__(self, boardSize, convSize, numFeatureMaps, **args):
inputdim = 2
FeedForwardNetwork.__init__(self, **args)
inlayer = LinearLayer(inputdim * boardSize * boardSize, name='in')
self.addInputModule(inlayer)
# we need some treatment of the border too - thus we pad the direct board input.
x = convSize / 2
insize = boardSize + 2 * x
if convSize % 2 == 0:
insize -= 1
paddedlayer = LinearLayer(inputdim * insize * insize, name='pad')
self.addModule(paddedlayer)
# we connect a bias to the padded-parts (with shared but trainable weights).
bias = BiasUnit()
self.addModule(bias)
biasConn = MotherConnection(inputdim)
paddable = []
if convSize % 2 == 0:
xs = range(x) + range(insize - x + 1, insize)
else:
xs = range(x) + range(insize - x, insize)
paddable.extend(crossproduct([range(insize), xs]))
paddable.extend(crossproduct([xs, range(x, boardSize + x)]))
for (i, j) in paddable:
self.addConnection(
SharedFullConnection(biasConn,
bias,
paddedlayer,
outSliceFrom=(i * insize + j) * inputdim,
outSliceTo=(i * insize + j + 1) *
inputdim))
for i in range(boardSize):
inmod = ModuleSlice(inlayer,
outSliceFrom=i * boardSize * inputdim,
outSliceTo=(i + 1) * boardSize * inputdim)
outmod = ModuleSlice(paddedlayer,
inSliceFrom=((i + x) * insize + x) * inputdim,
inSliceTo=((i + x) * insize + x + boardSize) *
inputdim)
self.addConnection(IdentityConnection(inmod, outmod))
self._buildStructure(inputdim, insize, paddedlayer, convSize,
numFeatureMaps)
self.sortModules()
def _buildSwipingStructure(self, inmesh, hiddenmesh, outmesh):
"""
:key inmesh: a mesh of input units
:key hiddenmesh: a mesh of hidden units
:key outmesh: a mesh of output units
"""
self._verifyDimensions(inmesh, hiddenmesh, outmesh)
# add the modules
for c in inmesh:
self.addInputModule(c)
for c in outmesh:
self.addOutputModule(c)
for c in hiddenmesh:
self.addModule(c)
# create the motherconnections if they are not provided
if 'inconn' not in self.predefined:
self.predefined['inconn'] = MotherConnection(
inmesh.componentOutdim * hiddenmesh.componentIndim,
name='inconn')
if 'outconn' not in self.predefined:
self.predefined['outconn'] = MotherConnection(
outmesh.componentIndim * hiddenmesh.componentOutdim,
name='outconn')
if 'hconns' not in self.predefined:
self.predefined['hconns'] = {}
for s in range(len(self.dims)):
if self.symmetricdirections:
if s > 0 and self.symmetricdimensions:
self.predefined['hconns'][s] = self.predefined[
'hconns'][0]
else:
self.predefined['hconns'][s] = MotherConnection(
hiddenmesh.componentIndim *
hiddenmesh.componentOutdim,
name='hconn' + str(s))
else:
for dir in ['-', '+']:
if s > 0 and self.symmetricdimensions:
self.predefined['hconns'][(
s, dir)] = self.predefined['hconns'][(0, dir)]
else:
self.predefined['hconns'][(
s, dir)] = MotherConnection(
hiddenmesh.componentIndim *
hiddenmesh.componentOutdim,
name='hconn' + str(s) + dir)
# establish the connections
for unit in self._iterateOverUnits():
for swipe in range(self.swipes):
hunit = tuple(list(unit) + [swipe])
self.addConnection(
SharedFullConnection(self.predefined['inconn'],
inmesh[unit], hiddenmesh[hunit]))
self.addConnection(
SharedFullConnection(self.predefined['outconn'],
hiddenmesh[hunit], outmesh[unit]))
# one swiping connection along every dimension
for dim, maxval in enumerate(self.dims):
# determine where the swipe is coming from in this direction:
# swipe directions are towards higher coordinates on dim D if the swipe%(2**D) = 0
# and towards lower coordinates otherwise.
previousunit = list(hunit)
if (swipe / 2**dim) % 2 == 0:
previousunit[dim] -= 1
dir = '+'
else:
previousunit[dim] += 1
dir = '-'
if self.symmetricdirections:
hconn = self.predefined['hconns'][dim]
else:
hconn = self.predefined['hconns'][(dim, dir)]
previousunit = tuple(previousunit)
if previousunit[dim] >= 0 and previousunit[dim] < maxval:
self.addConnection(
SharedFullConnection(hconn,
hiddenmesh[previousunit],
hiddenmesh[hunit]))