def _buildStructure(self, inputdim, insize, inlayer, convSize, numFeatureMaps): #build layers outdim = insize - convSize + 1 hlayer = TanhLayer(outdim * outdim * numFeatureMaps, name='h') self.addModule(hlayer) outlayer = SigmoidLayer(outdim * outdim, name='out') self.addOutputModule(outlayer) # build shared weights convConns = [] for i in range(convSize): convConns.append(MotherConnection(convSize * numFeatureMaps * inputdim, name='conv' + str(i))) outConn = MotherConnection(numFeatureMaps) # establish the connections. for i in range(outdim): for j in range(outdim): offset = i * outdim + j outmod = ModuleSlice(hlayer, inSliceFrom=offset * numFeatureMaps, inSliceTo=(offset + 1) * numFeatureMaps, outSliceFrom=offset * numFeatureMaps, outSliceTo=(offset + 1) * numFeatureMaps) self.addConnection(SharedFullConnection(outConn, outmod, outlayer, outSliceFrom=offset, outSliceTo=offset + 1)) for k, mc in enumerate(convConns): offset = insize * (i + k) + j inmod = ModuleSlice(inlayer, outSliceFrom=offset * inputdim, outSliceTo=offset * inputdim + convSize * inputdim) self.addConnection(SharedFullConnection(mc, inmod, outmod))
def _buildBorderStructure(self, inmesh, hiddenmesh, outmesh): self._buildSwipingStructure(inmesh, hiddenmesh, outmesh) self.addModule(BiasUnit(name='bias')) # build the motherconnections for the borders if self.simpleborders: if not 'borderconn' in self.predefined: self.predefined['borderconn'] = MotherConnection( hiddenmesh.componentIndim, name='bconn') else: if not 'bordconns' in self.predefined: self.predefined['bordconns'] = {} for dim, maxval in enumerate(self.dims): if dim > 0 and self.symmetricdimensions: self.predefined['bordconns'][dim] = self.predefined[ 'bordconns'][0] elif dim not in self.predefined['bordconns']: self.predefined['bordconns'][dim] = {} tmp = self.predefined['bordconns'][dim].copy() if len(self.dims) == 1 and () not in tmp: tmp[()] = MotherConnection(hiddenmesh.componentIndim, name='bconn') for t in iterCombinations(tupleRemoveItem(self.dims, dim)): tc = self._canonicForm(t, dim) if t == tc and t not in tmp: # the connections from the borders are symetrical, # so we need seperate ones only up to the middle tmp[t] = MotherConnection(hiddenmesh.componentIndim, name='bconn' + str(dim) + str(t)) if self.extrapolateBorderValues: p = self._extrapolateBorderAt( t, self.predefined['bordconns'][dim]) if p != None: tmp[t].params[:] = p self.predefined['bordconns'][dim] = tmp # link the bordering units to the bias, using the correct connection for dim, maxval in enumerate(self.dims): for unit in self._iterateOverUnits(): if self.simpleborders: bconn = self.predefined['borderconn'] else: tc = self._canonicForm(tupleRemoveItem(unit, dim), dim) bconn = self.predefined['bordconns'][dim][tc] hunits = [] if unit[dim] == 0: for swipe in range(self.swipes): if (swipe / 2**dim) % 2 == 0: hunits.append(tuple(list(unit) + [swipe])) if unit[dim] == maxval - 1: for swipe in range(self.swipes): if (swipe / 2**dim) % 2 == 1: hunits.append(tuple(list(unit) + [swipe])) for hunit in hunits: self.addConnection( SharedFullConnection(bconn, self['bias'], hiddenmesh[hunit]))
def _buildBorderStructure(self, inmesh, hiddenmesh, outmesh): self._buildSwipingStructure(inmesh, hiddenmesh, outmesh) self.addModule(BiasUnit(name = 'bias')) # build the motherconnections for the borders if self.simpleborders: if not 'borderconn' in self.predefined: self.predefined['borderconn'] = MotherConnection(hiddenmesh.componentIndim, name = 'bconn') else: if not 'bordconns' in self.predefined: self.predefined['bordconns'] = {} for dim, maxval in enumerate(self.dims): if dim > 0 and self.symmetricdimensions: self.predefined['bordconns'][dim] = self.predefined['bordconns'][0] elif dim not in self.predefined['bordconns']: self.predefined['bordconns'][dim] = {} tmp = self.predefined['bordconns'][dim].copy() if len(self.dims) == 1 and () not in tmp: tmp[()] = MotherConnection(hiddenmesh.componentIndim, name = 'bconn') for t in iterCombinations(tupleRemoveItem(self.dims, dim)): tc = self._canonicForm(t, dim) if t == tc and t not in tmp: # the connections from the borders are symmetrical, # so we need separate ones only up to the middle tmp[t] = MotherConnection(hiddenmesh.componentIndim, name = 'bconn'+str(dim)+str(t)) if self.extrapolateBorderValues: p = self._extrapolateBorderAt(t, self.predefined['bordconns'][dim]) if p != None: tmp[t].params[:] = p self.predefined['bordconns'][dim] = tmp # link the bordering units to the bias, using the correct connection for dim, maxval in enumerate(self.dims): for unit in self._iterateOverUnits(): if self.simpleborders: bconn = self.predefined['borderconn'] else: tc = self._canonicForm(tupleRemoveItem(unit, dim), dim) bconn = self.predefined['bordconns'][dim][tc] hunits = [] if unit[dim] == 0: for swipe in range(self.swipes): if (swipe/2**dim) % 2 == 0: hunits.append(tuple(list(unit)+[swipe])) if unit[dim] == maxval-1: for swipe in range(self.swipes): if (swipe/2**dim) % 2 == 1: hunits.append(tuple(list(unit)+[swipe])) for hunit in hunits: self.addConnection(SharedFullConnection(bconn, self['bias'], hiddenmesh[hunit]))
def __init__(self, predefined = None, **kwargs): """ For the current implementation, the sequence length needs to be fixed, and given at construction time. """ if predefined is not None: self.predefined = predefined else: self.predefined = {} FeedForwardNetwork.__init__(self, **kwargs) assert self.seqlen is not None # the input is a 1D-mesh (as a view on a flat input layer) inmod = LinearLayer(self.inputsize * self.seqlen, name='input') inmesh = ModuleMesh.viewOnFlatLayer(inmod, (self.seqlen,), 'inmesh') # the output is also a 1D-mesh outmod = self.outcomponentclass(self.outputsize * self.seqlen, name='output') outmesh = ModuleMesh.viewOnFlatLayer(outmod, (self.seqlen,), 'outmesh') # the hidden layers are places in a 2xseqlen mesh hiddenmesh = ModuleMesh.constructWithLayers(self.componentclass, self.hiddensize, (2, self.seqlen), 'hidden') # add the modules for c in inmesh: self.addInputModule(c) for c in outmesh: self.addOutputModule(c) for c in hiddenmesh: self.addModule(c) # set the connections weights to be shared inconnf = MotherConnection(inmesh.componentOutdim * hiddenmesh.componentIndim, name='inconn') outconnf = MotherConnection(outmesh.componentIndim * hiddenmesh.componentOutdim, name='outconn') forwardconn = MotherConnection(hiddenmesh.componentIndim * hiddenmesh.componentOutdim, name='fconn') if self.symmetric: backwardconn = forwardconn inconnb = inconnf outconnb = outconnf else: backwardconn = MotherConnection(hiddenmesh.componentIndim * hiddenmesh.componentOutdim, name='bconn') inconnb = MotherConnection(inmesh.componentOutdim * hiddenmesh.componentIndim, name='inconn') outconnb = MotherConnection(outmesh.componentIndim * hiddenmesh.componentOutdim, name='outconn') # build the connections for i in range(self.seqlen): # input to hidden self.addConnection(SharedFullConnection(inconnf, inmesh[(i,)], hiddenmesh[(0, i)])) self.addConnection(SharedFullConnection(inconnb, inmesh[(i,)], hiddenmesh[(1, i)])) # hidden to output self.addConnection(SharedFullConnection(outconnf, hiddenmesh[(0, i)], outmesh[(i,)])) self.addConnection(SharedFullConnection(outconnb, hiddenmesh[(1, i)], outmesh[(i,)])) if i > 0: # forward in time self.addConnection(SharedFullConnection(forwardconn, hiddenmesh[(0, i - 1)], hiddenmesh[(0, i)])) if i < self.seqlen - 1: # backward in time self.addConnection(SharedFullConnection(backwardconn, hiddenmesh[(1, i + 1)], hiddenmesh[(1, i)])) self.sortModules()
def __init__(self, boardSize, convSize, numFeatureMaps, **args): inputdim = 2 FeedForwardNetwork.__init__(self, **args) inlayer = LinearLayer(inputdim * boardSize * boardSize, name='in') self.addInputModule(inlayer) # we need some treatment of the border too - thus we pad the direct board input. x = convSize / 2 insize = boardSize + 2 * x if convSize % 2 == 0: insize -= 1 paddedlayer = LinearLayer(inputdim * insize * insize, name='pad') self.addModule(paddedlayer) # we connect a bias to the padded-parts (with shared but trainable weights). bias = BiasUnit() self.addModule(bias) biasConn = MotherConnection(inputdim) paddable = [] if convSize % 2 == 0: xs = range(x) + range(insize - x + 1, insize) else: xs = range(x) + range(insize - x, insize) paddable.extend(crossproduct([range(insize), xs])) paddable.extend(crossproduct([xs, range(x, boardSize + x)])) for (i, j) in paddable: self.addConnection( SharedFullConnection(biasConn, bias, paddedlayer, outSliceFrom=(i * insize + j) * inputdim, outSliceTo=(i * insize + j + 1) * inputdim)) for i in range(boardSize): inmod = ModuleSlice(inlayer, outSliceFrom=i * boardSize * inputdim, outSliceTo=(i + 1) * boardSize * inputdim) outmod = ModuleSlice(paddedlayer, inSliceFrom=((i + x) * insize + x) * inputdim, inSliceTo=((i + x) * insize + x + boardSize) * inputdim) self.addConnection(IdentityConnection(inmod, outmod)) self._buildStructure(inputdim, insize, paddedlayer, convSize, numFeatureMaps) self.sortModules()
def _buildSwipingStructure(self, inmesh, hiddenmesh, outmesh): """ :key inmesh: a mesh of input units :key hiddenmesh: a mesh of hidden units :key outmesh: a mesh of output units """ self._verifyDimensions(inmesh, hiddenmesh, outmesh) # add the modules for c in inmesh: self.addInputModule(c) for c in outmesh: self.addOutputModule(c) for c in hiddenmesh: self.addModule(c) # create the motherconnections if they are not provided if 'inconn' not in self.predefined: self.predefined['inconn'] = MotherConnection( inmesh.componentOutdim * hiddenmesh.componentIndim, name='inconn') if 'outconn' not in self.predefined: self.predefined['outconn'] = MotherConnection( outmesh.componentIndim * hiddenmesh.componentOutdim, name='outconn') if 'hconns' not in self.predefined: self.predefined['hconns'] = {} for s in range(len(self.dims)): if self.symmetricdirections: if s > 0 and self.symmetricdimensions: self.predefined['hconns'][s] = self.predefined[ 'hconns'][0] else: self.predefined['hconns'][s] = MotherConnection( hiddenmesh.componentIndim * hiddenmesh.componentOutdim, name='hconn' + str(s)) else: for dir in ['-', '+']: if s > 0 and self.symmetricdimensions: self.predefined['hconns'][( s, dir)] = self.predefined['hconns'][(0, dir)] else: self.predefined['hconns'][( s, dir)] = MotherConnection( hiddenmesh.componentIndim * hiddenmesh.componentOutdim, name='hconn' + str(s) + dir) # establish the connections for unit in self._iterateOverUnits(): for swipe in range(self.swipes): hunit = tuple(list(unit) + [swipe]) self.addConnection( SharedFullConnection(self.predefined['inconn'], inmesh[unit], hiddenmesh[hunit])) self.addConnection( SharedFullConnection(self.predefined['outconn'], hiddenmesh[hunit], outmesh[unit])) # one swiping connection along every dimension for dim, maxval in enumerate(self.dims): # determine where the swipe is coming from in this direction: # swipe directions are towards higher coordinates on dim D if the swipe%(2**D) = 0 # and towards lower coordinates otherwise. previousunit = list(hunit) if (swipe / 2**dim) % 2 == 0: previousunit[dim] -= 1 dir = '+' else: previousunit[dim] += 1 dir = '-' if self.symmetricdirections: hconn = self.predefined['hconns'][dim] else: hconn = self.predefined['hconns'][(dim, dir)] previousunit = tuple(previousunit) if previousunit[dim] >= 0 and previousunit[dim] < maxval: self.addConnection( SharedFullConnection(hconn, hiddenmesh[previousunit], hiddenmesh[hunit]))