def __call__(self, *args, **kwargs): # This function wraps the compiled theano function. # ------------------------------------------------------ # Don't allow args if self.inputs is a dictionary. This is because the user can not be expected to know # exactly how a dictionary is ordered. args = list(args) if isinstance(self.inputs, dict): assert args == [], "Antipasti function object expects keyword arguments because the " \ "provided input was a dict." if isinstance(self.inputs, list): assert kwargs == {}, "Keywords could not be parsed by the Antipasti function object." # Flatten kwargs or args if args: funcargs = list(pyk.flatten(args)) else: funcargs = list(pyk.flatten(kwargs.values())) # Evaluate function outlist = pyk.obj2list(self._thfunction(*funcargs), ndarray2list=False) # Parse output list expoutputs = self.outputs.values() if isinstance( self.outputs, dict) else self.outputs expoutputs = pyk.obj2list(expoutputs, ndarray2list=False) # Make sure the theano function has returned the correct number of outputs assert len(outlist) == len(list(pyk.flatten(expoutputs))), "Number of outputs returned by the theano function " \ "is not consistent with the number of expected " \ "outputs." # Unflatten theano function output (outlist) # Get list with sublist lengths lenlist = [pyk.smartlen(expoutput) for expoutput in expoutputs] # Unflatten outlist outputs = pyk.unflatten(outlist, lenlist) # Write to dictionary if self.outputs is a dictionary if isinstance(self.outputs, dict): outputs = { outname: outvar for outname, outvar in zip(self.outputs.keys(), outputs) } elif isinstance(self.outputs, list): outputs = tuple(outputs) else: outputs = pyk.delist(outputs) return outputs
def makelayerxy(inpdim, outdim, layerid): x = pyk.delist([ T.tensor('floatX', [ False, ] * indim, name='x{}:'.format(inpnum) + str(layerid)) for inpnum, indim in enumerate(pyk.obj2list(inpdim)) ]) y = pyk.delist([ T.tensor('floatX', [ False, ] * oudim, name='y{}:'.format(outnum) + str(layerid)) for outnum, oudim in enumerate(pyk.obj2list(outdim)) ]) return x, y
def feedforward(self, inp=None): if inp is None: inp = self.x else: self.x = inp # Get output from Lasagne out = las.layers.get_output( self.outputlayers, inputs={ inplayer: ishp for inplayer, ishp in zip(pyk.obj2list(self.inputlayers), pyk.obj2list(inp)) }) # In case out is a list: self.y = pyk.delist(out) return self.y
def __init__(self, splits, dim=None, issequence=None, inpshape=None): """ :type splits: list or int :param splits: Index of the split (along the channel axis). E.g. split = 3 would result in the input tensor split as: [inp[:, 0:3, ...], inp[:, 3:, ...]] for 2D inputs. :type issequence: bool :param issequence: Whether input is a sequence :type inpshape: list or tuple :param inpshape: Input shape :return: """ super(splitlayer, self).__init__() # Parse dim = 2 if issequence else dim assert not ( dim is None and inpshape is None ), "Data dimension can not be parsed. Provide dim or inpshape." # Meta self.dim = dim if dim is not None else {4: 2, 5: 3}[len(inpshape)] self.allowsequences = True self.issequence = self.dim == 2 and len( self.inpshape) == 5 if issequence is None else issequence self.inpdim = len( inpshape) if inpshape is not None else 5 if self.issequence else { 2: 4, 3: 5 }[dim] self.dim = 2 if self.issequence else self.dim # Correct dim if necessary self.splits = pyk.obj2list(splits) self.numsplits = len(self.splits) + 1 # More meta for layertrainyard self.numinp = 1 self.numout = self.numsplits # Shape inference self.inpshape = [ None, ] * self.inpdim if inpshape is None else list(inpshape) # Containers for input and output self.x = T.tensor('floatX', [ False, ] * self.inpdim, name='x:' + str(id(self))) self.y = [ T.tensor('floatX', [ False, ] * self.inpdim, name='y{}:'.format(splitnum) + str(id(self))) for splitnum in range(self.numsplits) ]
def setbaggage(objs, **baggage): baggage = baggage['baggage'] if 'baggage' in baggage.keys() else baggage for obj in pyk.obj2list(objs, ndarray2list=False): # Check if obj has a baggage. if hasattr(obj, 'baggage'): # Update baggage with new entries obj.baggage.update(baggage) else: # Make new baggage obj.baggage = baggage
def testmodel(model, inpshape, verbose=True, outshape=None): testpassed = None # Allocate numerical input values (model may have multiple inputs/outputs) inpvals = [np.random.uniform(size=ishp).astype(th.config.floatX) for ishp in pyk.list2listoflists(inpshape)] # Feedforward the model (i.e. build theano graph if it isn't built already) if any([y.owner is None for y in pyk.obj2list(model.y)]): model.feedforward() # Evaluate model for all inputs try: if verbose: print("Compiling...") outvals = [y.eval({x: xval for x, xval in zip(pyk.obj2list(model.x), inpvals)}) for y in pyk.obj2list(model.y)] if verbose: print("Output shapes are: {}".format([oval.shape for oval in outvals])) except Exception as e: testpassed = False if verbose: print("Test failed because an exception was raised. The original error message follows:") print(e.message) # Check if output shapes check-out if outshape is not None and testpassed is None: modeloutshape = [oval.shape for oval in outvals] shapecheck = modeloutshape == pyk.list2listoflists(outshape) if not shapecheck: if verbose: "Shape check failed. Expected output shape {}, got {} instead.".format(outshape, pyk.delist(modeloutshape)) testpassed = False if testpassed is None: testpassed = True return testpassed
def func(*args, **kwargs): # Compute argument length arglen = max([pyk.smartlen(arg) for arg in list(args) + list(kwargs.values())]) # Convert arguments to list args = [pyk.obj2list(arg) if not len(pyk.obj2list(arg)) == 1 else pyk.obj2list(arg)*arglen for arg in args] kwargs = {kw: pyk.obj2list(arg) if not len(pyk.obj2list(arg)) == 1 else pyk.obj2list(arg)*arglen for kw, arg in kwargs.items()} # Make sure list sizes check out assert all([pyk.smartlen(arg) == arglen for arg in args]) if pyk.smartlen(arg) != 0 else True, \ "Input lists must all have the same length." assert all([pyk.smartlen(kwarg) == pyk.smartlen(kwargs.values()[0]) == arglen for kwarg in kwargs.values()]) \ if pyk.smartlen(kwargs) != 0 else True, "Keyword argument vectors must have the same length (= argument " \ "vector length)" # Run the loop (can be done with a long-ass list comprehension, but I don't see the point) res = [] if not len(kwargs) == 0 and not len(args) == 0: for arg, kwarg in zip(zip(*args), zip(*kwargs.values())): res.append(fun(*arg, **{kw: kwa for kw, kwa in zip(kwargs.keys(), kwarg)})) else: if len(kwargs) == 0: res = [fun(*arg) for arg in zip(*args)] elif len(args) == 0: res = [fun(**{kw: kwa for kw, kwa in zip(kwargs.keys(), kwarg)}) for kwarg in zip(*kwargs.values())] else: return [] # Return results return pyk.delist(res)
def __init__(self, inputlayers, outputlayers): """ :type inputlayers: list :param inputlayers: List of Lasagne input layers. :type outputlayers: list :param outputlayers: List of output layers. """ # Init superclass super(lasagnelayer, self).__init__() # Make sure lasagne is available assert las is not None, "Lasagne could not be imported." # Meta self.inputlayers = pyk.delist(inputlayers) self.outputlayers = pyk.delist(outputlayers) # Get inpshape from lasagne self.lasinpshape = pyk.delist( [list(il.shape) for il in pyk.obj2list(self.inputlayers)]) # Parse layer info parsey = netutils.parselayerinfo(dim=2, allowsequences=True, numinp=pyk.smartlen(inputlayers), issequence=False, inpshape=self.lasinpshape) self.dim = parsey['dim'] self.inpdim = parsey['inpdim'] self.allowsequences = parsey['allowsequences'] self.issequence = parsey['issequence'] self.numinp = parsey['numinp'] # Read parameters self._params = las.layers.get_all_params(self.outputlayers) self._cparams = [ netutils.getshared(like=param, value=1.) for param in self.params ] # Name parameters right self.nameparams() # Shape inference self.inpshape = parsey['inpshape'] # Check numout for consistency assert self.numout == pyk.smartlen(self.outputlayers), "Number of outputs doesn't match " \ "the given number of output-layers" self.x, self.y = netutils.makelayerxy(self.inpdim, self.outdim, id(self))
def feedforward(self, inp=None): if inp is None: inp = self.x else: self.x = inp # Evaluate function y = self.func(inp, *self.funcargs, **self.funckwargs) # Convert y to list if it's a tuple y = list(y) if isinstance(y, tuple) else y # Make sure output is consistent with expectation (since func can do whatever the f it likes) len(pyk.obj2list(y)) == self.numout, "The given function must return {} outputs, " \ "got {} instead.".format(self.numout, len(pyk.obj2list(y))) self.y = y return self.y
def makeprinter(verbosity, extramonitors=None): if verbosity >= 4: monitors = [batchmonitor, costmonitor, lossmonitor, trEmonitor, gradnormmonitor, updatenormmonitor] elif verbosity >= 3: monitors = [batchmonitor, costmonitor, lossmonitor, trEmonitor] elif verbosity >= 2: monitors = [batchmonitor, costmonitor] else: monitors = [] if extramonitors is not None: extramonitors = pyk.obj2list(extramonitors) # Append extra monitors monitors.extend(extramonitors) # Build printer prntr = printer(monitors) return prntr
def feedforward(self, inp=None): if inp is None: inp = self.x else: self.x = inp # Loop over connections, merge and append to a buffer out = [] for connum, conn in enumerate(self.connections): if pyk.smartlen(conn) == 1: out.append(inp[pyk.delist(pyk.obj2list(conn))]) else: if self.merge: out.append(self.mergelayers[connum].feedforward(inp=[inp[node] for node in conn])) else: out.append([inp[node] for node in conn]) self.y = out return self.y
def inferoutshape(self, inpshape=None, checkinput=True): if inpshape is None: inpshape = self.inpshape # Buffer for outshape outshape = [] for connum, conn in enumerate(self.connections): if self.mergelayers[connum] is None: # conn is the index of an element in the input list. Fetch its shape: outshape.append(self.inpshape[pyk.delist(pyk.obj2list(conn))]) else: if self.merge: # Fetch merge layer's inpshape self.mergelayers[connum].inpshape = [inpshape[node] for node in conn] # Append outshape outshape.append(self.mergelayers[connum].outshape) else: outshape.append([inpshape[node] for node in conn]) return outshape
def inferoutshape(self, inpshape=None, checkinput=True): if inpshape is None: inpshape = self.inpshape if checkinput: # Compare shape with Antipasti inpshape assert netutils.shpcmp(self.lasinpshape, inpshape), "Lasagne input shape is not consistent with the " \ "inferred Antipasti input shape." # Get output shape from Lasagne outshape = las.layers.get_output_shape( self.outputlayers, { inplayer: ishp for inplayer, ishp in zip(pyk.obj2list(self.inputlayers), pyk.list2listoflists(inpshape)) }) outshape = pyk.listoftuples2listoflists(outshape) if pyk.islistoflists( outshape) else list(outshape) return outshape
def feedforward(self, inp=None): if inp is None: inp = self.x else: self.x = inp # Loop over connections, merge and append to a buffer out = [] for connum, conn in enumerate(self.connections): if pyk.smartlen(conn) == 1: out.append(inp[pyk.delist(pyk.obj2list(conn))]) else: if self.merge: out.append(self.mergelayers[connum].feedforward( inp=[inp[node] for node in conn])) else: out.append([inp[node] for node in conn]) self.y = out return self.y
def __init__(self, splits, dim=None, issequence=None, inpshape=None): """ :type splits: list or int :param splits: Index of the split (along the channel axis). E.g. split = 3 would result in the input tensor split as: [inp[:, 0:3, ...], inp[:, 3:, ...]] for 2D inputs. :type issequence: bool :param issequence: Whether input is a sequence :type inpshape: list or tuple :param inpshape: Input shape :return: """ super(splitlayer, self).__init__() # Parse dim = 2 if issequence else dim assert not (dim is None and inpshape is None), "Data dimension can not be parsed. Provide dim or inpshape." # Meta self.dim = dim if dim is not None else {4: 2, 5: 3}[len(inpshape)] self.allowsequences = True self.issequence = self.dim == 2 and len(self.inpshape) == 5 if issequence is None else issequence self.inpdim = len(inpshape) if inpshape is not None else 5 if self.issequence else {2: 4, 3: 5}[dim] self.dim = 2 if self.issequence else self.dim # Correct dim if necessary self.splits = pyk.obj2list(splits) self.numsplits = len(self.splits) + 1 # More meta for layertrainyard self.numinp = 1 self.numout = self.numsplits # Shape inference self.inpshape = [None, ] * self.inpdim if inpshape is None else list(inpshape) # Containers for input and output self.x = T.tensor('floatX', [False, ] * self.inpdim, name='x:' + str(id(self))) self.y = [T.tensor('floatX', [False, ] * self.inpdim, name='y{}:'.format(splitnum) + str(id(self))) for splitnum in range(self.numsplits)]
def inferoutshape(self, inpshape=None, checkinput=True): if inpshape is None: inpshape = self.inpshape # Buffer for outshape outshape = [] for connum, conn in enumerate(self.connections): if self.mergelayers[connum] is None: # conn is the index of an element in the input list. Fetch its shape: outshape.append(self.inpshape[pyk.delist(pyk.obj2list(conn))]) else: if self.merge: # Fetch merge layer's inpshape self.mergelayers[connum].inpshape = [ inpshape[node] for node in conn ] # Append outshape outshape.append(self.mergelayers[connum].outshape) else: outshape.append([inpshape[node] for node in conn]) return pyk.delist(outshape)
def inferoutshape(self, inpshape=None, checkinput=False): if inpshape is None: inpshape = self.inpshape if checkinput: assert len(pyk.obj2list(inpshape[0])) == 1, "Input shape must be a list of ints " \ "(split layer takes in 1 input)" # Recall that outshape must be a list of lists outshape = [] shape = inpshape indsplits = [0] + self.splits + [inpshape[-1]] for n in range(len(indsplits) - 1): # Copy shape (this is necessary, because python sets by reference) shape = copy.copy(shape) # Update channel size for all outputs shape[(1 if self.inpdim == 4 else 2)] = indsplits[n + 1] - indsplits[n] \ if indsplits[n + 1] is not None else None outshape.append(shape) # Return return outshape
def inferoutshape(self, inpshape=None, checkinput=False): if inpshape is None: inpshape = self.inpshape if checkinput: assert len(pyk.obj2list(inpshape[0])) == 1, "Input shape must be a list of ints " \ "(split layer takes in 1 input)" # Recall that outshape must be a list of lists outshape = [] shape = inpshape indsplits = [0] + self.splits + [inpshape[-1]] for n in range(len(indsplits) - 1): # Copy shape (this is necessary, because python sets by reference) shape = copy.copy(shape) # Update channel size for all outputs shape[(1 if self.inpdim == 4 else 2)] = indsplits[n + 1] - indsplits[n] \ if indsplits[n + 1] is not None else None outshape.append(shape) # Return return outshape
def __init__(self, func, shapefunc=None, funcargs=None, funckwargs=None, shapefuncargs=None, shapefunckwargs=None, numinp=1, numout=1, dim=None, issequence=None, inpshape=None): """ Layer to apply any given function to input(s). The function may require multiple inputs and return multiple outputs. The function may change the shape of the tensor, but then a `shapefunc` must be provided for automatic shape inference. :type func: callable :param func: Function to apply to input. :type shapefunc: callable :param shapefunc: Function to compute the output shape given input shape. :type funcargs: tuple or list :param funcargs: Arguments for the function (`func`) :type funckwargs: dict :param funckwargs: Keyword arguments for the function (`func`) :type shapefuncargs: tuple or list :param shapefuncargs: Arguments for the shape function (`shapefunc`) :type shapefunckwargs: dict :param shapefunckwargs: Keyword arguments for the shape function (`shapefunc`) :type numinp: int :param numinp: Number of inputs the function `func` takes. :type numout: int :param numout: Number of outputs the function `func` returns. :type dim: int or list of int :param dim: Dimensionality of the input data. Defaults to 2 when omitted. :type issequence: bool or list of bool :param issequence: Whether the input(s) is sequential. """ super(functionlayer, self).__init__() # Defaults shapefunc = (lambda x: x) if shapefunc is None else shapefunc dim = 2 if dim is None else dim issequence = False if issequence is None else issequence # Parse layer spec parsey = netutils.parselayerinfo(dim=dim, allowsequences=True, numinp=numinp, issequence=issequence, inpshape=inpshape) self.dim = parsey['dim'] self.inpdim = parsey['inpdim'] self.allowsequences = parsey['allowsequences'] self.issequence = parsey['issequence'] # Meta self.func = func self.funcargs = [] if funcargs is None else list(funcargs) self.funckwargs = {} if funckwargs is None else dict(funckwargs) self.shapefunc = shapefunc self.shapefuncargs = [] if shapefuncargs is None else list( shapefuncargs) self.shapefunckwargs = {} if shapefunckwargs is None else dict( shapefunckwargs) # Structure inference self.numinp = parsey['numinp'] self.numout = numout # Shape inference self.inpshape = parsey['inpshape'] # Containers for X and Y self.x = pyk.delist([ T.tensor('floatX', [ False, ] * indim, name='x{}:'.format(inpnum) + str(id(self))) for inpnum, indim in enumerate(pyk.obj2list(self.inpdim)) ]) self.y = pyk.delist([ T.tensor('floatX', [ False, ] * oudim, name='x{}:'.format(outnum) + str(id(self))) for outnum, oudim in enumerate(pyk.obj2list(self.outdim)) ])
def __init__(self, activation, dim=2, issequence=False, inpshape=None): """ :type activation: callable or dict :param activation: Activation function (any element-wise symbolic function) :type dim: int :param dim: Dimensionality of the input data :type issequence: bool :param issequence: Whether the input is a sequence :type inpshape: list :param inpshape: Input shape """ super(activationlayer, self).__init__() # Parse data dimensionality assert not ( dim is None and inpshape is None ), "Data dimension can not be parsed. Provide dim or inpshape." # Meta self.dim = dim if dim is not None else {4: 2, 5: 3}[len(inpshape)] self.allowsequences = True self.issequence = self.dim == 2 and len( inpshape) == 5 if issequence is None else issequence self.inpdim = len( inpshape) if inpshape is not None else 5 if self.issequence else { 2: 4, 3: 5 }[dim] # Parse activation if isinstance(activation, dict): self.activation = activation["function"] self._params = pyk.obj2list( activation["trainables"]) if "trainables" in activation.keys( ) else [] self._cparams = pyk.obj2list(activation["ctrainables"]) if "ctrainables" in activation.keys() else \ [netutils.getshared(like=trainable, value=1.) for trainable in self.params] # Name shared variables in params / cparams for n, (param, cparam) in enumerate(zip(self.params, self.cparams)): param.name += "-trainable{}:".format(n) + str(id(self)) cparam.name += "-ctrainable{}:".format(n) + str(id(self)) else: self.activation = activation # Shape inference self.inpshape = [ None, ] * self.inpdim if inpshape is None else list(inpshape) # Containers for input and output self.x = T.tensor('floatX', [ False, ] * self.inpdim, name='x:' + str(id(self))) self.y = T.tensor('floatX', [ False, ] * self.inpdim, name='y:' + str(id(self)))
def __init__(self, numinp=None, dim=None, issequence=None, inpshape=None): """ :type numinp: int :param numinp: Number of inputs :type dim: list or int :param dim: List (or int) of data dimensions of the input :type issequence: list or bool :param issequence: Whether inputs are sequences :type inpshape: list or tuple :param inpshape: Input shape :return: """ super(addlayer, self).__init__() # Convenience parse dim = [ dim, ] * numinp if isinstance(dim, int) and numinp is not None else dim issequence = [ issequence, ] * numinp if isinstance(issequence, bool) and numinp is not None else issequence # Parse inpshape if numinp, dim, issequence is given if numinp is not None and dim is not None and inpshape is None: if 3 in dim: # issequence doesn't matter, inpshape can still be filled inpshape = [[ None, ] * 5 for _ in dim] else: # dim is 2, but input might still be sequential if issequence is None: pass else: if issequence: # issequence is false and dim = 2, ergo 2D data inpshape = [[ None, ] * 4 for _ in dim] else: # issequence is true, ergo 2D sequential data inpshape = [[ None, ] * 5 for _ in dim] # Check assert not (numinp is None and inpshape is None), "Number of inputs could not be parsed. Provide numinp " \ "or inpshape." assert not (all([idim is None for idim in pyk.obj2list(dim)]) and inpshape is None), \ "Data dimension could not be parsed. Please provide dim or inpshape." assert isinstance( dim, list ) if dim is not None else True, "Dim must be a list for multiple inputs." assert all([isinstance(ishp, list) for ishp in inpshape]) if inpshape is not None else True, \ "mergelayer expects multiple inputs, but inpshape doesn't have the correct signature." # Meta self.numinp = numinp if numinp is not None else len(inpshape) self.numout = 1 self.dim = dim if dim is not None else [{ 4: 2, 5: 2 }[len(ishp)] for ishp in inpshape] self.allowsequences = True self.issequence = [idim == 2 and len(ishp) == 5 for idim, ishp in zip(self.dim, inpshape)] \ if issequence is None else issequence self.inpdim = [ len(ishp) if ishp is not None else 5 if iisseq else { 2: 4, 3: 5 }[idim] for ishp, iisseq, idim in zip(inpshape, self.issequence, self.dim) ] # Check if inpdim is consistent assert all([indim == self.inpdim[0] for indim in self.inpdim ]), "Can't concatenate 2D with 3D inputs." # Shape inference self.inpshape = list(inpshape) if inpshape is not None else [[ None, ] * indim for indim in self.inpdim] # Containers for input and output self.x = [ T.tensor('floatX', [ False, ] * indim, name='x{}:'.format(inpnum) + str(id(self))) for inpnum, indim in enumerate(self.inpdim) ] self.y = T.tensor('floatX', [ False, ] * self.inpdim[0], name='y:' + str(id(self)))
def __init__(self, connections, merge=True, dim=2, issequence=False, inpshape=None): """ :type connections: list :param connections: List of connections. E.g.: [[0, 1], 1, [0, 1, 2]] would merge inputs 0 & 1 and write to first output slot, 1 to second output slot and 0, 1 & 2 merged to the third output slot. :param dim: :param issequence: :param inpshape: :return: """ super(circuitlayer, self).__init__() # Compute the number of i/o slots self.numinp = len(pyk.unique(list(pyk.flatten(connections)))) self.numout = len(connections) self.merge = bool(merge) # TODO Fix this assert self.numinp != 1, "Circuit layer must have atleast 2 inputs. Consider using a replicatelayer." # Parse dim = [ dim, ] * self.numinp if isinstance(dim, int) and self.numinp is not None else dim issequence = [issequence, ] * self.numinp if isinstance(issequence, bool) and self.numinp is not None \ else issequence # Parse inpshape if numinp, dim, issequence is given if self.numinp is not None and dim is not None and inpshape is None: if 3 in dim: # issequence doesn't matter, inpshape can still be filled inpshape = [[ None, ] * 5 for _ in dim] else: # dim is 2, but input might still be sequential if issequence is None: pass else: if issequence: # issequence is false and dim = 2, ergo 2D data inpshape = [[ None, ] * 4 for _ in dim] else: # issequence is true, ergo 2D sequential data inpshape = [[ None, ] * 5 for _ in dim] # Meta self.connections = connections self.dim = dim if dim is not None else [{ 4: 2, 5: 2 }[len(ishp)] for ishp in inpshape] self.allowsequences = True self.issequence = [idim == 2 and len(ishp) == 5 for idim, ishp in zip(self.dim, inpshape)] \ if issequence is None else issequence self.inpdim = [ len(ishp) if ishp is not None else 5 if iisseq else { 2: 4, 3: 5 }[idim] for ishp, iisseq, idim in zip(inpshape, self.issequence, self.dim) ] # Check if inpdim is consistent assert all([indim == self.inpdim[0] for indim in self.inpdim ]), "Can't concatenate 2D with 3D inputs." # Make merge layers self.mergelayers = [ mergelayer(numinp=len(conn), dim=[self.dim[node] for node in conn], issequence=[self.issequence[node] for node in conn], inpshape=[inpshape[node] for node in conn]) if pyk.smartlen(conn) != 1 else None for conn in self.connections ] # Shape inference self.inpshape = list(inpshape) if inpshape is not None else [[ None, ] * indim for indim in self.inpdim] # Containers for input and output self.x = pyk.delist([ T.tensor('floatX', [ False, ] * indim, name='x{}:'.format(inpnum) + str(id(self))) for inpnum, indim in enumerate(self.inpdim) ]) self.y = pyk.delist([ T.tensor('floatX', [ False, ] * self.inpdim[pyk.obj2list(self.connections[outnum])[0]], name='y:' + str(id(self))) for outnum in range(self.numout) ])
def __theano__conv(self, inp, filters, stride=None, dilation=None, padding=None, bias=None, filtergradmask=None, biasgradmask=None, filtermask=None, biasmask=None, filtergradclips=None, biasgradclips=None, dim=None, convmode='same', issequence=False, implementation='auto'): # Do imports locally to prevent circular dependencies import netutils as nu import theanops as tho import pykit as pyk # Determine the dimensionality of convolution (2 or 3?) if dim is None: dim = 3 if not issequence and len( filters.get_value().shape) == 5 and inp.ndim == 5 else 2 # Smart fix: if convmode is 'same', stride != 1 and padding is None: set automagically set padding. # Defaults padding = [[0, 0]] * dim if padding is None else padding stride = [1] * dim if stride is None else stride dilation = [1] * dim if dilation is None else dilation filtergradclips = [ -np.inf, np.inf ] if filtergradclips is None else list(filtergradclips) biasgradclips = [-np.inf, np.inf ] if biasgradclips is None else list(biasgradclips) # Autofix inputs if isinstance(padding, int): padding = [padding] * dim if not pyk.islistoflists(pyk.obj2list(padding)): padding = [[padval] * dim for padval in pyk.obj2list(padding)] if isinstance(stride, int): stride = [stride] * dim if isinstance(dilation, int): dilation = [dilation] * dim # TODO: Tests pass # Reshape 2D sequential data if required # Log input shape inpshape = inp.shape reallyissequential = issequence and inp.ndim == 5 if issequence: if reallyissequential: inp = inp.reshape((inpshape[0] * inpshape[1], inpshape[2], inpshape[3], inpshape[4]), ndim=4) stride = stride[0:2] padding = padding[0:2] # TODO: Get rid of these restrictions assert stride == [ 1, 1 ], "Strided convolution is not implemented for sequential data." assert convmode == 'same', "Convmode must be 'same' for sequential data." else: warn( "Expected 5D sequential output, but got 4D non-sequential instead." ) # Apply gradient masks if required if filtergradmask is not None: filters = tho.maskgradient(filters, filtergradmask) if biasgradmask is not None and bias is not None: bias = tho.maskgradient(bias, biasgradmask) # Apply masks if required if filtermask is not None: filters = filtermask * filters if biasmask is not None: bias = biasmask * bias # Determine border_mode for CuDNN/3D conv autopaddable, bordermode, trim = self.__theano__bordermode( convmode, padding, filters.get_value().shape) # Pad input if required (warn that it's ridiculously slow) if not autopaddable and not all( [padval == 0 for padval in pyk.flatten(padding)]): if not isinstance(bordermode, str) and pyk.islistoflists(bordermode): # Override padding for 3D convolutions inp = nu.pad(inp, bordermode) bordermode = 'valid' else: inp = nu.pad(inp, padding) # Switch implementation if implementation == 'auto': # Fall back implementation: 'vanilla' implementation = 'vanilla' if dilation != [1, 1]: implementation = 'dilated' # Convolve 2D (with gradmask + bias), reshape sequential data if dim == 2: if implementation == 'vanilla': if list(dilation) != [1, 1]: warn( "Filter dilation is not possible with this implementation." ) # Convolve y = T.nnet.conv2d(input=inp, filters=th.gradient.grad_clip( filters, *filtergradclips), border_mode=tuple(bordermode) if isinstance( bordermode, list) else bordermode, filter_shape=filters.get_value().shape, subsample=tuple(stride)) elif implementation == 'dilated': # Make sure stride is 1 assert list(stride) == [ 1, 1 ], "Stride should equal [1, 1] for dilated convolutions." assert not issequence, "Dilated convolution is not supported for sequential data." # Dilated conv can't handle padding at the moment, do this manually if isinstance(bordermode, tuple): padding = [[bm, bm] for bm in bordermode] inp = nu.pad(inp, padding) elif bordermode == 'full': raise NotImplementedError( "Convolution mode 'full' is not implemented for dilated convolutions." ) elif bordermode == 'valid': pass elif bordermode == 'half': assert all([d % 2 == 0 for d in dilation]), "Dilation amount must be divisible by 2 for dilated " \ "convolution with 'same' border handling." padding = [[ (filters.get_value().shape[n] - 1) * d / 2, ] * 2 for n, d in zip([2, 3], dilation)] inp = nu.pad(inp, padding) else: raise NotImplementedError( "Unknown bordermode: {}.".format(bordermode)) # Get output image shape oishp = [ inp.shape[n] - (filters.shape[n] - 1) * d for n, d in zip([2, 3], dilation) ] # Get computin' op = T.nnet.abstract_conv.AbstractConv2d_gradWeights( subsample=tuple(dilation), border_mode='valid', filter_flip=False) y = op(inp.transpose(1, 0, 2, 3), filters.transpose(1, 0, 2, 3), tuple(oishp)) y = y.transpose(1, 0, 2, 3) else: raise NotImplementedError( "Implementation {} is not implemented.".format( implementation)) # Trim if required if trim: y = self.__theano__convtrim( inp=y, filtershape=filters.get_value().shape) # Add bias if required if bias is not None: y = y + th.gradient.grad_clip(bias, *biasgradclips).dimshuffle( 'x', 0, 'x', 'x') elif dim == 3: # Convolve 3D (with bias) if implementation == 'auto' or implementation == 'conv2d': assert stride == [ 1, 1, 1 ], "Implementation 'conv2d' does not support strided convolution in 3D." assert convmode == 'valid', "Implementation 'conv2d' only supports 'valid' convolutions." y = T.nnet.conv3d2d.conv3d( signals=inp, filters=th.gradient.grad_clip(filters, *filtergradclips), border_mode=bordermode, filters_shape=filters.get_value().shape) else: raise NotImplementedError( "Implementation {} is not implemented.".format( implementation)) # Trim if required if trim: y = self.__theano__convtrim( inp=y, filtershape=filters.get_value().shape) # Add bias if required if bias is not None: y = y + th.gradient.grad_clip(bias, *biasgradclips).dimshuffle( 'x', 'x', 0, 'x', 'x') else: raise NotImplementedError( "Convolution is implemented in 2D and 3D.") # Reshape sequential data if issequence and reallyissequential: y = y.reshape( (inpshape[0], inpshape[1], filters.get_value().shape[0], inpshape[3], inpshape[4]), ndim=5) # Return return y
def __init__(self, connections, merge=True, dim=2, issequence=False, inpshape=None): """ :type connections: list :param connections: List of connections. E.g.: [[0, 1], 1, [0, 1, 2]] would merge inputs 0 & 1 and write to first output slot, 1 to second output slot and 0, 1 & 2 merged to the third output slot. :param dim: :param issequence: :param inpshape: :return: """ super(circuitlayer, self).__init__() # Compute the number of i/o slots self.numinp = len(pyk.unique(list(pyk.flatten(connections)))) self.numout = len(connections) self.merge = bool(merge) # Parse dim = [dim, ] * self.numinp if isinstance(dim, int) and self.numinp is not None else dim issequence = [issequence, ] * self.numinp if isinstance(issequence, bool) and self.numinp is not None \ else issequence # Parse inpshape if numinp, dim, issequence is given if self.numinp is not None and dim is not None and inpshape is None: if 3 in dim: # issequence doesn't matter, inpshape can still be filled inpshape = [[None, ] * 5 for _ in dim] else: # dim is 2, but input might still be sequential if issequence is None: pass else: if issequence: # issequence is false and dim = 2, ergo 2D data inpshape = [[None, ] * 4 for _ in dim] else: # issequence is true, ergo 2D sequential data inpshape = [[None, ] * 5 for _ in dim] # Meta self.connections = connections self.dim = dim if dim is not None else [{4: 2, 5: 2}[len(ishp)] for ishp in inpshape] self.allowsequences = True self.issequence = [idim == 2 and len(ishp) == 5 for idim, ishp in zip(self.dim, inpshape)] \ if issequence is None else issequence self.inpdim = [len(ishp) if ishp is not None else 5 if iisseq else {2: 4, 3: 5}[idim] for ishp, iisseq, idim in zip(inpshape, self.issequence, self.dim)] # Check if inpdim is consistent assert all([indim == self.inpdim[0] for indim in self.inpdim]), "Can't concatenate 2D with 3D inputs." # Make merge layers self.mergelayers = [mergelayer(numinp=len(conn), dim=[self.dim[node] for node in conn], issequence=[self.issequence[node] for node in conn], inpshape=[inpshape[node] for node in conn]) if pyk.smartlen(conn) != 1 else None for conn in self.connections] # Shape inference self.inpshape = list(inpshape) if inpshape is not None else [[None, ] * indim for indim in self.inpdim] # Containers for input and output self.x = pyk.delist([T.tensor('floatX', [False, ] * indim, name='x{}:'.format(inpnum) + str(id(self))) for inpnum, indim in enumerate(self.inpdim)]) self.y = pyk.delist([T.tensor('floatX', [False, ] * self.inpdim[pyk.obj2list(self.connections[outnum])[0]], name='y:' + str(id(self))) for outnum in range(self.numout)])
def parselayerinfo(dim=None, inpdim=None, issequence=None, allowsequences=None, numinp=None, inpshape=None, verbose=True): parsey = { 'dim': dim, 'inpdim': inpdim, 'issequence': issequence, 'allowsequences': allowsequences, 'numinp': numinp, 'inpshape': inpshape } # Parse from inpshape if parsey['inpshape'] is not None: # Make sure inpshape is a list assert isinstance( parsey['inpshape'], list), "inpshape must be a list, e.g. [None, 3, None, None]." # Fetch number of inputs if pyk.islistoflists(parsey['inpshape']): _numinp = len(parsey['inpshape']) _inpdim = [len(ishp) for ishp in parsey['inpshape']] else: _numinp = 1 _inpdim = len(parsey['inpshape']) # Write to parsed (if not written already) # numinp parsey['numinp'] = _numinp if parsey['numinp'] is None else parsey[ 'numinp'] # Consistency check assert parsey['numinp'] == _numinp, "The provided inpshape requires numinp = {}, " \ "but the value given was {}".format(_numinp, parsey['numinp']) # inpdim parsey['inpdim'] = _inpdim if parsey['inpdim'] is None else parsey[ 'inpdim'] assert parsey['inpdim'] == _inpdim, "The provided inpshape requires inpdim = {}, " \ "but the value given was {}".format(_inpdim, parsey['inpdim']) # Check if dim, inpdim, issequence or allowsequences is a list of multiple elements and numinp is not given. if parsey['numinp'] is None: for argname in ['dim', 'inpdim', 'issequence', 'allowsequences']: if isinstance(parsey[argname], list): parsey['numinp'] = len(parsey[argname]) # Parse from numinp if parsey['numinp'] is not None: for argname, argtype in zip( ['dim', 'inpdim', 'issequence', 'allowsequences'], [int, int, bool, bool]): if isinstance(parsey[argname], argtype) and parsey['numinp'] > 1: # If numinp is > 1 and allowseqences or issequence or inpdim or dim is bool or bool or int or int, # assume that the user (or the author) is too lazy to type in a list. parsey[argname] = [ parsey[argname], ] * parsey['numinp'] elif isinstance(parsey[argname], list) and parsey['numinp'] > 1: # If the user was not lazy, make sure the given list sizes check out assert len(parsey[argname]) == parsey['numinp'], \ "{} must be a {} or a list of length {} (= numinp).".format(argname, argtype, parsey['numinp']) # Check if inpshape is consistent if parsey['inpshape'] is not None and parsey['numinp'] > 1: assert pyk.islistoflists(parsey['inpshape']) and len( parsey['inpshape']) == parsey['numinp'] else: if verbose: warn("Guessing that numinp = 1.") # Guess numinp = 1. parsey['numinp'] = 1 # Parse allowsequences # At this point, allowsequences must be known (or no conclusions can be drawn on issequence and dim) if parsey['allowsequences'] is None: if verbose: warn("Guessing that sequences are allowed.") parsey['allowsequences'] = pyk.delist([ True, ] * parsey['numinp']) else: # Okay, so it's known if sequences are allowed. Check if issequence is consistent. if pyk.obj2list(parsey['allowsequences']) == [ False, ] * parsey['numinp'] and parsey['issequence'] is not None: # If sequences are not allowed, make sure issequence is False assert pyk.obj2list(parsey['issequence']) == [False,] * parsey['numinp'], \ "Input(s) are not allowed to be sequential, yet they are." # Parse issequence if parsey['issequence'] is not None: # Delist issequence parsey['issequence'] = pyk.delist(parsey['issequence']) \ if isinstance(parsey['issequence'], list) else parsey['issequence'] # Check if issequence is consistent with everything if isinstance(parsey['issequence'], list): assert len(parsey['issequence']) == parsey['numinp'], "issequence must be a list of the same lenght as " \ "numinp = {} if numinp > 1.".format(parsey['numinp']) # Check if consistent with allowsequences. At this point, issequence may have None's. assert all([(bool(isseq) and allowseq) or not isseq for isseq, allowseq in zip(pyk.obj2list(parsey['issequence']), pyk.obj2list(parsey['allowsequences']))]), \ "Input is a sequence although it's not allowed to. " \ "issequence = {}, allowsequences = {}.".format(parsey['issequence'], parsey['allowsequences']) else: if verbose: warn("Guessing that input(s) is(are) not sequential.") parsey['issequence'] = pyk.delist([ False, ] * parsey['numinp']) # Parse inpdim # Compute expected inpdim from what's known # Check in from issequence _inpdim = pyk.delist( [5 if isseq else None for isseq in pyk.obj2list(parsey['issequence'])]) # Check in from dim if parsey['dim'] is not None: _inpdim = pyk.delist([ 5 if d == 3 else indim for d, indim in zip( pyk.obj2list(parsey['dim']), pyk.obj2list(_inpdim)) ]) _inpdim = pyk.delist([ 4 if (d == 2 and not isseq) else indim for d, indim, isseq in zip( pyk.obj2list(parsey['dim']), pyk.obj2list(_inpdim), pyk.obj2list(parsey['issequence'])) ]) if parsey['inpdim'] is None: # Make sure there are no None's remaining in _inpdim assert None not in pyk.obj2list( _inpdim ), "Input dimensionality could not be parsed due to missing information." parsey['inpdim'] = _inpdim else: assert pyk.smartlen(parsey['inpdim']) == pyk.smartlen(_inpdim), \ "Expected {} elements in inpdim, got {}.".format(pyk.smartlen(_inpdim), pyk.smartlen(parsey['inpdim'])) # Check consistency with the expected _inpdim assert all([_indim == indim for _indim, indim in zip(pyk.obj2list(_inpdim), pyk.obj2list(parsey['inpdim'])) if _indim is not None]), \ "Provided inpdim is inconsistent with either dim or issequence." # Parse dim # Compute expected _inpdim from what's known _dim = pyk.delist([ 2 if (indim == 4 or isseq) else 3 for indim, isseq in zip( pyk.obj2list(parsey['inpdim']), pyk.obj2list(parsey['issequence'])) ]) # Check in from dim if parsey['dim'] is None: parsey['dim'] = _dim else: assert parsey[ 'dim'] == _dim, "Given dim ({}) is not consistent with expectation ({})".format( parsey['dim'], _dim) # Reparse inpshape if parsey['inpshape'] is None: parsey['inpshape'] = pyk.delist([[ None, ] * indim for indim in pyk.obj2list(parsey['inpdim'])]) # Return parsey :( return parsey
def __init__(self, numinp=None, dim=None, issequence=None, inpshape=None): """ :type numinp: int :param numinp: Number of inputs :type dim: list or int :param dim: List (or int) of data dimensions of the input :type issequence: list or bool :param issequence: Whether inputs are sequences :type inpshape: list or tuple :param inpshape: Input shape :return: """ super(addlayer, self).__init__() # Convenience parse dim = [dim, ] * numinp if isinstance(dim, int) and numinp is not None else dim issequence = [issequence, ] * numinp if isinstance(issequence, bool) and numinp is not None else issequence # Parse inpshape if numinp, dim, issequence is given if numinp is not None and dim is not None and inpshape is None: if 3 in dim: # issequence doesn't matter, inpshape can still be filled inpshape = [[None, ] * 5 for _ in dim] else: # dim is 2, but input might still be sequential if issequence is None: pass else: if issequence: # issequence is false and dim = 2, ergo 2D data inpshape = [[None, ] * 4 for _ in dim] else: # issequence is true, ergo 2D sequential data inpshape = [[None, ] * 5 for _ in dim] # Check assert not (numinp is None and inpshape is None), "Number of inputs could not be parsed. Provide numinp " \ "or inpshape." assert not (all([idim is None for idim in pyk.obj2list(dim)]) and inpshape is None), \ "Data dimension could not be parsed. Please provide dim or inpshape." assert isinstance(dim, list) if dim is not None else True, "Dim must be a list for multiple inputs." assert all([isinstance(ishp, list) for ishp in inpshape]) if inpshape is not None else True, \ "mergelayer expects multiple inputs, but inpshape doesn't have the correct signature." # Meta self.numinp = numinp if numinp is not None else len(inpshape) self.numout = 1 self.dim = dim if dim is not None else [{4: 2, 5: 2}[len(ishp)] for ishp in inpshape] self.allowsequences = True self.issequence = [idim == 2 and len(ishp) == 5 for idim, ishp in zip(self.dim, inpshape)] \ if issequence is None else issequence self.inpdim = [len(ishp) if ishp is not None else 5 if iisseq else {2: 4, 3: 5}[idim] for ishp, iisseq, idim in zip(inpshape, self.issequence, self.dim)] # Check if inpdim is consistent assert all([indim == self.inpdim[0] for indim in self.inpdim]), "Can't concatenate 2D with 3D inputs." # Shape inference self.inpshape = list(inpshape) if inpshape is not None else [[None, ] * indim for indim in self.inpdim] # Containers for input and output self.x = [T.tensor('floatX', [False, ] * indim, name='x{}:'.format(inpnum) + str(id(self))) for inpnum, indim in enumerate(self.inpdim)] self.y = T.tensor('floatX', [False, ] * self.inpdim[0], name='y:' + str(id(self)))
def __theano__conv(self, inp, filters, stride=None, padding=None, bias=None, filtergradmask=None, biasgradmask=None, filtermask=None, biasmask=None, filtergradclips=None, biasgradclips=None, dim=None, convmode='same', issequence=False, implementation='auto'): # Do imports locally to prevent circular dependencies import netutils as nu import theanops as tho import pykit as pyk # Determine the dimensionality of convolution (2 or 3?) if dim is None: dim = 3 if not issequence and len( filters.get_value().shape) == 5 and inp.ndim == 5 else 2 # Smart fix: if convmode is 'same', stride != 1 and padding is None: set automagically set padding. # Defaults padding = [[0, 0]] * dim if padding is None else padding stride = [1] * dim if stride is None else stride filtergradclips = [ -np.inf, np.inf ] if filtergradclips is None else list(filtergradclips) biasgradclips = [-np.inf, np.inf ] if biasgradclips is None else list(biasgradclips) # Autofix inputs if isinstance(padding, int): padding = [padding] * dim if not pyk.islistoflists(pyk.obj2list(padding)): padding = [[padval] * dim for padval in pyk.obj2list(padding)] if isinstance(stride, int): stride = [stride] * dim # TODO: Tests pass # Reshape 2D sequential data if required # Log input shape inpshape = inp.shape reallyissequential = issequence and inp.ndim == 5 if issequence: if reallyissequential: inp = inp.reshape((inpshape[0] * inpshape[1], inpshape[2], inpshape[3], inpshape[4]), ndim=4) stride = stride[0:2] padding = padding[0:2] # TODO: Get rid of these restrictions assert stride == [ 1, 1 ], "Strided convolution is not implemented for sequential data." assert convmode == 'same', "Convmode must be 'same' for sequential data." else: warn( "Expected 5D sequential output, but got 4D non-sequential instead." ) # Apply gradient masks if required if filtergradmask is not None: filters = tho.maskgradient(filters, filtergradmask) if biasgradmask is not None and bias is not None: bias = tho.maskgradient(bias, biasgradmask) # Apply masks if required if filtermask is not None: filters = filtermask * filters if biasmask is not None: bias = biasmask * bias # Determine border_mode for CuDNN/3D conv autopaddable, bordermode, trim = self.__theano__bordermode( convmode, padding, filters.get_value().shape) # Pad input if required (warn that it's ridiculously slow) if not autopaddable and not all( [padval == 0 for padval in pyk.flatten(padding)]): if not th.config.device == 'cpu' and not self.cpupadwarned: warn( "Padding might occur on the CPU, which tends to slow things down." ) self.cpupadwarned = True if not isinstance(bordermode, str) and pyk.islistoflists(bordermode): # Override padding for 3D convolutions inp = nu.pad(inp, bordermode) bordermode = 'valid' else: inp = nu.pad(inp, padding) # Convolve 2D (with gradmask + bias), reshape sequential data if dim == 2: if implementation == 'auto': # Convolve y = T.nnet.conv2d(input=inp, filters=th.gradient.grad_clip( filters, *filtergradclips), border_mode=tuple(bordermode) if isinstance( bordermode, list) else bordermode, filter_shape=filters.get_value().shape, subsample=tuple(stride)) else: raise NotImplementedError( "Implementation {} is not implemented.".format( implementation)) # Trim if required if trim: y = self.__theano__convtrim( inp=y, filtershape=filters.get_value().shape) # Add bias if required if bias is not None: y = y + th.gradient.grad_clip(bias, *biasgradclips).dimshuffle( 'x', 0, 'x', 'x') elif dim == 3: # Convolve 3D (with bias) if implementation == 'auto' or implementation == 'conv2d': assert stride == [ 1, 1, 1 ], "Implementation 'conv2d' does not support strided convolution in 3D." assert convmode == 'valid', "Implementation 'conv2d' only supports 'valid' convolutions." y = T.nnet.conv3d2d.conv3d( signals=inp, filters=th.gradient.grad_clip(filters, *filtergradclips), border_mode=bordermode, filters_shape=filters.get_value().shape) else: raise NotImplementedError( "Implementation {} is not implemented.".format( implementation)) # Trim if required if trim: y = self.__theano__convtrim( inp=y, filtershape=filters.get_value().shape) # Add bias if required if bias is not None: y = y + th.gradient.grad_clip(bias, *biasgradclips).dimshuffle( 'x', 'x', 0, 'x', 'x') else: raise NotImplementedError( "Convolution is implemented in 2D and 3D.") # Reshape sequential data if issequence and reallyissequential: y = y.reshape( (inpshape[0], inpshape[1], filters.get_value().shape[0], inpshape[3], inpshape[4]), ndim=5) # Return return y
def __theano__pool(self, inp, ds, stride=None, padding=None, poolmode='max', dim=None, ignoreborder=True, issequence=False): # Do imports locally to prevent circular dependencies import netutils as nu import pykit as pyk from theano.tensor.signal import downsample # Determine the dimensionality of convolution (2 or 3?) if dim is None: dim = 3 if not issequence and len(ds) == 3 and inp.ndim == 5 else 2 # Defaults poolmode = 'average_exc_pad' if poolmode in ['mean', 'average', 'average_exc_pad'] else poolmode padding = [[0, 0]] * dim if padding is None else padding stride = ds if stride is None else stride # Autofix inputs if isinstance(padding, int): padding = [padding] * dim if not pyk.islistoflists(pyk.obj2list(padding)): padding = [[padval] * dim for padval in pyk.obj2list(padding)] if isinstance(stride, int): stride = [stride] * dim # Check if theano can pad input as required autopaddable = all([all([dimpad == pad[0] for dimpad in pad]) for pad in padding]) # Reshape 2D sequential data if required # Log input shape inpshape = inp.shape reallyissequential = issequence and inp.ndim == 5 if issequence: if reallyissequential: # Sequential input must be paddable by theano. This is required to reshape the sequential input back to # its original shape after pooling. assert autopaddable, "Sequential inputs must be paddable by theano. Provided padding {} cannot be " \ "handled at present.".format(padding) inp = inp.reshape((inpshape[0] * inpshape[1], inpshape[2], inpshape[3], inpshape[4]), ndim=4) ds = ds[0:2] stride = stride[0:2] padding = padding[0:2] else: warn("Expected 5D sequential output, but got 4D non-sequential instead.") # Determine what theano needs to be told about how to pad the input if autopaddable: autopadding = tuple([pad[0] for pad in padding]) else: autopadding = (0,) * dim if not autopaddable and not all([padval == 0 for padval in pyk.flatten(padding)]): if not th.config.device == 'cpu' and not self.cpupadwarned: warn("Padding might occur on the CPU, which tends to slow things down.") self.cpupadwarned = True inp = nu.pad(inp, padding) if dim == 2: y = downsample.max_pool_2d(input=inp, ds=ds, st=stride, padding=autopadding, ignore_border=ignoreborder, mode=poolmode) elif dim == 3: # parse downsampling ratio, stride and padding dsyx = ds[0:2] styx = stride[0:2] padyx = autopadding[0:2] ds0z = (1, ds[2]) st0z = (1, stride[2]) pad0z = (0, autopadding[2]) # Dowsnample yx H = downsample.max_pool_2d(input=inp, ds=dsyx, st=styx, padding=padyx, mode=poolmode) # Rotate tensor H = H.dimshuffle(0, 2, 3, 4, 1) # Downsample 0z H = downsample.max_pool_2d(input=H, ds=ds0z, st=st0z, padding=pad0z, mode=poolmode) # Undo rotate tensor y = H.dimshuffle(0, 4, 1, 2, 3) else: raise NotImplementedError("Pooling is implemented in 2D and 3D.") if issequence and reallyissequential: # Compute symbolic pool output length if ignoreborder: pooleny, poolenx = \ [T.floor((inpshape[tensorindex] + 2 * autopadding[index] - ds[index] + stride[index])/stride[index]) for index, tensorindex in enumerate([3, 4])] else: poolen = [None, None] for index, tensorindex in enumerate([3, 4]): if stride[index] >= ds[index]: poolen[index] = T.floor((inpshape[tensorindex] + stride[index] - 1)/stride[index]) else: plen = T.floor((inpshape[tensorindex] - ds[index] + stride[index] - 1)/stride[index]) poolen[index] = T.switch(plen > 0, plen, 0) pooleny, poolenx = poolen y = y.reshape((inpshape[0], inpshape[1], inpshape[2], pooleny, poolenx), ndim=5) return y
def __theano__conv(self, inp, filters, stride=None, padding=None, bias=None, filtergradmask=None, biasgradmask=None, filtermask=None, biasmask=None, filtergradclips=None, biasgradclips=None, dim=None, convmode='same', issequence=False, implementation='auto'): # Do imports locally to prevent circular dependencies import netutils as nu import theanops as tho import pykit as pyk # Determine the dimensionality of convolution (2 or 3?) if dim is None: dim = 3 if not issequence and len(filters.get_value().shape) == 5 and inp.ndim == 5 else 2 # Smart fix: if convmode is 'same', stride != 1 and padding is None: set automagically set padding. # Defaults padding = [[0, 0]] * dim if padding is None else padding stride = [1] * dim if stride is None else stride filtergradclips = [-np.inf, np.inf] if filtergradclips is None else list(filtergradclips) biasgradclips = [-np.inf, np.inf] if biasgradclips is None else list(biasgradclips) # Autofix inputs if isinstance(padding, int): padding = [padding] * dim if not pyk.islistoflists(pyk.obj2list(padding)): padding = [[padval] * dim for padval in pyk.obj2list(padding)] if isinstance(stride, int): stride = [stride] * dim # TODO: Tests pass # Reshape 2D sequential data if required # Log input shape inpshape = inp.shape reallyissequential = issequence and inp.ndim == 5 if issequence: if reallyissequential: inp = inp.reshape((inpshape[0] * inpshape[1], inpshape[2], inpshape[3], inpshape[4]), ndim=4) stride = stride[0:2] padding = padding[0:2] # TODO: Get rid of these restrictions assert stride == [1, 1], "Strided convolution is not implemented for sequential data." assert convmode == 'same', "Convmode must be 'same' for sequential data." else: warn("Expected 5D sequential output, but got 4D non-sequential instead.") # Apply gradient masks if required if filtergradmask is not None: filters = tho.maskgradient(filters, filtergradmask) if biasgradmask is not None and bias is not None: bias = tho.maskgradient(bias, biasgradmask) # Apply masks if required if filtermask is not None: filters = filtermask * filters if biasmask is not None: bias = biasmask * bias # Determine border_mode for CuDNN/3D conv autopaddable, bordermode, trim = self.__theano__bordermode(convmode, padding, filters.get_value().shape) # Pad input if required (warn that it's ridiculously slow) if not autopaddable and not all([padval == 0 for padval in pyk.flatten(padding)]): if not th.config.device == 'cpu' and not self.cpupadwarned: warn("Padding might occur on the CPU, which tends to slow things down.") self.cpupadwarned = True if not isinstance(bordermode, str) and pyk.islistoflists(bordermode): # Override padding for 3D convolutions inp = nu.pad(inp, bordermode) bordermode = 'valid' else: inp = nu.pad(inp, padding) # Convolve 2D (with gradmask + bias), reshape sequential data if dim == 2: if implementation == 'auto': # Convolve y = T.nnet.conv2d(input=inp, filters=th.gradient.grad_clip(filters, *filtergradclips), border_mode=tuple(bordermode) if isinstance(bordermode, list) else bordermode, filter_shape=filters.get_value().shape, subsample=tuple(stride)) else: raise NotImplementedError("Implementation {} is not implemented.".format(implementation)) # Trim if required if trim: y = self.__theano__convtrim(inp=y, filtershape=filters.get_value().shape) # Add bias if required if bias is not None: y = y + th.gradient.grad_clip(bias, *biasgradclips).dimshuffle('x', 0, 'x', 'x') elif dim == 3: # Convolve 3D (with bias) if implementation == 'auto' or implementation == 'conv2d': assert stride == [1, 1, 1], "Implementation 'conv2d' does not support strided convolution in 3D." assert convmode == 'valid', "Implementation 'conv2d' only supports 'valid' convolutions." y = T.nnet.conv3d2d.conv3d(signals=inp, filters=th.gradient.grad_clip(filters, *filtergradclips), border_mode=bordermode, filters_shape=filters.get_value().shape) else: raise NotImplementedError("Implementation {} is not implemented.".format(implementation)) # Trim if required if trim: y = self.__theano__convtrim(inp=y, filtershape=filters.get_value().shape) # Add bias if required if bias is not None: y = y + th.gradient.grad_clip(bias, *biasgradclips).dimshuffle('x', 'x', 0, 'x', 'x') else: raise NotImplementedError("Convolution is implemented in 2D and 3D.") # Reshape sequential data if issequence and reallyissequential: y = y.reshape((inpshape[0], inpshape[1], filters.get_value().shape[0], inpshape[3], inpshape[4]), ndim=5) # Return return y
def slidingwindowslices(shape, nhoodsize, stride=1, ds=1, window=None, ignoreborder=True, shuffle=True, rngseed=None, startmins=None, startmaxs=None, shufflebuffersize=1000): """ Returns a generator yielding (shuffled) sliding window slice objects. :type shape: int or list of int :param shape: Shape of the input data :type nhoodsize: int or list of int :param nhoodsize: Window size of the sliding window. :type stride: int or list of int :param stride: Stride of the sliding window. :type window: list :param window: Configure the sliding window. Examples: With axistags 'yxz': - window = ['x', 'x', 'x'] ==> 3D sliding windows over the 3D volume - window = [[0, 1], 'x', 'x'] ==> 2D sliding window over the 0-th and 1-st xz planes - window = ['x', [8, 9], 'x'] ==> 2D sliding window over the 8-th and 9-st yz planes :type ignoreborder: bool :param ignoreborder: Whether to skip border windows (i.e. windows without enough pixels to fill the specified nhoodsize). :type shuffle: bool :param shuffle: Whether to shuffle the iterator. :type rngseed: int :param rngseed: Random number generator seed. Use to synchronize shuffled generators. :returns: A python generator whose next method yields a tuple of slices. """ # Determine dimensionality of the data datadim = len(shape) # Parse window if window is None: window = ['x'] * datadim else: assert len(window) == datadim, "Window must have the same length as the number of data dimensions." # Parse nhoodsize and stride nhoodsize = [nhoodsize, ] * datadim if isinstance(nhoodsize, int) else nhoodsize stride = [stride, ] * datadim if isinstance(stride, int) else stride ds = [ds, ] * datadim if isinstance(ds, int) else ds # Seed RNG if a seed is provided if rngseed is not None: random.seed(rngseed) # Define a function that gets a 1D slice def _1Dwindow(startmin, startmax, nhoodsize, stride, ds, seqsize, shuffle): starts = range(startmin, startmax + 1, stride) if ignoreborder: slices = [slice(st, st + nhoodsize, ds) for st in starts if st + nhoodsize <= seqsize] else: slices = [slice(st, ((st + nhoodsize) if st + nhoodsize <= seqsize else None), ds) for st in starts] if shuffle: random.shuffle(slices) return slices # Get window start limits startmins = [0, ] * datadim if startmins is None else startmins startmaxs = [shp - nhoodsiz for shp, nhoodsiz in zip(shape, nhoodsize)] if startmaxs is None else startmaxs # The final iterator is going to be a cartesian product of the lists in nslices nslices = [_1Dwindow(startmin, startmax, nhoodsiz, st, dsample, datalen, shuffle) if windowspec == 'x' else [slice(ws, ws + 1) for ws in pyk.obj2list(windowspec)] for startmin, startmax, datalen, nhoodsiz, st, windowspec, dsample in zip(startmins, startmaxs, shape, nhoodsize, stride, window, ds)] return it.product(*nslices)
def __theano__pool(self, inp, ds, stride=None, padding=None, poolmode='max', dim=None, ignoreborder=True, issequence=False): # Do imports locally to prevent circular dependencies import netutils as nu import pykit as pyk from theano.tensor.signal import pool as downsample # Determine the dimensionality of convolution (2 or 3?) if dim is None: dim = 3 if not issequence and len(ds) == 3 and inp.ndim == 5 else 2 # Defaults poolmode = 'average_exc_pad' if poolmode in [ 'mean', 'average', 'average_exc_pad' ] else poolmode padding = [[0, 0]] * dim if padding is None else padding stride = ds if stride is None else stride # Autofix inputs if isinstance(padding, int): padding = [padding] * dim if not pyk.islistoflists(pyk.obj2list(padding)): padding = [[padval] * dim for padval in pyk.obj2list(padding)] if isinstance(stride, int): stride = [stride] * dim # Check if theano can pad input as required autopaddable = all( [all([dimpad == pad[0] for dimpad in pad]) for pad in padding]) # Reshape 2D sequential data if required # Log input shape inpshape = inp.shape reallyissequential = issequence and inp.ndim == 5 if issequence: if reallyissequential: # Sequential input must be paddable by theano. This is required to reshape the sequential input back to # its original shape after pooling. assert autopaddable, "Sequential inputs must be paddable by theano. Provided padding {} cannot be " \ "handled at present.".format(padding) inp = inp.reshape((inpshape[0] * inpshape[1], inpshape[2], inpshape[3], inpshape[4]), ndim=4) ds = ds[0:2] stride = stride[0:2] padding = padding[0:2] else: warn( "Expected 5D sequential output, but got 4D non-sequential instead." ) # Determine what theano needs to be told about how to pad the input if autopaddable: autopadding = tuple([pad[0] for pad in padding]) else: autopadding = (0, ) * dim if not autopaddable and not all( [padval == 0 for padval in pyk.flatten(padding)]): if not th.config.device == 'cpu' and not self.cpupadwarned: warn( "Padding might occur on the CPU, which tends to slow things down." ) self.cpupadwarned = True inp = nu.pad(inp, padding) if dim == 2: y = downsample.pool_2d(input=inp, ds=ds, st=stride, padding=autopadding, ignore_border=ignoreborder, mode=poolmode) elif dim == 3: # parse downsampling ratio, stride and padding dsyx = ds[0:2] styx = stride[0:2] padyx = autopadding[0:2] ds0z = (1, ds[2]) st0z = (1, stride[2]) pad0z = (0, autopadding[2]) # Dowsnample yx H = downsample.pool_2d(input=inp, ds=dsyx, st=styx, padding=padyx, mode=poolmode) # Rotate tensor H = H.dimshuffle(0, 2, 3, 4, 1) # Downsample 0z H = downsample.pool_2d(input=H, ds=ds0z, st=st0z, padding=pad0z, mode=poolmode) # Undo rotate tensor y = H.dimshuffle(0, 4, 1, 2, 3) else: raise NotImplementedError("Pooling is implemented in 2D and 3D.") if issequence and reallyissequential: # Compute symbolic pool output length if ignoreborder: pooleny, poolenx = \ [T.floor((inpshape[tensorindex] + 2 * autopadding[index] - ds[index] + stride[index])/stride[index]) for index, tensorindex in enumerate([3, 4])] else: poolen = [None, None] for index, tensorindex in enumerate([3, 4]): if stride[index] >= ds[index]: poolen[index] = T.floor( (inpshape[tensorindex] + stride[index] - 1) / stride[index]) else: plen = T.floor((inpshape[tensorindex] - ds[index] + stride[index] - 1) / stride[index]) poolen[index] = T.switch(plen > 0, plen, 0) pooleny, poolenx = poolen y = y.reshape( (inpshape[0], inpshape[1], inpshape[2], pooleny, poolenx), ndim=5) return y