def __call__(self, *args, **kwargs):
# This function wraps the compiled theano function.
# ------------------------------------------------------
# Don't allow args if self.inputs is a dictionary. This is because the user can not be expected to know
# exactly how a dictionary is ordered.
args = list(args)
if isinstance(self.inputs, dict):
assert args == [], "Antipasti function object expects keyword arguments because the " \
"provided input was a dict."
if isinstance(self.inputs, list):
assert kwargs == {}, "Keywords could not be parsed by the Antipasti function object."
# Flatten kwargs or args
if args:
funcargs = list(pyk.flatten(args))
else:
funcargs = list(pyk.flatten(kwargs.values()))
# Evaluate function
outlist = pyk.obj2list(self._thfunction(*funcargs), ndarray2list=False)
# Parse output list
expoutputs = self.outputs.values() if isinstance(
self.outputs, dict) else self.outputs
expoutputs = pyk.obj2list(expoutputs, ndarray2list=False)
# Make sure the theano function has returned the correct number of outputs
assert len(outlist) == len(list(pyk.flatten(expoutputs))), "Number of outputs returned by the theano function " \
"is not consistent with the number of expected " \
"outputs."
# Unflatten theano function output (outlist)
# Get list with sublist lengths
lenlist = [pyk.smartlen(expoutput) for expoutput in expoutputs]
# Unflatten outlist
outputs = pyk.unflatten(outlist, lenlist)
# Write to dictionary if self.outputs is a dictionary
if isinstance(self.outputs, dict):
outputs = {
outname: outvar
for outname, outvar in zip(self.outputs.keys(), outputs)
}
elif isinstance(self.outputs, list):
outputs = tuple(outputs)
else:
outputs = pyk.delist(outputs)
return outputs
def makelayerxy(inpdim, outdim, layerid):
x = pyk.delist([
T.tensor('floatX', [
False,
] * indim,
name='x{}:'.format(inpnum) + str(layerid))
for inpnum, indim in enumerate(pyk.obj2list(inpdim))
])
y = pyk.delist([
T.tensor('floatX', [
False,
] * oudim,
name='y{}:'.format(outnum) + str(layerid))
for outnum, oudim in enumerate(pyk.obj2list(outdim))
])
return x, y
def feedforward(self, inp=None):
if inp is None:
inp = self.x
else:
self.x = inp
# Get output from Lasagne
out = las.layers.get_output(
self.outputlayers,
inputs={
inplayer: ishp
for inplayer, ishp in zip(pyk.obj2list(self.inputlayers),
pyk.obj2list(inp))
})
# In case out is a list:
self.y = pyk.delist(out)
return self.y
def __init__(self, splits, dim=None, issequence=None, inpshape=None):
"""
:type splits: list or int
:param splits: Index of the split (along the channel axis). E.g. split = 3 would result in the input tensor
split as: [inp[:, 0:3, ...], inp[:, 3:, ...]] for 2D inputs.
:type issequence: bool
:param issequence: Whether input is a sequence
:type inpshape: list or tuple
:param inpshape: Input shape
:return:
"""
super(splitlayer, self).__init__()
# Parse
dim = 2 if issequence else dim
assert not (
dim is None and inpshape is None
), "Data dimension can not be parsed. Provide dim or inpshape."
# Meta
self.dim = dim if dim is not None else {4: 2, 5: 3}[len(inpshape)]
self.allowsequences = True
self.issequence = self.dim == 2 and len(
self.inpshape) == 5 if issequence is None else issequence
self.inpdim = len(
inpshape) if inpshape is not None else 5 if self.issequence else {
2: 4,
3: 5
}[dim]
self.dim = 2 if self.issequence else self.dim # Correct dim if necessary
self.splits = pyk.obj2list(splits)
self.numsplits = len(self.splits) + 1
# More meta for layertrainyard
self.numinp = 1
self.numout = self.numsplits
# Shape inference
self.inpshape = [
None,
] * self.inpdim if inpshape is None else list(inpshape)
# Containers for input and output
self.x = T.tensor('floatX', [
False,
] * self.inpdim,
name='x:' + str(id(self)))
self.y = [
T.tensor('floatX', [
False,
] * self.inpdim,
name='y{}:'.format(splitnum) + str(id(self)))
for splitnum in range(self.numsplits)
]
def setbaggage(objs, **baggage):
baggage = baggage['baggage'] if 'baggage' in baggage.keys() else baggage
for obj in pyk.obj2list(objs, ndarray2list=False):
# Check if obj has a baggage.
if hasattr(obj, 'baggage'):
# Update baggage with new entries
obj.baggage.update(baggage)
else:
# Make new baggage
obj.baggage = baggage
def testmodel(model, inpshape, verbose=True, outshape=None):
testpassed = None
# Allocate numerical input values (model may have multiple inputs/outputs)
inpvals = [np.random.uniform(size=ishp).astype(th.config.floatX) for ishp in pyk.list2listoflists(inpshape)]
# Feedforward the model (i.e. build theano graph if it isn't built already)
if any([y.owner is None for y in pyk.obj2list(model.y)]):
model.feedforward()
# Evaluate model for all inputs
try:
if verbose:
print("Compiling...")
outvals = [y.eval({x: xval for x, xval in zip(pyk.obj2list(model.x), inpvals)}) for y in pyk.obj2list(model.y)]
if verbose:
print("Output shapes are: {}".format([oval.shape for oval in outvals]))
except Exception as e:
testpassed = False
if verbose:
print("Test failed because an exception was raised. The original error message follows:")
print(e.message)
# Check if output shapes check-out
if outshape is not None and testpassed is None:
modeloutshape = [oval.shape for oval in outvals]
shapecheck = modeloutshape == pyk.list2listoflists(outshape)
if not shapecheck:
if verbose:
"Shape check failed. Expected output shape {}, got {} instead.".format(outshape,
pyk.delist(modeloutshape))
testpassed = False
if testpassed is None:
testpassed = True
return testpassed
def func(*args, **kwargs):
# Compute argument length
arglen = max([pyk.smartlen(arg) for arg in list(args) + list(kwargs.values())])
# Convert arguments to list
args = [pyk.obj2list(arg) if not len(pyk.obj2list(arg)) == 1 else pyk.obj2list(arg)*arglen for arg in args]
kwargs = {kw: pyk.obj2list(arg) if not len(pyk.obj2list(arg)) == 1 else pyk.obj2list(arg)*arglen
for kw, arg in kwargs.items()}
# Make sure list sizes check out
assert all([pyk.smartlen(arg) == arglen for arg in args]) if pyk.smartlen(arg) != 0 else True, \
"Input lists must all have the same length."
assert all([pyk.smartlen(kwarg) == pyk.smartlen(kwargs.values()[0]) == arglen for kwarg in kwargs.values()]) \
if pyk.smartlen(kwargs) != 0 else True, "Keyword argument vectors must have the same length (= argument " \
"vector length)"
# Run the loop (can be done with a long-ass list comprehension, but I don't see the point)
res = []
if not len(kwargs) == 0 and not len(args) == 0:
for arg, kwarg in zip(zip(*args), zip(*kwargs.values())):
res.append(fun(*arg, **{kw: kwa for kw, kwa in zip(kwargs.keys(), kwarg)}))
else:
if len(kwargs) == 0:
res = [fun(*arg) for arg in zip(*args)]
elif len(args) == 0:
res = [fun(**{kw: kwa for kw, kwa in zip(kwargs.keys(), kwarg)}) for kwarg in zip(*kwargs.values())]
else:
return []
# Return results
return pyk.delist(res)
def __init__(self, inputlayers, outputlayers):
"""
:type inputlayers: list
:param inputlayers: List of Lasagne input layers.
:type outputlayers: list
:param outputlayers: List of output layers.
"""
# Init superclass
super(lasagnelayer, self).__init__()
# Make sure lasagne is available
assert las is not None, "Lasagne could not be imported."
# Meta
self.inputlayers = pyk.delist(inputlayers)
self.outputlayers = pyk.delist(outputlayers)
# Get inpshape from lasagne
self.lasinpshape = pyk.delist(
[list(il.shape) for il in pyk.obj2list(self.inputlayers)])
# Parse layer info
parsey = netutils.parselayerinfo(dim=2,
allowsequences=True,
numinp=pyk.smartlen(inputlayers),
issequence=False,
inpshape=self.lasinpshape)
self.dim = parsey['dim']
self.inpdim = parsey['inpdim']
self.allowsequences = parsey['allowsequences']
self.issequence = parsey['issequence']
self.numinp = parsey['numinp']
# Read parameters
self._params = las.layers.get_all_params(self.outputlayers)
self._cparams = [
netutils.getshared(like=param, value=1.) for param in self.params
]
# Name parameters right
self.nameparams()
# Shape inference
self.inpshape = parsey['inpshape']
# Check numout for consistency
assert self.numout == pyk.smartlen(self.outputlayers), "Number of outputs doesn't match " \
"the given number of output-layers"
self.x, self.y = netutils.makelayerxy(self.inpdim, self.outdim,
id(self))
def feedforward(self, inp=None):
if inp is None:
inp = self.x
else:
self.x = inp
# Evaluate function
y = self.func(inp, *self.funcargs, **self.funckwargs)
# Convert y to list if it's a tuple
y = list(y) if isinstance(y, tuple) else y
# Make sure output is consistent with expectation (since func can do whatever the f it likes)
len(pyk.obj2list(y)) == self.numout, "The given function must return {} outputs, " \
"got {} instead.".format(self.numout, len(pyk.obj2list(y)))
self.y = y
return self.y
def makeprinter(verbosity, extramonitors=None):
if verbosity >= 4:
monitors = [batchmonitor, costmonitor, lossmonitor, trEmonitor, gradnormmonitor, updatenormmonitor]
elif verbosity >= 3:
monitors = [batchmonitor, costmonitor, lossmonitor, trEmonitor]
elif verbosity >= 2:
monitors = [batchmonitor, costmonitor]
else:
monitors = []
if extramonitors is not None:
extramonitors = pyk.obj2list(extramonitors)
# Append extra monitors
monitors.extend(extramonitors)
# Build printer
prntr = printer(monitors)
return prntr
def feedforward(self, inp=None):
if inp is None:
inp = self.x
else:
self.x = inp
# Loop over connections, merge and append to a buffer
out = []
for connum, conn in enumerate(self.connections):
if pyk.smartlen(conn) == 1:
out.append(inp[pyk.delist(pyk.obj2list(conn))])
else:
if self.merge:
out.append(self.mergelayers[connum].feedforward(inp=[inp[node] for node in conn]))
else:
out.append([inp[node] for node in conn])
self.y = out
return self.y
def inferoutshape(self, inpshape=None, checkinput=True):
if inpshape is None:
inpshape = self.inpshape
# Buffer for outshape
outshape = []
for connum, conn in enumerate(self.connections):
if self.mergelayers[connum] is None:
# conn is the index of an element in the input list. Fetch its shape:
outshape.append(self.inpshape[pyk.delist(pyk.obj2list(conn))])
else:
if self.merge:
# Fetch merge layer's inpshape
self.mergelayers[connum].inpshape = [inpshape[node] for node in conn]
# Append outshape
outshape.append(self.mergelayers[connum].outshape)
else:
outshape.append([inpshape[node] for node in conn])
return outshape
def inferoutshape(self, inpshape=None, checkinput=True):
if inpshape is None:
inpshape = self.inpshape
if checkinput:
# Compare shape with Antipasti inpshape
assert netutils.shpcmp(self.lasinpshape, inpshape), "Lasagne input shape is not consistent with the " \
"inferred Antipasti input shape."
# Get output shape from Lasagne
outshape = las.layers.get_output_shape(
self.outputlayers, {
inplayer: ishp
for inplayer, ishp in zip(pyk.obj2list(self.inputlayers),
pyk.list2listoflists(inpshape))
})
outshape = pyk.listoftuples2listoflists(outshape) if pyk.islistoflists(
outshape) else list(outshape)
return outshape
def feedforward(self, inp=None):
if inp is None:
inp = self.x
else:
self.x = inp
# Loop over connections, merge and append to a buffer
out = []
for connum, conn in enumerate(self.connections):
if pyk.smartlen(conn) == 1:
out.append(inp[pyk.delist(pyk.obj2list(conn))])
else:
if self.merge:
out.append(self.mergelayers[connum].feedforward(
inp=[inp[node] for node in conn]))
else:
out.append([inp[node] for node in conn])
self.y = out
return self.y
def __init__(self, splits, dim=None, issequence=None, inpshape=None):
"""
:type splits: list or int
:param splits: Index of the split (along the channel axis). E.g. split = 3 would result in the input tensor
split as: [inp[:, 0:3, ...], inp[:, 3:, ...]] for 2D inputs.
:type issequence: bool
:param issequence: Whether input is a sequence
:type inpshape: list or tuple
:param inpshape: Input shape
:return:
"""
super(splitlayer, self).__init__()
# Parse
dim = 2 if issequence else dim
assert not (dim is None and inpshape is None), "Data dimension can not be parsed. Provide dim or inpshape."
# Meta
self.dim = dim if dim is not None else {4: 2, 5: 3}[len(inpshape)]
self.allowsequences = True
self.issequence = self.dim == 2 and len(self.inpshape) == 5 if issequence is None else issequence
self.inpdim = len(inpshape) if inpshape is not None else 5 if self.issequence else {2: 4, 3: 5}[dim]
self.dim = 2 if self.issequence else self.dim # Correct dim if necessary
self.splits = pyk.obj2list(splits)
self.numsplits = len(self.splits) + 1
# More meta for layertrainyard
self.numinp = 1
self.numout = self.numsplits
# Shape inference
self.inpshape = [None, ] * self.inpdim if inpshape is None else list(inpshape)
# Containers for input and output
self.x = T.tensor('floatX', [False, ] * self.inpdim, name='x:' + str(id(self)))
self.y = [T.tensor('floatX', [False, ] * self.inpdim, name='y{}:'.format(splitnum) + str(id(self)))
for splitnum in range(self.numsplits)]
def inferoutshape(self, inpshape=None, checkinput=True):
if inpshape is None:
inpshape = self.inpshape
# Buffer for outshape
outshape = []
for connum, conn in enumerate(self.connections):
if self.mergelayers[connum] is None:
# conn is the index of an element in the input list. Fetch its shape:
outshape.append(self.inpshape[pyk.delist(pyk.obj2list(conn))])
else:
if self.merge:
# Fetch merge layer's inpshape
self.mergelayers[connum].inpshape = [
inpshape[node] for node in conn
]
# Append outshape
outshape.append(self.mergelayers[connum].outshape)
else:
outshape.append([inpshape[node] for node in conn])
return pyk.delist(outshape)
def inferoutshape(self, inpshape=None, checkinput=False):
if inpshape is None:
inpshape = self.inpshape
if checkinput:
assert len(pyk.obj2list(inpshape[0])) == 1, "Input shape must be a list of ints " \
"(split layer takes in 1 input)"
# Recall that outshape must be a list of lists
outshape = []
shape = inpshape
indsplits = [0] + self.splits + [inpshape[-1]]
for n in range(len(indsplits) - 1):
# Copy shape (this is necessary, because python sets by reference)
shape = copy.copy(shape)
# Update channel size for all outputs
shape[(1 if self.inpdim == 4 else 2)] = indsplits[n + 1] - indsplits[n] \
if indsplits[n + 1] is not None else None
outshape.append(shape)
# Return
return outshape
def inferoutshape(self, inpshape=None, checkinput=False):
if inpshape is None:
inpshape = self.inpshape
if checkinput:
assert len(pyk.obj2list(inpshape[0])) == 1, "Input shape must be a list of ints " \
"(split layer takes in 1 input)"
# Recall that outshape must be a list of lists
outshape = []
shape = inpshape
indsplits = [0] + self.splits + [inpshape[-1]]
for n in range(len(indsplits) - 1):
# Copy shape (this is necessary, because python sets by reference)
shape = copy.copy(shape)
# Update channel size for all outputs
shape[(1 if self.inpdim == 4 else 2)] = indsplits[n + 1] - indsplits[n] \
if indsplits[n + 1] is not None else None
outshape.append(shape)
# Return
return outshape
def __init__(self,
func,
shapefunc=None,
funcargs=None,
funckwargs=None,
shapefuncargs=None,
shapefunckwargs=None,
numinp=1,
numout=1,
dim=None,
issequence=None,
inpshape=None):
"""
Layer to apply any given function to input(s). The function may require multiple inputs and
return multiple outputs. The function may change the shape of the tensor, but then a `shapefunc`
must be provided for automatic shape inference.
:type func: callable
:param func: Function to apply to input.
:type shapefunc: callable
:param shapefunc: Function to compute the output shape given input shape.
:type funcargs: tuple or list
:param funcargs: Arguments for the function (`func`)
:type funckwargs: dict
:param funckwargs: Keyword arguments for the function (`func`)
:type shapefuncargs: tuple or list
:param shapefuncargs: Arguments for the shape function (`shapefunc`)
:type shapefunckwargs: dict
:param shapefunckwargs: Keyword arguments for the shape function (`shapefunc`)
:type numinp: int
:param numinp: Number of inputs the function `func` takes.
:type numout: int
:param numout: Number of outputs the function `func` returns.
:type dim: int or list of int
:param dim: Dimensionality of the input data. Defaults to 2 when omitted.
:type issequence: bool or list of bool
:param issequence: Whether the input(s) is sequential.
"""
super(functionlayer, self).__init__()
# Defaults
shapefunc = (lambda x: x) if shapefunc is None else shapefunc
dim = 2 if dim is None else dim
issequence = False if issequence is None else issequence
# Parse layer spec
parsey = netutils.parselayerinfo(dim=dim,
allowsequences=True,
numinp=numinp,
issequence=issequence,
inpshape=inpshape)
self.dim = parsey['dim']
self.inpdim = parsey['inpdim']
self.allowsequences = parsey['allowsequences']
self.issequence = parsey['issequence']
# Meta
self.func = func
self.funcargs = [] if funcargs is None else list(funcargs)
self.funckwargs = {} if funckwargs is None else dict(funckwargs)
self.shapefunc = shapefunc
self.shapefuncargs = [] if shapefuncargs is None else list(
shapefuncargs)
self.shapefunckwargs = {} if shapefunckwargs is None else dict(
shapefunckwargs)
# Structure inference
self.numinp = parsey['numinp']
self.numout = numout
# Shape inference
self.inpshape = parsey['inpshape']
# Containers for X and Y
self.x = pyk.delist([
T.tensor('floatX', [
False,
] * indim,
name='x{}:'.format(inpnum) + str(id(self)))
for inpnum, indim in enumerate(pyk.obj2list(self.inpdim))
])
self.y = pyk.delist([
T.tensor('floatX', [
False,
] * oudim,
name='x{}:'.format(outnum) + str(id(self)))
for outnum, oudim in enumerate(pyk.obj2list(self.outdim))
])
def __init__(self, activation, dim=2, issequence=False, inpshape=None):
"""
:type activation: callable or dict
:param activation: Activation function (any element-wise symbolic function)
:type dim: int
:param dim: Dimensionality of the input data
:type issequence: bool
:param issequence: Whether the input is a sequence
:type inpshape: list
:param inpshape: Input shape
"""
super(activationlayer, self).__init__()
# Parse data dimensionality
assert not (
dim is None and inpshape is None
), "Data dimension can not be parsed. Provide dim or inpshape."
# Meta
self.dim = dim if dim is not None else {4: 2, 5: 3}[len(inpshape)]
self.allowsequences = True
self.issequence = self.dim == 2 and len(
inpshape) == 5 if issequence is None else issequence
self.inpdim = len(
inpshape) if inpshape is not None else 5 if self.issequence else {
2: 4,
3: 5
}[dim]
# Parse activation
if isinstance(activation, dict):
self.activation = activation["function"]
self._params = pyk.obj2list(
activation["trainables"]) if "trainables" in activation.keys(
) else []
self._cparams = pyk.obj2list(activation["ctrainables"]) if "ctrainables" in activation.keys() else \
[netutils.getshared(like=trainable, value=1.) for trainable in self.params]
# Name shared variables in params / cparams
for n, (param, cparam) in enumerate(zip(self.params,
self.cparams)):
param.name += "-trainable{}:".format(n) + str(id(self))
cparam.name += "-ctrainable{}:".format(n) + str(id(self))
else:
self.activation = activation
# Shape inference
self.inpshape = [
None,
] * self.inpdim if inpshape is None else list(inpshape)
# Containers for input and output
self.x = T.tensor('floatX', [
False,
] * self.inpdim,
name='x:' + str(id(self)))
self.y = T.tensor('floatX', [
False,
] * self.inpdim,
name='y:' + str(id(self)))
def __init__(self, numinp=None, dim=None, issequence=None, inpshape=None):
"""
:type numinp: int
:param numinp: Number of inputs
:type dim: list or int
:param dim: List (or int) of data dimensions of the input
:type issequence: list or bool
:param issequence: Whether inputs are sequences
:type inpshape: list or tuple
:param inpshape: Input shape
:return:
"""
super(addlayer, self).__init__()
# Convenience parse
dim = [
dim,
] * numinp if isinstance(dim, int) and numinp is not None else dim
issequence = [
issequence,
] * numinp if isinstance(issequence,
bool) and numinp is not None else issequence
# Parse inpshape if numinp, dim, issequence is given
if numinp is not None and dim is not None and inpshape is None:
if 3 in dim:
# issequence doesn't matter, inpshape can still be filled
inpshape = [[
None,
] * 5 for _ in dim]
else:
# dim is 2, but input might still be sequential
if issequence is None:
pass
else:
if issequence:
# issequence is false and dim = 2, ergo 2D data
inpshape = [[
None,
] * 4 for _ in dim]
else:
# issequence is true, ergo 2D sequential data
inpshape = [[
None,
] * 5 for _ in dim]
# Check
assert not (numinp is None and inpshape is None), "Number of inputs could not be parsed. Provide numinp " \
"or inpshape."
assert not (all([idim is None for idim in pyk.obj2list(dim)]) and inpshape is None), \
"Data dimension could not be parsed. Please provide dim or inpshape."
assert isinstance(
dim, list
) if dim is not None else True, "Dim must be a list for multiple inputs."
assert all([isinstance(ishp, list) for ishp in inpshape]) if inpshape is not None else True, \
"mergelayer expects multiple inputs, but inpshape doesn't have the correct signature."
# Meta
self.numinp = numinp if numinp is not None else len(inpshape)
self.numout = 1
self.dim = dim if dim is not None else [{
4: 2,
5: 2
}[len(ishp)] for ishp in inpshape]
self.allowsequences = True
self.issequence = [idim == 2 and len(ishp) == 5 for idim, ishp in zip(self.dim, inpshape)] \
if issequence is None else issequence
self.inpdim = [
len(ishp) if ishp is not None else 5 if iisseq else {
2: 4,
3: 5
}[idim]
for ishp, iisseq, idim in zip(inpshape, self.issequence, self.dim)
]
# Check if inpdim is consistent
assert all([indim == self.inpdim[0] for indim in self.inpdim
]), "Can't concatenate 2D with 3D inputs."
# Shape inference
self.inpshape = list(inpshape) if inpshape is not None else [[
None,
] * indim for indim in self.inpdim]
# Containers for input and output
self.x = [
T.tensor('floatX', [
False,
] * indim,
name='x{}:'.format(inpnum) + str(id(self)))
for inpnum, indim in enumerate(self.inpdim)
]
self.y = T.tensor('floatX', [
False,
] * self.inpdim[0],
name='y:' + str(id(self)))
def __init__(self,
connections,
merge=True,
dim=2,
issequence=False,
inpshape=None):
"""
:type connections: list
:param connections: List of connections. E.g.: [[0, 1], 1, [0, 1, 2]] would merge inputs 0 & 1 and write to
first output slot, 1 to second output slot and 0, 1 & 2 merged to the third output slot.
:param dim:
:param issequence:
:param inpshape:
:return:
"""
super(circuitlayer, self).__init__()
# Compute the number of i/o slots
self.numinp = len(pyk.unique(list(pyk.flatten(connections))))
self.numout = len(connections)
self.merge = bool(merge)
# TODO Fix this
assert self.numinp != 1, "Circuit layer must have atleast 2 inputs. Consider using a replicatelayer."
# Parse
dim = [
dim,
] * self.numinp if isinstance(dim,
int) and self.numinp is not None else dim
issequence = [issequence, ] * self.numinp if isinstance(issequence, bool) and self.numinp is not None \
else issequence
# Parse inpshape if numinp, dim, issequence is given
if self.numinp is not None and dim is not None and inpshape is None:
if 3 in dim:
# issequence doesn't matter, inpshape can still be filled
inpshape = [[
None,
] * 5 for _ in dim]
else:
# dim is 2, but input might still be sequential
if issequence is None:
pass
else:
if issequence:
# issequence is false and dim = 2, ergo 2D data
inpshape = [[
None,
] * 4 for _ in dim]
else:
# issequence is true, ergo 2D sequential data
inpshape = [[
None,
] * 5 for _ in dim]
# Meta
self.connections = connections
self.dim = dim if dim is not None else [{
4: 2,
5: 2
}[len(ishp)] for ishp in inpshape]
self.allowsequences = True
self.issequence = [idim == 2 and len(ishp) == 5 for idim, ishp in zip(self.dim, inpshape)] \
if issequence is None else issequence
self.inpdim = [
len(ishp) if ishp is not None else 5 if iisseq else {
2: 4,
3: 5
}[idim]
for ishp, iisseq, idim in zip(inpshape, self.issequence, self.dim)
]
# Check if inpdim is consistent
assert all([indim == self.inpdim[0] for indim in self.inpdim
]), "Can't concatenate 2D with 3D inputs."
# Make merge layers
self.mergelayers = [
mergelayer(numinp=len(conn),
dim=[self.dim[node] for node in conn],
issequence=[self.issequence[node] for node in conn],
inpshape=[inpshape[node] for node in conn])
if pyk.smartlen(conn) != 1 else None for conn in self.connections
]
# Shape inference
self.inpshape = list(inpshape) if inpshape is not None else [[
None,
] * indim for indim in self.inpdim]
# Containers for input and output
self.x = pyk.delist([
T.tensor('floatX', [
False,
] * indim,
name='x{}:'.format(inpnum) + str(id(self)))
for inpnum, indim in enumerate(self.inpdim)
])
self.y = pyk.delist([
T.tensor('floatX', [
False,
] * self.inpdim[pyk.obj2list(self.connections[outnum])[0]],
name='y:' + str(id(self)))
for outnum in range(self.numout)
])
def __theano__conv(self,
inp,
filters,
stride=None,
dilation=None,
padding=None,
bias=None,
filtergradmask=None,
biasgradmask=None,
filtermask=None,
biasmask=None,
filtergradclips=None,
biasgradclips=None,
dim=None,
convmode='same',
issequence=False,
implementation='auto'):
# Do imports locally to prevent circular dependencies
import netutils as nu
import theanops as tho
import pykit as pyk
# Determine the dimensionality of convolution (2 or 3?)
if dim is None:
dim = 3 if not issequence and len(
filters.get_value().shape) == 5 and inp.ndim == 5 else 2
# Smart fix: if convmode is 'same', stride != 1 and padding is None: set automagically set padding.
# Defaults
padding = [[0, 0]] * dim if padding is None else padding
stride = [1] * dim if stride is None else stride
dilation = [1] * dim if dilation is None else dilation
filtergradclips = [
-np.inf, np.inf
] if filtergradclips is None else list(filtergradclips)
biasgradclips = [-np.inf, np.inf
] if biasgradclips is None else list(biasgradclips)
# Autofix inputs
if isinstance(padding, int):
padding = [padding] * dim
if not pyk.islistoflists(pyk.obj2list(padding)):
padding = [[padval] * dim for padval in pyk.obj2list(padding)]
if isinstance(stride, int):
stride = [stride] * dim
if isinstance(dilation, int):
dilation = [dilation] * dim
# TODO: Tests
pass
# Reshape 2D sequential data if required
# Log input shape
inpshape = inp.shape
reallyissequential = issequence and inp.ndim == 5
if issequence:
if reallyissequential:
inp = inp.reshape((inpshape[0] * inpshape[1], inpshape[2],
inpshape[3], inpshape[4]),
ndim=4)
stride = stride[0:2]
padding = padding[0:2]
# TODO: Get rid of these restrictions
assert stride == [
1, 1
], "Strided convolution is not implemented for sequential data."
assert convmode == 'same', "Convmode must be 'same' for sequential data."
else:
warn(
"Expected 5D sequential output, but got 4D non-sequential instead."
)
# Apply gradient masks if required
if filtergradmask is not None:
filters = tho.maskgradient(filters, filtergradmask)
if biasgradmask is not None and bias is not None:
bias = tho.maskgradient(bias, biasgradmask)
# Apply masks if required
if filtermask is not None:
filters = filtermask * filters
if biasmask is not None:
bias = biasmask * bias
# Determine border_mode for CuDNN/3D conv
autopaddable, bordermode, trim = self.__theano__bordermode(
convmode, padding,
filters.get_value().shape)
# Pad input if required (warn that it's ridiculously slow)
if not autopaddable and not all(
[padval == 0 for padval in pyk.flatten(padding)]):
if not isinstance(bordermode,
str) and pyk.islistoflists(bordermode):
# Override padding for 3D convolutions
inp = nu.pad(inp, bordermode)
bordermode = 'valid'
else:
inp = nu.pad(inp, padding)
# Switch implementation
if implementation == 'auto':
# Fall back implementation: 'vanilla'
implementation = 'vanilla'
if dilation != [1, 1]:
implementation = 'dilated'
# Convolve 2D (with gradmask + bias), reshape sequential data
if dim == 2:
if implementation == 'vanilla':
if list(dilation) != [1, 1]:
warn(
"Filter dilation is not possible with this implementation."
)
# Convolve
y = T.nnet.conv2d(input=inp,
filters=th.gradient.grad_clip(
filters, *filtergradclips),
border_mode=tuple(bordermode) if isinstance(
bordermode, list) else bordermode,
filter_shape=filters.get_value().shape,
subsample=tuple(stride))
elif implementation == 'dilated':
# Make sure stride is 1
assert list(stride) == [
1, 1
], "Stride should equal [1, 1] for dilated convolutions."
assert not issequence, "Dilated convolution is not supported for sequential data."
# Dilated conv can't handle padding at the moment, do this manually
if isinstance(bordermode, tuple):
padding = [[bm, bm] for bm in bordermode]
inp = nu.pad(inp, padding)
elif bordermode == 'full':
raise NotImplementedError(
"Convolution mode 'full' is not implemented for dilated convolutions."
)
elif bordermode == 'valid':
pass
elif bordermode == 'half':
assert all([d % 2 == 0 for d in dilation]), "Dilation amount must be divisible by 2 for dilated " \
"convolution with 'same' border handling."
padding = [[
(filters.get_value().shape[n] - 1) * d / 2,
] * 2 for n, d in zip([2, 3], dilation)]
inp = nu.pad(inp, padding)
else:
raise NotImplementedError(
"Unknown bordermode: {}.".format(bordermode))
# Get output image shape
oishp = [
inp.shape[n] - (filters.shape[n] - 1) * d
for n, d in zip([2, 3], dilation)
]
# Get computin'
op = T.nnet.abstract_conv.AbstractConv2d_gradWeights(
subsample=tuple(dilation),
border_mode='valid',
filter_flip=False)
y = op(inp.transpose(1, 0, 2, 3),
filters.transpose(1, 0, 2, 3), tuple(oishp))
y = y.transpose(1, 0, 2, 3)
else:
raise NotImplementedError(
"Implementation {} is not implemented.".format(
implementation))
# Trim if required
if trim:
y = self.__theano__convtrim(
inp=y, filtershape=filters.get_value().shape)
# Add bias if required
if bias is not None:
y = y + th.gradient.grad_clip(bias, *biasgradclips).dimshuffle(
'x', 0, 'x', 'x')
elif dim == 3:
# Convolve 3D (with bias)
if implementation == 'auto' or implementation == 'conv2d':
assert stride == [
1, 1, 1
], "Implementation 'conv2d' does not support strided convolution in 3D."
assert convmode == 'valid', "Implementation 'conv2d' only supports 'valid' convolutions."
y = T.nnet.conv3d2d.conv3d(
signals=inp,
filters=th.gradient.grad_clip(filters, *filtergradclips),
border_mode=bordermode,
filters_shape=filters.get_value().shape)
else:
raise NotImplementedError(
"Implementation {} is not implemented.".format(
implementation))
# Trim if required
if trim:
y = self.__theano__convtrim(
inp=y, filtershape=filters.get_value().shape)
# Add bias if required
if bias is not None:
y = y + th.gradient.grad_clip(bias, *biasgradclips).dimshuffle(
'x', 'x', 0, 'x', 'x')
else:
raise NotImplementedError(
"Convolution is implemented in 2D and 3D.")
# Reshape sequential data
if issequence and reallyissequential:
y = y.reshape(
(inpshape[0], inpshape[1], filters.get_value().shape[0],
inpshape[3], inpshape[4]),
ndim=5)
# Return
return y
def __init__(self, connections, merge=True, dim=2, issequence=False, inpshape=None):
"""
:type connections: list
:param connections: List of connections. E.g.: [[0, 1], 1, [0, 1, 2]] would merge inputs 0 & 1 and write to
first output slot, 1 to second output slot and 0, 1 & 2 merged to the third output slot.
:param dim:
:param issequence:
:param inpshape:
:return:
"""
super(circuitlayer, self).__init__()
# Compute the number of i/o slots
self.numinp = len(pyk.unique(list(pyk.flatten(connections))))
self.numout = len(connections)
self.merge = bool(merge)
# Parse
dim = [dim, ] * self.numinp if isinstance(dim, int) and self.numinp is not None else dim
issequence = [issequence, ] * self.numinp if isinstance(issequence, bool) and self.numinp is not None \
else issequence
# Parse inpshape if numinp, dim, issequence is given
if self.numinp is not None and dim is not None and inpshape is None:
if 3 in dim:
# issequence doesn't matter, inpshape can still be filled
inpshape = [[None, ] * 5 for _ in dim]
else:
# dim is 2, but input might still be sequential
if issequence is None:
pass
else:
if issequence:
# issequence is false and dim = 2, ergo 2D data
inpshape = [[None, ] * 4 for _ in dim]
else:
# issequence is true, ergo 2D sequential data
inpshape = [[None, ] * 5 for _ in dim]
# Meta
self.connections = connections
self.dim = dim if dim is not None else [{4: 2, 5: 2}[len(ishp)] for ishp in inpshape]
self.allowsequences = True
self.issequence = [idim == 2 and len(ishp) == 5 for idim, ishp in zip(self.dim, inpshape)] \
if issequence is None else issequence
self.inpdim = [len(ishp) if ishp is not None else 5 if iisseq else {2: 4, 3: 5}[idim]
for ishp, iisseq, idim in zip(inpshape, self.issequence, self.dim)]
# Check if inpdim is consistent
assert all([indim == self.inpdim[0] for indim in self.inpdim]), "Can't concatenate 2D with 3D inputs."
# Make merge layers
self.mergelayers = [mergelayer(numinp=len(conn),
dim=[self.dim[node] for node in conn],
issequence=[self.issequence[node] for node in conn],
inpshape=[inpshape[node] for node in conn])
if pyk.smartlen(conn) != 1 else None for conn in self.connections]
# Shape inference
self.inpshape = list(inpshape) if inpshape is not None else [[None, ] * indim for indim in self.inpdim]
# Containers for input and output
self.x = pyk.delist([T.tensor('floatX', [False, ] * indim, name='x{}:'.format(inpnum) + str(id(self)))
for inpnum, indim in enumerate(self.inpdim)])
self.y = pyk.delist([T.tensor('floatX', [False, ] * self.inpdim[pyk.obj2list(self.connections[outnum])[0]],
name='y:' + str(id(self)))
for outnum in range(self.numout)])
def parselayerinfo(dim=None,
inpdim=None,
issequence=None,
allowsequences=None,
numinp=None,
inpshape=None,
verbose=True):
parsey = {
'dim': dim,
'inpdim': inpdim,
'issequence': issequence,
'allowsequences': allowsequences,
'numinp': numinp,
'inpshape': inpshape
}
# Parse from inpshape
if parsey['inpshape'] is not None:
# Make sure inpshape is a list
assert isinstance(
parsey['inpshape'],
list), "inpshape must be a list, e.g. [None, 3, None, None]."
# Fetch number of inputs
if pyk.islistoflists(parsey['inpshape']):
_numinp = len(parsey['inpshape'])
_inpdim = [len(ishp) for ishp in parsey['inpshape']]
else:
_numinp = 1
_inpdim = len(parsey['inpshape'])
# Write to parsed (if not written already)
# numinp
parsey['numinp'] = _numinp if parsey['numinp'] is None else parsey[
'numinp']
# Consistency check
assert parsey['numinp'] == _numinp, "The provided inpshape requires numinp = {}, " \
"but the value given was {}".format(_numinp, parsey['numinp'])
# inpdim
parsey['inpdim'] = _inpdim if parsey['inpdim'] is None else parsey[
'inpdim']
assert parsey['inpdim'] == _inpdim, "The provided inpshape requires inpdim = {}, " \
"but the value given was {}".format(_inpdim, parsey['inpdim'])
# Check if dim, inpdim, issequence or allowsequences is a list of multiple elements and numinp is not given.
if parsey['numinp'] is None:
for argname in ['dim', 'inpdim', 'issequence', 'allowsequences']:
if isinstance(parsey[argname], list):
parsey['numinp'] = len(parsey[argname])
# Parse from numinp
if parsey['numinp'] is not None:
for argname, argtype in zip(
['dim', 'inpdim', 'issequence', 'allowsequences'],
[int, int, bool, bool]):
if isinstance(parsey[argname], argtype) and parsey['numinp'] > 1:
# If numinp is > 1 and allowseqences or issequence or inpdim or dim is bool or bool or int or int,
# assume that the user (or the author) is too lazy to type in a list.
parsey[argname] = [
parsey[argname],
] * parsey['numinp']
elif isinstance(parsey[argname], list) and parsey['numinp'] > 1:
# If the user was not lazy, make sure the given list sizes check out
assert len(parsey[argname]) == parsey['numinp'], \
"{} must be a {} or a list of length {} (= numinp).".format(argname, argtype, parsey['numinp'])
# Check if inpshape is consistent
if parsey['inpshape'] is not None and parsey['numinp'] > 1:
assert pyk.islistoflists(parsey['inpshape']) and len(
parsey['inpshape']) == parsey['numinp']
else:
if verbose:
warn("Guessing that numinp = 1.")
# Guess numinp = 1.
parsey['numinp'] = 1
# Parse allowsequences
# At this point, allowsequences must be known (or no conclusions can be drawn on issequence and dim)
if parsey['allowsequences'] is None:
if verbose:
warn("Guessing that sequences are allowed.")
parsey['allowsequences'] = pyk.delist([
True,
] * parsey['numinp'])
else:
# Okay, so it's known if sequences are allowed. Check if issequence is consistent.
if pyk.obj2list(parsey['allowsequences']) == [
False,
] * parsey['numinp'] and parsey['issequence'] is not None:
# If sequences are not allowed, make sure issequence is False
assert pyk.obj2list(parsey['issequence']) == [False,] * parsey['numinp'], \
"Input(s) are not allowed to be sequential, yet they are."
# Parse issequence
if parsey['issequence'] is not None:
# Delist issequence
parsey['issequence'] = pyk.delist(parsey['issequence']) \
if isinstance(parsey['issequence'], list) else parsey['issequence']
# Check if issequence is consistent with everything
if isinstance(parsey['issequence'], list):
assert len(parsey['issequence']) == parsey['numinp'], "issequence must be a list of the same lenght as " \
"numinp = {} if numinp > 1.".format(parsey['numinp'])
# Check if consistent with allowsequences. At this point, issequence may have None's.
assert all([(bool(isseq) and allowseq) or not isseq
for isseq, allowseq in zip(pyk.obj2list(parsey['issequence']),
pyk.obj2list(parsey['allowsequences']))]), \
"Input is a sequence although it's not allowed to. " \
"issequence = {}, allowsequences = {}.".format(parsey['issequence'], parsey['allowsequences'])
else:
if verbose:
warn("Guessing that input(s) is(are) not sequential.")
parsey['issequence'] = pyk.delist([
False,
] * parsey['numinp'])
# Parse inpdim
# Compute expected inpdim from what's known
# Check in from issequence
_inpdim = pyk.delist(
[5 if isseq else None for isseq in pyk.obj2list(parsey['issequence'])])
# Check in from dim
if parsey['dim'] is not None:
_inpdim = pyk.delist([
5 if d == 3 else indim for d, indim in zip(
pyk.obj2list(parsey['dim']), pyk.obj2list(_inpdim))
])
_inpdim = pyk.delist([
4 if (d == 2 and not isseq) else indim for d, indim, isseq in zip(
pyk.obj2list(parsey['dim']), pyk.obj2list(_inpdim),
pyk.obj2list(parsey['issequence']))
])
if parsey['inpdim'] is None:
# Make sure there are no None's remaining in _inpdim
assert None not in pyk.obj2list(
_inpdim
), "Input dimensionality could not be parsed due to missing information."
parsey['inpdim'] = _inpdim
else:
assert pyk.smartlen(parsey['inpdim']) == pyk.smartlen(_inpdim), \
"Expected {} elements in inpdim, got {}.".format(pyk.smartlen(_inpdim), pyk.smartlen(parsey['inpdim']))
# Check consistency with the expected _inpdim
assert all([_indim == indim for _indim, indim in zip(pyk.obj2list(_inpdim), pyk.obj2list(parsey['inpdim']))
if _indim is not None]), \
"Provided inpdim is inconsistent with either dim or issequence."
# Parse dim
# Compute expected _inpdim from what's known
_dim = pyk.delist([
2 if (indim == 4 or isseq) else 3 for indim, isseq in zip(
pyk.obj2list(parsey['inpdim']), pyk.obj2list(parsey['issequence']))
])
# Check in from dim
if parsey['dim'] is None:
parsey['dim'] = _dim
else:
assert parsey[
'dim'] == _dim, "Given dim ({}) is not consistent with expectation ({})".format(
parsey['dim'], _dim)
# Reparse inpshape
if parsey['inpshape'] is None:
parsey['inpshape'] = pyk.delist([[
None,
] * indim for indim in pyk.obj2list(parsey['inpdim'])])
# Return parsey :(
return parsey
def __init__(self, numinp=None, dim=None, issequence=None, inpshape=None):
"""
:type numinp: int
:param numinp: Number of inputs
:type dim: list or int
:param dim: List (or int) of data dimensions of the input
:type issequence: list or bool
:param issequence: Whether inputs are sequences
:type inpshape: list or tuple
:param inpshape: Input shape
:return:
"""
super(addlayer, self).__init__()
# Convenience parse
dim = [dim, ] * numinp if isinstance(dim, int) and numinp is not None else dim
issequence = [issequence, ] * numinp if isinstance(issequence, bool) and numinp is not None else issequence
# Parse inpshape if numinp, dim, issequence is given
if numinp is not None and dim is not None and inpshape is None:
if 3 in dim:
# issequence doesn't matter, inpshape can still be filled
inpshape = [[None, ] * 5 for _ in dim]
else:
# dim is 2, but input might still be sequential
if issequence is None:
pass
else:
if issequence:
# issequence is false and dim = 2, ergo 2D data
inpshape = [[None, ] * 4 for _ in dim]
else:
# issequence is true, ergo 2D sequential data
inpshape = [[None, ] * 5 for _ in dim]
# Check
assert not (numinp is None and inpshape is None), "Number of inputs could not be parsed. Provide numinp " \
"or inpshape."
assert not (all([idim is None for idim in pyk.obj2list(dim)]) and inpshape is None), \
"Data dimension could not be parsed. Please provide dim or inpshape."
assert isinstance(dim, list) if dim is not None else True, "Dim must be a list for multiple inputs."
assert all([isinstance(ishp, list) for ishp in inpshape]) if inpshape is not None else True, \
"mergelayer expects multiple inputs, but inpshape doesn't have the correct signature."
# Meta
self.numinp = numinp if numinp is not None else len(inpshape)
self.numout = 1
self.dim = dim if dim is not None else [{4: 2, 5: 2}[len(ishp)] for ishp in inpshape]
self.allowsequences = True
self.issequence = [idim == 2 and len(ishp) == 5 for idim, ishp in zip(self.dim, inpshape)] \
if issequence is None else issequence
self.inpdim = [len(ishp) if ishp is not None else 5 if iisseq else {2: 4, 3: 5}[idim]
for ishp, iisseq, idim in zip(inpshape, self.issequence, self.dim)]
# Check if inpdim is consistent
assert all([indim == self.inpdim[0] for indim in self.inpdim]), "Can't concatenate 2D with 3D inputs."
# Shape inference
self.inpshape = list(inpshape) if inpshape is not None else [[None, ] * indim for indim in self.inpdim]
# Containers for input and output
self.x = [T.tensor('floatX', [False, ] * indim, name='x{}:'.format(inpnum) + str(id(self)))
for inpnum, indim in enumerate(self.inpdim)]
self.y = T.tensor('floatX', [False, ] * self.inpdim[0], name='y:' + str(id(self)))
def __theano__conv(self,
inp,
filters,
stride=None,
padding=None,
bias=None,
filtergradmask=None,
biasgradmask=None,
filtermask=None,
biasmask=None,
filtergradclips=None,
biasgradclips=None,
dim=None,
convmode='same',
issequence=False,
implementation='auto'):
# Do imports locally to prevent circular dependencies
import netutils as nu
import theanops as tho
import pykit as pyk
# Determine the dimensionality of convolution (2 or 3?)
if dim is None:
dim = 3 if not issequence and len(
filters.get_value().shape) == 5 and inp.ndim == 5 else 2
# Smart fix: if convmode is 'same', stride != 1 and padding is None: set automagically set padding.
# Defaults
padding = [[0, 0]] * dim if padding is None else padding
stride = [1] * dim if stride is None else stride
filtergradclips = [
-np.inf, np.inf
] if filtergradclips is None else list(filtergradclips)
biasgradclips = [-np.inf, np.inf
] if biasgradclips is None else list(biasgradclips)
# Autofix inputs
if isinstance(padding, int):
padding = [padding] * dim
if not pyk.islistoflists(pyk.obj2list(padding)):
padding = [[padval] * dim for padval in pyk.obj2list(padding)]
if isinstance(stride, int):
stride = [stride] * dim
# TODO: Tests
pass
# Reshape 2D sequential data if required
# Log input shape
inpshape = inp.shape
reallyissequential = issequence and inp.ndim == 5
if issequence:
if reallyissequential:
inp = inp.reshape((inpshape[0] * inpshape[1], inpshape[2],
inpshape[3], inpshape[4]),
ndim=4)
stride = stride[0:2]
padding = padding[0:2]
# TODO: Get rid of these restrictions
assert stride == [
1, 1
], "Strided convolution is not implemented for sequential data."
assert convmode == 'same', "Convmode must be 'same' for sequential data."
else:
warn(
"Expected 5D sequential output, but got 4D non-sequential instead."
)
# Apply gradient masks if required
if filtergradmask is not None:
filters = tho.maskgradient(filters, filtergradmask)
if biasgradmask is not None and bias is not None:
bias = tho.maskgradient(bias, biasgradmask)
# Apply masks if required
if filtermask is not None:
filters = filtermask * filters
if biasmask is not None:
bias = biasmask * bias
# Determine border_mode for CuDNN/3D conv
autopaddable, bordermode, trim = self.__theano__bordermode(
convmode, padding,
filters.get_value().shape)
# Pad input if required (warn that it's ridiculously slow)
if not autopaddable and not all(
[padval == 0 for padval in pyk.flatten(padding)]):
if not th.config.device == 'cpu' and not self.cpupadwarned:
warn(
"Padding might occur on the CPU, which tends to slow things down."
)
self.cpupadwarned = True
if not isinstance(bordermode,
str) and pyk.islistoflists(bordermode):
# Override padding for 3D convolutions
inp = nu.pad(inp, bordermode)
bordermode = 'valid'
else:
inp = nu.pad(inp, padding)
# Convolve 2D (with gradmask + bias), reshape sequential data
if dim == 2:
if implementation == 'auto':
# Convolve
y = T.nnet.conv2d(input=inp,
filters=th.gradient.grad_clip(
filters, *filtergradclips),
border_mode=tuple(bordermode) if isinstance(
bordermode, list) else bordermode,
filter_shape=filters.get_value().shape,
subsample=tuple(stride))
else:
raise NotImplementedError(
"Implementation {} is not implemented.".format(
implementation))
# Trim if required
if trim:
y = self.__theano__convtrim(
inp=y, filtershape=filters.get_value().shape)
# Add bias if required
if bias is not None:
y = y + th.gradient.grad_clip(bias, *biasgradclips).dimshuffle(
'x', 0, 'x', 'x')
elif dim == 3:
# Convolve 3D (with bias)
if implementation == 'auto' or implementation == 'conv2d':
assert stride == [
1, 1, 1
], "Implementation 'conv2d' does not support strided convolution in 3D."
assert convmode == 'valid', "Implementation 'conv2d' only supports 'valid' convolutions."
y = T.nnet.conv3d2d.conv3d(
signals=inp,
filters=th.gradient.grad_clip(filters, *filtergradclips),
border_mode=bordermode,
filters_shape=filters.get_value().shape)
else:
raise NotImplementedError(
"Implementation {} is not implemented.".format(
implementation))
# Trim if required
if trim:
y = self.__theano__convtrim(
inp=y, filtershape=filters.get_value().shape)
# Add bias if required
if bias is not None:
y = y + th.gradient.grad_clip(bias, *biasgradclips).dimshuffle(
'x', 'x', 0, 'x', 'x')
else:
raise NotImplementedError(
"Convolution is implemented in 2D and 3D.")
# Reshape sequential data
if issequence and reallyissequential:
y = y.reshape(
(inpshape[0], inpshape[1], filters.get_value().shape[0],
inpshape[3], inpshape[4]),
ndim=5)
# Return
return y
def __theano__pool(self, inp, ds, stride=None, padding=None, poolmode='max', dim=None,
ignoreborder=True, issequence=False):
# Do imports locally to prevent circular dependencies
import netutils as nu
import pykit as pyk
from theano.tensor.signal import downsample
# Determine the dimensionality of convolution (2 or 3?)
if dim is None:
dim = 3 if not issequence and len(ds) == 3 and inp.ndim == 5 else 2
# Defaults
poolmode = 'average_exc_pad' if poolmode in ['mean', 'average', 'average_exc_pad'] else poolmode
padding = [[0, 0]] * dim if padding is None else padding
stride = ds if stride is None else stride
# Autofix inputs
if isinstance(padding, int):
padding = [padding] * dim
if not pyk.islistoflists(pyk.obj2list(padding)):
padding = [[padval] * dim for padval in pyk.obj2list(padding)]
if isinstance(stride, int):
stride = [stride] * dim
# Check if theano can pad input as required
autopaddable = all([all([dimpad == pad[0] for dimpad in pad]) for pad in padding])
# Reshape 2D sequential data if required
# Log input shape
inpshape = inp.shape
reallyissequential = issequence and inp.ndim == 5
if issequence:
if reallyissequential:
# Sequential input must be paddable by theano. This is required to reshape the sequential input back to
# its original shape after pooling.
assert autopaddable, "Sequential inputs must be paddable by theano. Provided padding {} cannot be " \
"handled at present.".format(padding)
inp = inp.reshape((inpshape[0] * inpshape[1], inpshape[2], inpshape[3], inpshape[4]), ndim=4)
ds = ds[0:2]
stride = stride[0:2]
padding = padding[0:2]
else:
warn("Expected 5D sequential output, but got 4D non-sequential instead.")
# Determine what theano needs to be told about how to pad the input
if autopaddable:
autopadding = tuple([pad[0] for pad in padding])
else:
autopadding = (0,) * dim
if not autopaddable and not all([padval == 0 for padval in pyk.flatten(padding)]):
if not th.config.device == 'cpu' and not self.cpupadwarned:
warn("Padding might occur on the CPU, which tends to slow things down.")
self.cpupadwarned = True
inp = nu.pad(inp, padding)
if dim == 2:
y = downsample.max_pool_2d(input=inp, ds=ds, st=stride, padding=autopadding, ignore_border=ignoreborder,
mode=poolmode)
elif dim == 3:
# parse downsampling ratio, stride and padding
dsyx = ds[0:2]
styx = stride[0:2]
padyx = autopadding[0:2]
ds0z = (1, ds[2])
st0z = (1, stride[2])
pad0z = (0, autopadding[2])
# Dowsnample yx
H = downsample.max_pool_2d(input=inp, ds=dsyx, st=styx, padding=padyx, mode=poolmode)
# Rotate tensor
H = H.dimshuffle(0, 2, 3, 4, 1)
# Downsample 0z
H = downsample.max_pool_2d(input=H, ds=ds0z, st=st0z, padding=pad0z, mode=poolmode)
# Undo rotate tensor
y = H.dimshuffle(0, 4, 1, 2, 3)
else:
raise NotImplementedError("Pooling is implemented in 2D and 3D.")
if issequence and reallyissequential:
# Compute symbolic pool output length
if ignoreborder:
pooleny, poolenx = \
[T.floor((inpshape[tensorindex] + 2 * autopadding[index] - ds[index] + stride[index])/stride[index])
for index, tensorindex in enumerate([3, 4])]
else:
poolen = [None, None]
for index, tensorindex in enumerate([3, 4]):
if stride[index] >= ds[index]:
poolen[index] = T.floor((inpshape[tensorindex] + stride[index] - 1)/stride[index])
else:
plen = T.floor((inpshape[tensorindex] - ds[index] + stride[index] - 1)/stride[index])
poolen[index] = T.switch(plen > 0, plen, 0)
pooleny, poolenx = poolen
y = y.reshape((inpshape[0], inpshape[1], inpshape[2], pooleny, poolenx), ndim=5)
return y
def __theano__conv(self, inp, filters, stride=None, padding=None, bias=None, filtergradmask=None,
biasgradmask=None, filtermask=None, biasmask=None, filtergradclips=None, biasgradclips=None,
dim=None, convmode='same', issequence=False, implementation='auto'):
# Do imports locally to prevent circular dependencies
import netutils as nu
import theanops as tho
import pykit as pyk
# Determine the dimensionality of convolution (2 or 3?)
if dim is None:
dim = 3 if not issequence and len(filters.get_value().shape) == 5 and inp.ndim == 5 else 2
# Smart fix: if convmode is 'same', stride != 1 and padding is None: set automagically set padding.
# Defaults
padding = [[0, 0]] * dim if padding is None else padding
stride = [1] * dim if stride is None else stride
filtergradclips = [-np.inf, np.inf] if filtergradclips is None else list(filtergradclips)
biasgradclips = [-np.inf, np.inf] if biasgradclips is None else list(biasgradclips)
# Autofix inputs
if isinstance(padding, int):
padding = [padding] * dim
if not pyk.islistoflists(pyk.obj2list(padding)):
padding = [[padval] * dim for padval in pyk.obj2list(padding)]
if isinstance(stride, int):
stride = [stride] * dim
# TODO: Tests
pass
# Reshape 2D sequential data if required
# Log input shape
inpshape = inp.shape
reallyissequential = issequence and inp.ndim == 5
if issequence:
if reallyissequential:
inp = inp.reshape((inpshape[0] * inpshape[1], inpshape[2], inpshape[3], inpshape[4]), ndim=4)
stride = stride[0:2]
padding = padding[0:2]
# TODO: Get rid of these restrictions
assert stride == [1, 1], "Strided convolution is not implemented for sequential data."
assert convmode == 'same', "Convmode must be 'same' for sequential data."
else:
warn("Expected 5D sequential output, but got 4D non-sequential instead.")
# Apply gradient masks if required
if filtergradmask is not None:
filters = tho.maskgradient(filters, filtergradmask)
if biasgradmask is not None and bias is not None:
bias = tho.maskgradient(bias, biasgradmask)
# Apply masks if required
if filtermask is not None:
filters = filtermask * filters
if biasmask is not None:
bias = biasmask * bias
# Determine border_mode for CuDNN/3D conv
autopaddable, bordermode, trim = self.__theano__bordermode(convmode, padding, filters.get_value().shape)
# Pad input if required (warn that it's ridiculously slow)
if not autopaddable and not all([padval == 0 for padval in pyk.flatten(padding)]):
if not th.config.device == 'cpu' and not self.cpupadwarned:
warn("Padding might occur on the CPU, which tends to slow things down.")
self.cpupadwarned = True
if not isinstance(bordermode, str) and pyk.islistoflists(bordermode):
# Override padding for 3D convolutions
inp = nu.pad(inp, bordermode)
bordermode = 'valid'
else:
inp = nu.pad(inp, padding)
# Convolve 2D (with gradmask + bias), reshape sequential data
if dim == 2:
if implementation == 'auto':
# Convolve
y = T.nnet.conv2d(input=inp, filters=th.gradient.grad_clip(filters, *filtergradclips),
border_mode=tuple(bordermode) if isinstance(bordermode, list) else bordermode,
filter_shape=filters.get_value().shape, subsample=tuple(stride))
else:
raise NotImplementedError("Implementation {} is not implemented.".format(implementation))
# Trim if required
if trim:
y = self.__theano__convtrim(inp=y, filtershape=filters.get_value().shape)
# Add bias if required
if bias is not None:
y = y + th.gradient.grad_clip(bias, *biasgradclips).dimshuffle('x', 0, 'x', 'x')
elif dim == 3:
# Convolve 3D (with bias)
if implementation == 'auto' or implementation == 'conv2d':
assert stride == [1, 1, 1], "Implementation 'conv2d' does not support strided convolution in 3D."
assert convmode == 'valid', "Implementation 'conv2d' only supports 'valid' convolutions."
y = T.nnet.conv3d2d.conv3d(signals=inp, filters=th.gradient.grad_clip(filters, *filtergradclips),
border_mode=bordermode,
filters_shape=filters.get_value().shape)
else:
raise NotImplementedError("Implementation {} is not implemented.".format(implementation))
# Trim if required
if trim:
y = self.__theano__convtrim(inp=y, filtershape=filters.get_value().shape)
# Add bias if required
if bias is not None:
y = y + th.gradient.grad_clip(bias, *biasgradclips).dimshuffle('x', 'x', 0, 'x', 'x')
else:
raise NotImplementedError("Convolution is implemented in 2D and 3D.")
# Reshape sequential data
if issequence and reallyissequential:
y = y.reshape((inpshape[0], inpshape[1], filters.get_value().shape[0], inpshape[3], inpshape[4]), ndim=5)
# Return
return y
def slidingwindowslices(shape, nhoodsize, stride=1, ds=1, window=None, ignoreborder=True, shuffle=True, rngseed=None,
startmins=None, startmaxs=None, shufflebuffersize=1000):
"""
Returns a generator yielding (shuffled) sliding window slice objects.
:type shape: int or list of int
:param shape: Shape of the input data
:type nhoodsize: int or list of int
:param nhoodsize: Window size of the sliding window.
:type stride: int or list of int
:param stride: Stride of the sliding window.
:type window: list
:param window: Configure the sliding window. Examples:
With axistags 'yxz':
- window = ['x', 'x', 'x'] ==> 3D sliding windows over the 3D volume
- window = [[0, 1], 'x', 'x'] ==> 2D sliding window over the 0-th and 1-st xz planes
- window = ['x', [8, 9], 'x'] ==> 2D sliding window over the 8-th and 9-st yz planes
:type ignoreborder: bool
:param ignoreborder: Whether to skip border windows (i.e. windows without enough pixels to fill the specified
nhoodsize).
:type shuffle: bool
:param shuffle: Whether to shuffle the iterator.
:type rngseed: int
:param rngseed: Random number generator seed. Use to synchronize shuffled generators.
:returns: A python generator whose next method yields a tuple of slices.
"""
# Determine dimensionality of the data
datadim = len(shape)
# Parse window
if window is None:
window = ['x'] * datadim
else:
assert len(window) == datadim, "Window must have the same length as the number of data dimensions."
# Parse nhoodsize and stride
nhoodsize = [nhoodsize, ] * datadim if isinstance(nhoodsize, int) else nhoodsize
stride = [stride, ] * datadim if isinstance(stride, int) else stride
ds = [ds, ] * datadim if isinstance(ds, int) else ds
# Seed RNG if a seed is provided
if rngseed is not None:
random.seed(rngseed)
# Define a function that gets a 1D slice
def _1Dwindow(startmin, startmax, nhoodsize, stride, ds, seqsize, shuffle):
starts = range(startmin, startmax + 1, stride)
if ignoreborder:
slices = [slice(st, st + nhoodsize, ds) for st in starts if st + nhoodsize <= seqsize]
else:
slices = [slice(st, ((st + nhoodsize) if st + nhoodsize <= seqsize else None), ds) for st in starts]
if shuffle:
random.shuffle(slices)
return slices
# Get window start limits
startmins = [0, ] * datadim if startmins is None else startmins
startmaxs = [shp - nhoodsiz for shp, nhoodsiz in zip(shape, nhoodsize)] if startmaxs is None else startmaxs
# The final iterator is going to be a cartesian product of the lists in nslices
nslices = [_1Dwindow(startmin, startmax, nhoodsiz, st, dsample, datalen, shuffle) if windowspec == 'x'
else [slice(ws, ws + 1) for ws in pyk.obj2list(windowspec)]
for startmin, startmax, datalen, nhoodsiz, st, windowspec, dsample in zip(startmins, startmaxs, shape,
nhoodsize, stride, window, ds)]
return it.product(*nslices)
def __theano__pool(self,
inp,
ds,
stride=None,
padding=None,
poolmode='max',
dim=None,
ignoreborder=True,
issequence=False):
# Do imports locally to prevent circular dependencies
import netutils as nu
import pykit as pyk
from theano.tensor.signal import pool as downsample
# Determine the dimensionality of convolution (2 or 3?)
if dim is None:
dim = 3 if not issequence and len(ds) == 3 and inp.ndim == 5 else 2
# Defaults
poolmode = 'average_exc_pad' if poolmode in [
'mean', 'average', 'average_exc_pad'
] else poolmode
padding = [[0, 0]] * dim if padding is None else padding
stride = ds if stride is None else stride
# Autofix inputs
if isinstance(padding, int):
padding = [padding] * dim
if not pyk.islistoflists(pyk.obj2list(padding)):
padding = [[padval] * dim for padval in pyk.obj2list(padding)]
if isinstance(stride, int):
stride = [stride] * dim
# Check if theano can pad input as required
autopaddable = all(
[all([dimpad == pad[0] for dimpad in pad]) for pad in padding])
# Reshape 2D sequential data if required
# Log input shape
inpshape = inp.shape
reallyissequential = issequence and inp.ndim == 5
if issequence:
if reallyissequential:
# Sequential input must be paddable by theano. This is required to reshape the sequential input back to
# its original shape after pooling.
assert autopaddable, "Sequential inputs must be paddable by theano. Provided padding {} cannot be " \
"handled at present.".format(padding)
inp = inp.reshape((inpshape[0] * inpshape[1], inpshape[2],
inpshape[3], inpshape[4]),
ndim=4)
ds = ds[0:2]
stride = stride[0:2]
padding = padding[0:2]
else:
warn(
"Expected 5D sequential output, but got 4D non-sequential instead."
)
# Determine what theano needs to be told about how to pad the input
if autopaddable:
autopadding = tuple([pad[0] for pad in padding])
else:
autopadding = (0, ) * dim
if not autopaddable and not all(
[padval == 0 for padval in pyk.flatten(padding)]):
if not th.config.device == 'cpu' and not self.cpupadwarned:
warn(
"Padding might occur on the CPU, which tends to slow things down."
)
self.cpupadwarned = True
inp = nu.pad(inp, padding)
if dim == 2:
y = downsample.pool_2d(input=inp,
ds=ds,
st=stride,
padding=autopadding,
ignore_border=ignoreborder,
mode=poolmode)
elif dim == 3:
# parse downsampling ratio, stride and padding
dsyx = ds[0:2]
styx = stride[0:2]
padyx = autopadding[0:2]
ds0z = (1, ds[2])
st0z = (1, stride[2])
pad0z = (0, autopadding[2])
# Dowsnample yx
H = downsample.pool_2d(input=inp,
ds=dsyx,
st=styx,
padding=padyx,
mode=poolmode)
# Rotate tensor
H = H.dimshuffle(0, 2, 3, 4, 1)
# Downsample 0z
H = downsample.pool_2d(input=H,
ds=ds0z,
st=st0z,
padding=pad0z,
mode=poolmode)
# Undo rotate tensor
y = H.dimshuffle(0, 4, 1, 2, 3)
else:
raise NotImplementedError("Pooling is implemented in 2D and 3D.")
if issequence and reallyissequential:
# Compute symbolic pool output length
if ignoreborder:
pooleny, poolenx = \
[T.floor((inpshape[tensorindex] + 2 * autopadding[index] - ds[index] + stride[index])/stride[index])
for index, tensorindex in enumerate([3, 4])]
else:
poolen = [None, None]
for index, tensorindex in enumerate([3, 4]):
if stride[index] >= ds[index]:
poolen[index] = T.floor(
(inpshape[tensorindex] + stride[index] - 1) /
stride[index])
else:
plen = T.floor((inpshape[tensorindex] - ds[index] +
stride[index] - 1) / stride[index])
poolen[index] = T.switch(plen > 0, plen, 0)
pooleny, poolenx = poolen
y = y.reshape(
(inpshape[0], inpshape[1], inpshape[2], pooleny, poolenx),
ndim=5)
return y
