def __init__(self, config, epsilon=1e-12, name=None):
super().__init__(name)
self.registerBlueprint(locals())
self.epsilon = epsilon
self.scale = None
self.setVar("scale", Variable(gpuarray.to_gpu(np.ones(config.hidden_size, dtype=np.float32))))
self.bias = None
self.setVar("bias", Variable(gpuarray.to_gpu(np.zeros(config.hidden_size, dtype=np.float32))))
self.mul = Mul()
def __init__(self, config, bertModelEmbeddingWeights, name=None):
super().__init__(name)
self.registerBlueprint(locals())
self.append(Linear(config.hidden_size, config.hidden_size, name='dense'))
self.append(Gelu(name='gelu'))
self.append(BertLayerNorm(config, name='LayerNorm'))
self.append(Linear(bertModelEmbeddingWeights.shape[1], bertModelEmbeddingWeights.shape[0], name='decoder'))
self.modules['decoder'].setVar('W', Variable(bertModelEmbeddingWeights))
self.modules['decoder'].setVar('b', Variable(gpuarray.zeros((bertModelEmbeddingWeights.shape[1],), dtype = np.float32)))
def __init__(self,
nd,
inmaps,
outmaps,
size,
stride=1,
pad=0,
dilation=1,
wscale=1.0,
useBias=True,
name=None,
initscheme=None,
empty=False,
groups=1):
super().__init__(name)
self.stride = self.repeat(stride, nd)
self.pad = self.repeat(pad, nd)
self.dilation = self.repeat(dilation, nd)
self.useBias = useBias
self.groups = groups
self.fwdAlgo, self.bwdFilterAlgo, self.bwdDataAlgo = None, None, None
self.installDefaultAlgos()
if inmaps % groups != 0 or outmaps % groups != 0:
raise ModuleError(
"Number of input and output maps must be divisible by number of groups "
"(%d inmaps, %d outmaps, %d groups)" %
(inmaps, outmaps, groups))
outmaps //= groups
self.W = None
self.b = None
if empty:
return
Wshape = (inmaps, outmaps, *self.repeat(size, nd))
W = self.createTensorWithScheme(initscheme, Wshape, wscale,
self.calcNeuronsNumber(Wshape, True))
self.setVar(
"W",
Variable(
gpuarray.empty(Wshape, dtype=self.calctype
) if W is None else gpuarray.to_gpu(W)))
if useBias:
bshape = (1, outmaps) + self.repeat(1, nd)
self.setVar("b",
Variable(gpuarray.zeros(bshape, dtype=self.calctype)))
def setupGlobalState(self, vartable):
variables = [(names, var) for var, names in vartable.items()]
variables = sorted(variables, key=lambda elem: elem[0][0])
for names, var in variables:
if var.hasUpdater:
assert self.nodeinfo is None
self.customVars.append(names[0])
continue
shape, dtype = var.data.shape, var.data.dtype.type
shParams = self.shParams.get(dtype, SharedArray(dtype))
shGrads = self.shGrads.get(dtype, SharedArray(dtype))
shParams.register(var.data.shape, var.data.dtype.type, names[0])
shGrads.register(var.grad.shape, var.grad.dtype.type, names[0])
self.shParams[dtype] = shParams
self.shGrads[dtype] = shGrads
for shParams, shGrads in zip(self.shParams.values(),
self.shGrads.values()):
shParams.build()
shGrads.build()
self.globalVar[shParams.dtype] = Variable(shParams.ary,
grad=shGrads.ary)
for names, var in variables:
if var.hasUpdater:
continue
dtype = var.data.dtype.type
data, grad = self.shParams[dtype][names[0]], self.shGrads[dtype][
names[0]]
data.set(var.data)
grad.set(var.grad)
for name in names:
self.module.setVar(name, Variable(data, grad=grad))
for dtype, globalVar in self.globalVar.items():
if self.nodeinfo is not None:
self.nodeinfo.broadcastBuffer("data", globalVar.data.gpudata)
self.states[dtype] = self.setupState(globalVar)
def __init__(self,
nd,
maps,
epsilon=1e-5,
initFactor=1.0,
minFactor=0.1,
sscale=0.01,
affine=True,
name=None,
empty=False,
inplace=False):
super().__init__(name)
self.inplace = inplace
if inplace and Config.showWarnings:
print("[%s] Warning: %s is using inplace flag" %
(Config.libname, self))
self.maps = maps
self.epsilon = epsilon
self.initFactor = initFactor
self.minFactor = minFactor
self.numOfProps = 0
self.affine = affine
self.scale, self.bias, self.mean, self.var = None, None, None, None
self.savemean, self.saveinvvar, self.scalegrad, self.biasgrad = None, None, None, None
if empty:
return
if not affine:
sscale = 0
shape = (1, maps) + self.repeat(1, nd)
self.setVar(
"scale",
Variable(
gpuarray.to_gpu(
np.random.normal(1.0, sscale,
shape).astype(self.calctype))))
self.setVar("bias", Variable(gpuarray.zeros(shape,
dtype=self.calctype)))
self.setAttr("mean", gpuarray.zeros(shape, dtype=self.calctype))
self.setAttr("var", gpuarray.to_gpu(np.ones(shape,
dtype=self.calctype)))
def __init__(self, name=None):
super().__init__(name)
self.setVar(
"var",
Variable(gpuarray.to_gpu(
np.random.randn(10).astype(self.calctype)),
withgrad=False))
def __init__(self,
insize,
hsize,
layers=1,
mode="relu",
direction="uni",
dropout=0.0,
getSequences=False,
initscheme=None,
modifier="orthogonal",
wscale=1.0,
hintBatchSize=None,
name=None):
super().__init__(name)
self.registerBlueprint(locals())
self.gradUsesOutData = True
self.insize = insize
self.hsize = hsize
self.layers = layers
self.mode = RNNMode(mode)
self.direction = DirectionMode(direction)
self.dropout = dropout
self.getSequences = getSequences
self.hintBatchSize = hintBatchSize
mode = {
RNNMode.relu: BackendRNNMode.relu,
RNNMode.tanh: BackendRNNMode.tanh,
RNNMode.lstm: BackendRNNMode.lstm,
RNNMode.gru: BackendRNNMode.gru
}[self.mode]
direction = {
DirectionMode.uni: BackendDirectionMode.uni,
DirectionMode.bi: BackendDirectionMode.bi
}[self.direction]
self.descRnn, W, params = createRnn(insize,
hsize,
layers,
mode,
direction,
dropout,
seed=np.random.randint(1 << 31),
batchsize=hintBatchSize)
self.W = None
self.setVar("W", Variable(W))
self.params = params
self.initParams(initscheme, wscale, modifier)
self.reserve, self.fulldata, self.dw = None, None, None
def setupBias(self, insize, outsize):
if not self.useBias:
return
size = outsize if self.useW else insize
bshape = (self.groups,
size) if self.wmode == GroupMode.full else (1, size)
self.setVar("b", Variable(gpuarray.zeros(bshape, dtype=np.float32)))
def __init__(self, numOfMaps, epsilon=1e-5, affine=True, name=None):
super().__init__(name)
self.registerBlueprint(locals())
self.numOfMaps = numOfMaps
self.epsilon = epsilon
self.affine = affine
shape = (1, numOfMaps, 1, 1)
scale = np.ones(shape, dtype=np.float32)
self.scale = None
self.bias = None
self.setVar("scale", Variable(gpuarray.to_gpu(scale)))
self.setVar("bias", Variable(gpuarray.zeros(shape, dtype=np.float32)))
self.savemean, self.saveinvvar, self.extscale, self.scalegrad, self.biasgrad = None, None, None, None, None
def __init__(self,
vocabulary,
sentlength,
embsize,
onVocabulary=None,
initscheme="uniform",
wscale=1.0,
learnable=True,
name=None):
super().__init__(name)
args = dict(locals())
self.embsize = embsize
self.sentlength = sentlength
self.wgrad = None
self.learnable = learnable
self.outgrad = None
dt = h5py.special_dtype(vlen=str)
if isinstance(vocabulary, dict):
vocabsize = len(vocabulary)
vocab = np.empty(shape=(vocabsize, ), dtype=dt)
for word, idx in vocabulary.items():
vocab[int(idx)] = word
elif isinstance(vocabulary, int):
vocabsize = vocabulary
vocab = np.empty(shape=(0, ), dtype=dt)
else:
raise ModuleError("Unrecognized vocabulary parameter type")
self.vocab = None
self.setAttr("vocab", vocab)
args["vocabulary"] = vocabsize
self.registerBlueprint(args, exclude=["onVocabulary"])
Wshape = (vocabsize, embsize)
W = self.createTensorWithScheme(initscheme, Wshape, wscale,
self.calcNeuronsNumber(Wshape))
if W is None:
W = np.empty(Wshape, dtype=np.float32)
if onVocabulary is not None:
onVocabulary(W)
self.W = None
self.setVar("W", Variable(gpuarray.to_gpu(W)))
self.loadVarHook = self.checkVarOnLoad
self.loadAttrHook = self.checkAttrOnLoad
def checkVarOnLoad(self, paramName, dataset):
if paramName == "W":
if dataset.shape[1] != self.embsize:
raise ModuleError("Expected embedding size %s, was given %s" %
(self.embsize, dataset.shape[1]))
self.setVar("W", Variable(gpuarray.to_gpu(dataset)))
else:
raise ModuleError("Unknown parameter name '%s' for embedder" %
paramName)
def __init__(self,
insize,
outsize,
wscale=1.0,
useBias=True,
initscheme=None,
name=None,
empty=False,
transpose=False):
super().__init__(name)
self.registerBlueprint(locals())
self.transpose = transpose
self.useBias = useBias
self.W = None
self.b = None
if empty:
return
Wshape, bshape = ((outsize, insize),
(insize, )) if transpose else ((insize, outsize),
(outsize, ))
W = self.createTensorWithScheme(initscheme,
Wshape,
wscale,
factorShape=Wshape)
self.setVar(
"W",
Variable(
gpuarray.empty(Wshape, dtype=self.calctype
) if W is None else gpuarray.to_gpu(W)))
if useBias:
self.setVar("b",
Variable(gpuarray.zeros(bshape, dtype=self.calctype)))
def calcMode(self, T):
if self.calctype == T:
return
variables = self.vars
self.vars = {}
for varName, var in variables.items():
self.setVar(
varName,
Variable(var.data.astype(T),
name=var.name,
grad=var.grad.astype(T)))
self.calctype = T
def __init__(self,
vsize,
hsize,
wscale=1.0,
rng=globalRng,
useBias=True,
name=None):
super().__init__(name)
self.rng = rng
W = np.random.normal(0.0, wscale / math.sqrt(vsize + hsize),
(vsize, hsize)).astype(np.float32)
self.W = None
self.setVar("W", Variable(gpuarray.to_gpu(W, allocator=memPool)))
self.useBias = useBias
if useBias:
self.b = None
self.setVar(
"b",
Variable(
gpuarray.zeros((vsize, ),
dtype=np.float32,
allocator=memPool)))
self.c = None
self.setVar(
"c",
Variable(
gpuarray.zeros((hsize, ),
dtype=np.float32,
allocator=memPool)))
self.particles = None
def calcMode(self, T):
if Config.backend == Config.Backend.cuda:
if self.calctype == T:
return
variables = self.vars
self.vars = {}
for varName, var in variables.items():
self.setVar(varName, Variable(var.data.astype(T), name=var.name, grad=var.grad.astype(T)))
self.calctype = T
else:
super().calcMode(T)
def setupW(self, insize, outsize, initscheme, wscale):
if not self.useW:
return
asize, bsize = (outsize, insize) if self.transpW else (insize, outsize)
groups = self.groups if self.wmode == GroupMode.full else 1
Wshape = (groups, asize, bsize)
W = self.createTensorWithScheme(
initscheme, Wshape, wscale,
self.calcNeuronsNumber(Wshape, self.transpW))
W = gpuarray.empty(
Wshape, dtype=np.float32) if W is None else gpuarray.to_gpu(W)
self.setVar("W", Variable(W))
def calcMode(self, T):
if self.calctype == T:
return
self.mul.calcMode(T)
variables = self.vars
self.vars = {}
for varName, var in variables.items():
self.setVar(varName, Variable(var.data.astype(T), name=var.name, grad=var.grad.astype(T)))
attrs = self.attrs
self.attrs = {}
for attrName, attr in attrs.items():
self.setAttr(attrName, attr.astype(T))
self.mul.calctype = T
self.calctype = T