def __init__(self,
nd,
maps,
epsilon=1e-5,
initFactor=1.0,
minFactor=0.1,
sscale=0.01,
affine=True,
name=None,
empty=False,
inplace=False):
super().__init__(name)
self.inplace = inplace
if inplace and Config.showWarnings:
print("[%s] Warning: %s is using inplace flag" %
(Config.libname, self))
self.maps = maps
self.epsilon = epsilon
self.initFactor = initFactor
self.minFactor = minFactor
self.numOfProps = 0
self.affine = affine
self.scale, self.bias, self.mean, self.var = None, None, None, None
self.savemean, self.saveinvvar, self.scalegrad, self.biasgrad = None, None, None, None
if empty:
return
if not affine:
sscale = 0
shape = (1, maps) + self.repeat(1, nd)
self.setVar(
"scale",
Variable(
gpuarray.to_gpu(
np.random.normal(1.0, sscale,
shape).astype(self.calctype))))
self.setVar("bias", Variable(gpuarray.zeros(shape,
dtype=self.calctype)))
self.setAttr("mean", gpuarray.zeros(shape, dtype=self.calctype))
self.setAttr("var", gpuarray.to_gpu(np.ones(shape,
dtype=self.calctype)))
def updateGrad(self, grad):
if self.getSequences:
fullgrad = grad
else:
seqlen = self.fulldata.shape[0]
if self.direction == DirectionMode.uni:
fullgrad = gpuarray.empty((seqlen, ) + grad.shape,
dtype=grad.dtype,
allocator=memPool)
fullgrad[:seqlen - 1].fill(0.0)
fullgrad[seqlen - 1].set(grad)
else:
fwdgrad, bwdgrad = grad
batchsize, hsize = fwdgrad.shape[0], 2 * self.hsize
fullgrad = gpuarray.zeros((seqlen, batchsize, hsize),
dtype=fwdgrad.dtype,
allocator=memPool)
fullgrad[0, :, bwdgrad.shape[1]:].set(bwdgrad)
fullgrad[-1, :, :fwdgrad.shape[1]].set(fwdgrad)
self.grad, self.reserve = backwardDataRnn(fullgrad, self.fulldata,
self.W, self.reserve,
self.descRnn)
def setupBias(self, insize, outsize):
if not self.useBias:
return
size = outsize if self.useW else insize
bshape = (self.groups,
size) if self.wmode == GroupMode.full else (1, size)
self.setVar("b", Variable(gpuarray.zeros(bshape, dtype=np.float32)))
def calcVal(self, pred, target): diff = gpuarray.empty(pred.shape, dtype=np.float32, allocator=memPool) fullnorm = 1.0 / np.prod(target.shape) devErr = gpuarray.zeros((), dtype=np.float32, allocator=memPool) smoothL1Ker(pred, target, devErr, diff, fullnorm, fullnorm) return devErr.get()
def __init__(self, config, bertModelEmbeddingWeights, name=None):
super().__init__(name)
self.registerBlueprint(locals())
self.append(Linear(config.hidden_size, config.hidden_size, name='dense'))
self.append(Gelu(name='gelu'))
self.append(BertLayerNorm(config, name='LayerNorm'))
self.append(Linear(bertModelEmbeddingWeights.shape[1], bertModelEmbeddingWeights.shape[0], name='decoder'))
self.modules['decoder'].setVar('W', Variable(bertModelEmbeddingWeights))
self.modules['decoder'].setVar('b', Variable(gpuarray.zeros((bertModelEmbeddingWeights.shape[1],), dtype = np.float32)))
def __init__(self,
nd,
inmaps,
outmaps,
size,
stride=1,
pad=0,
dilation=1,
wscale=1.0,
useBias=True,
name=None,
initscheme=None,
empty=False,
groups=1):
super().__init__(name)
self.stride = self.repeat(stride, nd)
self.pad = self.repeat(pad, nd)
self.dilation = self.repeat(dilation, nd)
self.useBias = useBias
self.groups = groups
self.fwdAlgo, self.bwdFilterAlgo, self.bwdDataAlgo = None, None, None
self.installDefaultAlgos()
if inmaps % groups != 0 or outmaps % groups != 0:
raise ModuleError(
"Number of input and output maps must be divisible by number of groups "
"(%d inmaps, %d outmaps, %d groups)" %
(inmaps, outmaps, groups))
outmaps //= groups
self.W = None
self.b = None
if empty:
return
Wshape = (inmaps, outmaps, *self.repeat(size, nd))
W = self.createTensorWithScheme(initscheme, Wshape, wscale,
self.calcNeuronsNumber(Wshape, True))
self.setVar(
"W",
Variable(
gpuarray.empty(Wshape, dtype=self.calctype
) if W is None else gpuarray.to_gpu(W)))
if useBias:
bshape = (1, outmaps) + self.repeat(1, nd)
self.setVar("b",
Variable(gpuarray.zeros(bshape, dtype=self.calctype)))
def calcVal(self, scores, labels):
if Config.verifyData:
self.verifyLabels(labels)
diff = gpuarray.empty(scores.shape,
dtype=np.float32,
allocator=memPool)
devErr = gpuarray.zeros((), dtype=np.float32, allocator=memPool)
hingeKer(scores, labels, devErr, diff, scores.shape[0],
scores.shape[1])
return devErr.get() / scores.shape[0]
def __init__(self,
vsize,
hsize,
wscale=1.0,
rng=globalRng,
useBias=True,
name=None):
super().__init__(name)
self.rng = rng
W = np.random.normal(0.0, wscale / math.sqrt(vsize + hsize),
(vsize, hsize)).astype(np.float32)
self.W = None
self.setVar("W", Variable(gpuarray.to_gpu(W, allocator=memPool)))
self.useBias = useBias
if useBias:
self.b = None
self.setVar(
"b",
Variable(
gpuarray.zeros((vsize, ),
dtype=np.float32,
allocator=memPool)))
self.c = None
self.setVar(
"c",
Variable(
gpuarray.zeros((hsize, ),
dtype=np.float32,
allocator=memPool)))
self.particles = None
def updateData(self, data):
if self.acquireDtypesFrom(data) == np.int32:
inputIds = data
tokenTypeIds = gpuarray.zeros(inputIds.shape, dtype=np.int32)
else:
inputIds, tokenTypeIds = data
seqlength = inputIds.shape[1]
positionIds = gpuarray.to_gpu(np.array([range(seqlength)]*inputIds.shape[0]).astype(np.int32))
wordsEmbeddings = self.modules['wordEmbedder'](inputIds)
positionEmbeddings = self.modules['positionEmbedder'](positionIds)
tokenTypeEmbeddings = self.modules['tokenTypeEmbedder'](tokenTypeIds)
embeddings = wordsEmbeddings + positionEmbeddings + tokenTypeEmbeddings
embeddings = self.modules['LayerNorm'](embeddings)
self.data = embeddings
def __init__(self, numOfMaps, epsilon=1e-5, affine=True, name=None):
super().__init__(name)
self.registerBlueprint(locals())
self.numOfMaps = numOfMaps
self.epsilon = epsilon
self.affine = affine
shape = (1, numOfMaps, 1, 1)
scale = np.ones(shape, dtype=np.float32)
self.scale = None
self.bias = None
self.setVar("scale", Variable(gpuarray.to_gpu(scale)))
self.setVar("bias", Variable(gpuarray.zeros(shape, dtype=np.float32)))
self.savemean, self.saveinvvar, self.extscale, self.scalegrad, self.biasgrad = None, None, None, None, None
def __init__(self,
insize,
outsize,
wscale=1.0,
useBias=True,
initscheme=None,
name=None,
empty=False,
transpose=False):
super().__init__(name)
self.registerBlueprint(locals())
self.transpose = transpose
self.useBias = useBias
self.W = None
self.b = None
if empty:
return
Wshape, bshape = ((outsize, insize),
(insize, )) if transpose else ((insize, outsize),
(outsize, ))
W = self.createTensorWithScheme(initscheme,
Wshape,
wscale,
factorShape=Wshape)
self.setVar(
"W",
Variable(
gpuarray.empty(Wshape, dtype=self.calctype
) if W is None else gpuarray.to_gpu(W)))
if useBias:
self.setVar("b",
Variable(gpuarray.zeros(bshape, dtype=self.calctype)))
def __init__(self, data, name=None, withgrad=True, grad=None, updater=None, postUpdater=None): if name is None: self.name = str(type(self).index) type(self).index += 1 else: self.name = name self.data = data self.updater = updater if updater is not None: return self.postUpdater = postUpdater self.grad = None if grad is not None: self.grad = grad elif withgrad and not Config.globalEvalMode: self.grad = gpuarray.zeros(shape=self.data.shape, dtype=self.data.dtype) self.learnRate, self.momRate = 1.0, 1.0 self.wc = 0.0
def setupState(self, var):
return {"mom": gpuarray.zeros(var.data.shape, dtype=var.data.dtype)}
def setupState(self, var):
return {
"mg": gpuarray.zeros(var.data.shape, dtype=var.data.dtype),
"ms": gpuarray.zeros(var.data.shape, dtype=var.data.dtype),
"delta": gpuarray.zeros(var.data.shape, dtype=var.data.dtype)
}
def setupState(self, var):
return {
"mg": gpuarray.zeros(var.data.shape, dtype=np.float32),
"ms": gpuarray.zeros(var.data.shape, dtype=np.float32)
}