def __init__(self, nd, maps, epsilon=1e-5, initFactor=1.0, minFactor=0.1, sscale=0.01, affine=True, name=None, empty=False, inplace=False): super().__init__(name) self.inplace = inplace if inplace and Config.showWarnings: print("[%s] Warning: %s is using inplace flag" % (Config.libname, self)) self.maps = maps self.epsilon = epsilon self.initFactor = initFactor self.minFactor = minFactor self.numOfProps = 0 self.affine = affine self.scale, self.bias, self.mean, self.var = None, None, None, None self.savemean, self.saveinvvar, self.scalegrad, self.biasgrad = None, None, None, None if empty: return if not affine: sscale = 0 shape = (1, maps) + self.repeat(1, nd) self.setVar( "scale", Variable( gpuarray.to_gpu( np.random.normal(1.0, sscale, shape).astype(self.calctype)))) self.setVar("bias", Variable(gpuarray.zeros(shape, dtype=self.calctype))) self.setAttr("mean", gpuarray.zeros(shape, dtype=self.calctype)) self.setAttr("var", gpuarray.to_gpu(np.ones(shape, dtype=self.calctype)))
def updateGrad(self, grad): if self.getSequences: fullgrad = grad else: seqlen = self.fulldata.shape[0] if self.direction == DirectionMode.uni: fullgrad = gpuarray.empty((seqlen, ) + grad.shape, dtype=grad.dtype, allocator=memPool) fullgrad[:seqlen - 1].fill(0.0) fullgrad[seqlen - 1].set(grad) else: fwdgrad, bwdgrad = grad batchsize, hsize = fwdgrad.shape[0], 2 * self.hsize fullgrad = gpuarray.zeros((seqlen, batchsize, hsize), dtype=fwdgrad.dtype, allocator=memPool) fullgrad[0, :, bwdgrad.shape[1]:].set(bwdgrad) fullgrad[-1, :, :fwdgrad.shape[1]].set(fwdgrad) self.grad, self.reserve = backwardDataRnn(fullgrad, self.fulldata, self.W, self.reserve, self.descRnn)
def setupBias(self, insize, outsize): if not self.useBias: return size = outsize if self.useW else insize bshape = (self.groups, size) if self.wmode == GroupMode.full else (1, size) self.setVar("b", Variable(gpuarray.zeros(bshape, dtype=np.float32)))
def calcVal(self, pred, target): diff = gpuarray.empty(pred.shape, dtype=np.float32, allocator=memPool) fullnorm = 1.0 / np.prod(target.shape) devErr = gpuarray.zeros((), dtype=np.float32, allocator=memPool) smoothL1Ker(pred, target, devErr, diff, fullnorm, fullnorm) return devErr.get()
def __init__(self, config, bertModelEmbeddingWeights, name=None): super().__init__(name) self.registerBlueprint(locals()) self.append(Linear(config.hidden_size, config.hidden_size, name='dense')) self.append(Gelu(name='gelu')) self.append(BertLayerNorm(config, name='LayerNorm')) self.append(Linear(bertModelEmbeddingWeights.shape[1], bertModelEmbeddingWeights.shape[0], name='decoder')) self.modules['decoder'].setVar('W', Variable(bertModelEmbeddingWeights)) self.modules['decoder'].setVar('b', Variable(gpuarray.zeros((bertModelEmbeddingWeights.shape[1],), dtype = np.float32)))
def __init__(self, nd, inmaps, outmaps, size, stride=1, pad=0, dilation=1, wscale=1.0, useBias=True, name=None, initscheme=None, empty=False, groups=1): super().__init__(name) self.stride = self.repeat(stride, nd) self.pad = self.repeat(pad, nd) self.dilation = self.repeat(dilation, nd) self.useBias = useBias self.groups = groups self.fwdAlgo, self.bwdFilterAlgo, self.bwdDataAlgo = None, None, None self.installDefaultAlgos() if inmaps % groups != 0 or outmaps % groups != 0: raise ModuleError( "Number of input and output maps must be divisible by number of groups " "(%d inmaps, %d outmaps, %d groups)" % (inmaps, outmaps, groups)) outmaps //= groups self.W = None self.b = None if empty: return Wshape = (inmaps, outmaps, *self.repeat(size, nd)) W = self.createTensorWithScheme(initscheme, Wshape, wscale, self.calcNeuronsNumber(Wshape, True)) self.setVar( "W", Variable( gpuarray.empty(Wshape, dtype=self.calctype ) if W is None else gpuarray.to_gpu(W))) if useBias: bshape = (1, outmaps) + self.repeat(1, nd) self.setVar("b", Variable(gpuarray.zeros(bshape, dtype=self.calctype)))
def calcVal(self, scores, labels): if Config.verifyData: self.verifyLabels(labels) diff = gpuarray.empty(scores.shape, dtype=np.float32, allocator=memPool) devErr = gpuarray.zeros((), dtype=np.float32, allocator=memPool) hingeKer(scores, labels, devErr, diff, scores.shape[0], scores.shape[1]) return devErr.get() / scores.shape[0]
def __init__(self, vsize, hsize, wscale=1.0, rng=globalRng, useBias=True, name=None): super().__init__(name) self.rng = rng W = np.random.normal(0.0, wscale / math.sqrt(vsize + hsize), (vsize, hsize)).astype(np.float32) self.W = None self.setVar("W", Variable(gpuarray.to_gpu(W, allocator=memPool))) self.useBias = useBias if useBias: self.b = None self.setVar( "b", Variable( gpuarray.zeros((vsize, ), dtype=np.float32, allocator=memPool))) self.c = None self.setVar( "c", Variable( gpuarray.zeros((hsize, ), dtype=np.float32, allocator=memPool))) self.particles = None
def updateData(self, data): if self.acquireDtypesFrom(data) == np.int32: inputIds = data tokenTypeIds = gpuarray.zeros(inputIds.shape, dtype=np.int32) else: inputIds, tokenTypeIds = data seqlength = inputIds.shape[1] positionIds = gpuarray.to_gpu(np.array([range(seqlength)]*inputIds.shape[0]).astype(np.int32)) wordsEmbeddings = self.modules['wordEmbedder'](inputIds) positionEmbeddings = self.modules['positionEmbedder'](positionIds) tokenTypeEmbeddings = self.modules['tokenTypeEmbedder'](tokenTypeIds) embeddings = wordsEmbeddings + positionEmbeddings + tokenTypeEmbeddings embeddings = self.modules['LayerNorm'](embeddings) self.data = embeddings
def __init__(self, numOfMaps, epsilon=1e-5, affine=True, name=None): super().__init__(name) self.registerBlueprint(locals()) self.numOfMaps = numOfMaps self.epsilon = epsilon self.affine = affine shape = (1, numOfMaps, 1, 1) scale = np.ones(shape, dtype=np.float32) self.scale = None self.bias = None self.setVar("scale", Variable(gpuarray.to_gpu(scale))) self.setVar("bias", Variable(gpuarray.zeros(shape, dtype=np.float32))) self.savemean, self.saveinvvar, self.extscale, self.scalegrad, self.biasgrad = None, None, None, None, None
def __init__(self, insize, outsize, wscale=1.0, useBias=True, initscheme=None, name=None, empty=False, transpose=False): super().__init__(name) self.registerBlueprint(locals()) self.transpose = transpose self.useBias = useBias self.W = None self.b = None if empty: return Wshape, bshape = ((outsize, insize), (insize, )) if transpose else ((insize, outsize), (outsize, )) W = self.createTensorWithScheme(initscheme, Wshape, wscale, factorShape=Wshape) self.setVar( "W", Variable( gpuarray.empty(Wshape, dtype=self.calctype ) if W is None else gpuarray.to_gpu(W))) if useBias: self.setVar("b", Variable(gpuarray.zeros(bshape, dtype=self.calctype)))
def __init__(self, data, name=None, withgrad=True, grad=None, updater=None, postUpdater=None): if name is None: self.name = str(type(self).index) type(self).index += 1 else: self.name = name self.data = data self.updater = updater if updater is not None: return self.postUpdater = postUpdater self.grad = None if grad is not None: self.grad = grad elif withgrad and not Config.globalEvalMode: self.grad = gpuarray.zeros(shape=self.data.shape, dtype=self.data.dtype) self.learnRate, self.momRate = 1.0, 1.0 self.wc = 0.0
def setupState(self, var): return {"mom": gpuarray.zeros(var.data.shape, dtype=var.data.dtype)}
def setupState(self, var): return { "mg": gpuarray.zeros(var.data.shape, dtype=var.data.dtype), "ms": gpuarray.zeros(var.data.shape, dtype=var.data.dtype), "delta": gpuarray.zeros(var.data.shape, dtype=var.data.dtype) }
def setupState(self, var): return { "mg": gpuarray.zeros(var.data.shape, dtype=np.float32), "ms": gpuarray.zeros(var.data.shape, dtype=np.float32) }