def __init__(self, config, epsilon=1e-12, name=None): super().__init__(name) self.registerBlueprint(locals()) self.epsilon = epsilon self.scale = None self.setVar("scale", Variable(gpuarray.to_gpu(np.ones(config.hidden_size, dtype=np.float32)))) self.bias = None self.setVar("bias", Variable(gpuarray.to_gpu(np.zeros(config.hidden_size, dtype=np.float32)))) self.mul = Mul()
def __init__(self, config, bertModelEmbeddingWeights, name=None): super().__init__(name) self.registerBlueprint(locals()) self.append(Linear(config.hidden_size, config.hidden_size, name='dense')) self.append(Gelu(name='gelu')) self.append(BertLayerNorm(config, name='LayerNorm')) self.append(Linear(bertModelEmbeddingWeights.shape[1], bertModelEmbeddingWeights.shape[0], name='decoder')) self.modules['decoder'].setVar('W', Variable(bertModelEmbeddingWeights)) self.modules['decoder'].setVar('b', Variable(gpuarray.zeros((bertModelEmbeddingWeights.shape[1],), dtype = np.float32)))
def __init__(self, nd, inmaps, outmaps, size, stride=1, pad=0, dilation=1, wscale=1.0, useBias=True, name=None, initscheme=None, empty=False, groups=1): super().__init__(name) self.stride = self.repeat(stride, nd) self.pad = self.repeat(pad, nd) self.dilation = self.repeat(dilation, nd) self.useBias = useBias self.groups = groups self.fwdAlgo, self.bwdFilterAlgo, self.bwdDataAlgo = None, None, None self.installDefaultAlgos() if inmaps % groups != 0 or outmaps % groups != 0: raise ModuleError( "Number of input and output maps must be divisible by number of groups " "(%d inmaps, %d outmaps, %d groups)" % (inmaps, outmaps, groups)) outmaps //= groups self.W = None self.b = None if empty: return Wshape = (inmaps, outmaps, *self.repeat(size, nd)) W = self.createTensorWithScheme(initscheme, Wshape, wscale, self.calcNeuronsNumber(Wshape, True)) self.setVar( "W", Variable( gpuarray.empty(Wshape, dtype=self.calctype ) if W is None else gpuarray.to_gpu(W))) if useBias: bshape = (1, outmaps) + self.repeat(1, nd) self.setVar("b", Variable(gpuarray.zeros(bshape, dtype=self.calctype)))
def setupGlobalState(self, vartable): variables = [(names, var) for var, names in vartable.items()] variables = sorted(variables, key=lambda elem: elem[0][0]) for names, var in variables: if var.hasUpdater: assert self.nodeinfo is None self.customVars.append(names[0]) continue shape, dtype = var.data.shape, var.data.dtype.type shParams = self.shParams.get(dtype, SharedArray(dtype)) shGrads = self.shGrads.get(dtype, SharedArray(dtype)) shParams.register(var.data.shape, var.data.dtype.type, names[0]) shGrads.register(var.grad.shape, var.grad.dtype.type, names[0]) self.shParams[dtype] = shParams self.shGrads[dtype] = shGrads for shParams, shGrads in zip(self.shParams.values(), self.shGrads.values()): shParams.build() shGrads.build() self.globalVar[shParams.dtype] = Variable(shParams.ary, grad=shGrads.ary) for names, var in variables: if var.hasUpdater: continue dtype = var.data.dtype.type data, grad = self.shParams[dtype][names[0]], self.shGrads[dtype][ names[0]] data.set(var.data) grad.set(var.grad) for name in names: self.module.setVar(name, Variable(data, grad=grad)) for dtype, globalVar in self.globalVar.items(): if self.nodeinfo is not None: self.nodeinfo.broadcastBuffer("data", globalVar.data.gpudata) self.states[dtype] = self.setupState(globalVar)
def __init__(self, nd, maps, epsilon=1e-5, initFactor=1.0, minFactor=0.1, sscale=0.01, affine=True, name=None, empty=False, inplace=False): super().__init__(name) self.inplace = inplace if inplace and Config.showWarnings: print("[%s] Warning: %s is using inplace flag" % (Config.libname, self)) self.maps = maps self.epsilon = epsilon self.initFactor = initFactor self.minFactor = minFactor self.numOfProps = 0 self.affine = affine self.scale, self.bias, self.mean, self.var = None, None, None, None self.savemean, self.saveinvvar, self.scalegrad, self.biasgrad = None, None, None, None if empty: return if not affine: sscale = 0 shape = (1, maps) + self.repeat(1, nd) self.setVar( "scale", Variable( gpuarray.to_gpu( np.random.normal(1.0, sscale, shape).astype(self.calctype)))) self.setVar("bias", Variable(gpuarray.zeros(shape, dtype=self.calctype))) self.setAttr("mean", gpuarray.zeros(shape, dtype=self.calctype)) self.setAttr("var", gpuarray.to_gpu(np.ones(shape, dtype=self.calctype)))
def __init__(self, name=None): super().__init__(name) self.setVar( "var", Variable(gpuarray.to_gpu( np.random.randn(10).astype(self.calctype)), withgrad=False))
def __init__(self, insize, hsize, layers=1, mode="relu", direction="uni", dropout=0.0, getSequences=False, initscheme=None, modifier="orthogonal", wscale=1.0, hintBatchSize=None, name=None): super().__init__(name) self.registerBlueprint(locals()) self.gradUsesOutData = True self.insize = insize self.hsize = hsize self.layers = layers self.mode = RNNMode(mode) self.direction = DirectionMode(direction) self.dropout = dropout self.getSequences = getSequences self.hintBatchSize = hintBatchSize mode = { RNNMode.relu: BackendRNNMode.relu, RNNMode.tanh: BackendRNNMode.tanh, RNNMode.lstm: BackendRNNMode.lstm, RNNMode.gru: BackendRNNMode.gru }[self.mode] direction = { DirectionMode.uni: BackendDirectionMode.uni, DirectionMode.bi: BackendDirectionMode.bi }[self.direction] self.descRnn, W, params = createRnn(insize, hsize, layers, mode, direction, dropout, seed=np.random.randint(1 << 31), batchsize=hintBatchSize) self.W = None self.setVar("W", Variable(W)) self.params = params self.initParams(initscheme, wscale, modifier) self.reserve, self.fulldata, self.dw = None, None, None
def setupBias(self, insize, outsize): if not self.useBias: return size = outsize if self.useW else insize bshape = (self.groups, size) if self.wmode == GroupMode.full else (1, size) self.setVar("b", Variable(gpuarray.zeros(bshape, dtype=np.float32)))
def __init__(self, numOfMaps, epsilon=1e-5, affine=True, name=None): super().__init__(name) self.registerBlueprint(locals()) self.numOfMaps = numOfMaps self.epsilon = epsilon self.affine = affine shape = (1, numOfMaps, 1, 1) scale = np.ones(shape, dtype=np.float32) self.scale = None self.bias = None self.setVar("scale", Variable(gpuarray.to_gpu(scale))) self.setVar("bias", Variable(gpuarray.zeros(shape, dtype=np.float32))) self.savemean, self.saveinvvar, self.extscale, self.scalegrad, self.biasgrad = None, None, None, None, None
def __init__(self, vocabulary, sentlength, embsize, onVocabulary=None, initscheme="uniform", wscale=1.0, learnable=True, name=None): super().__init__(name) args = dict(locals()) self.embsize = embsize self.sentlength = sentlength self.wgrad = None self.learnable = learnable self.outgrad = None dt = h5py.special_dtype(vlen=str) if isinstance(vocabulary, dict): vocabsize = len(vocabulary) vocab = np.empty(shape=(vocabsize, ), dtype=dt) for word, idx in vocabulary.items(): vocab[int(idx)] = word elif isinstance(vocabulary, int): vocabsize = vocabulary vocab = np.empty(shape=(0, ), dtype=dt) else: raise ModuleError("Unrecognized vocabulary parameter type") self.vocab = None self.setAttr("vocab", vocab) args["vocabulary"] = vocabsize self.registerBlueprint(args, exclude=["onVocabulary"]) Wshape = (vocabsize, embsize) W = self.createTensorWithScheme(initscheme, Wshape, wscale, self.calcNeuronsNumber(Wshape)) if W is None: W = np.empty(Wshape, dtype=np.float32) if onVocabulary is not None: onVocabulary(W) self.W = None self.setVar("W", Variable(gpuarray.to_gpu(W))) self.loadVarHook = self.checkVarOnLoad self.loadAttrHook = self.checkAttrOnLoad
def checkVarOnLoad(self, paramName, dataset): if paramName == "W": if dataset.shape[1] != self.embsize: raise ModuleError("Expected embedding size %s, was given %s" % (self.embsize, dataset.shape[1])) self.setVar("W", Variable(gpuarray.to_gpu(dataset))) else: raise ModuleError("Unknown parameter name '%s' for embedder" % paramName)
def __init__(self, insize, outsize, wscale=1.0, useBias=True, initscheme=None, name=None, empty=False, transpose=False): super().__init__(name) self.registerBlueprint(locals()) self.transpose = transpose self.useBias = useBias self.W = None self.b = None if empty: return Wshape, bshape = ((outsize, insize), (insize, )) if transpose else ((insize, outsize), (outsize, )) W = self.createTensorWithScheme(initscheme, Wshape, wscale, factorShape=Wshape) self.setVar( "W", Variable( gpuarray.empty(Wshape, dtype=self.calctype ) if W is None else gpuarray.to_gpu(W))) if useBias: self.setVar("b", Variable(gpuarray.zeros(bshape, dtype=self.calctype)))
def calcMode(self, T): if self.calctype == T: return variables = self.vars self.vars = {} for varName, var in variables.items(): self.setVar( varName, Variable(var.data.astype(T), name=var.name, grad=var.grad.astype(T))) self.calctype = T
def __init__(self, vsize, hsize, wscale=1.0, rng=globalRng, useBias=True, name=None): super().__init__(name) self.rng = rng W = np.random.normal(0.0, wscale / math.sqrt(vsize + hsize), (vsize, hsize)).astype(np.float32) self.W = None self.setVar("W", Variable(gpuarray.to_gpu(W, allocator=memPool))) self.useBias = useBias if useBias: self.b = None self.setVar( "b", Variable( gpuarray.zeros((vsize, ), dtype=np.float32, allocator=memPool))) self.c = None self.setVar( "c", Variable( gpuarray.zeros((hsize, ), dtype=np.float32, allocator=memPool))) self.particles = None
def calcMode(self, T): if Config.backend == Config.Backend.cuda: if self.calctype == T: return variables = self.vars self.vars = {} for varName, var in variables.items(): self.setVar(varName, Variable(var.data.astype(T), name=var.name, grad=var.grad.astype(T))) self.calctype = T else: super().calcMode(T)
def setupW(self, insize, outsize, initscheme, wscale): if not self.useW: return asize, bsize = (outsize, insize) if self.transpW else (insize, outsize) groups = self.groups if self.wmode == GroupMode.full else 1 Wshape = (groups, asize, bsize) W = self.createTensorWithScheme( initscheme, Wshape, wscale, self.calcNeuronsNumber(Wshape, self.transpW)) W = gpuarray.empty( Wshape, dtype=np.float32) if W is None else gpuarray.to_gpu(W) self.setVar("W", Variable(W))
def calcMode(self, T): if self.calctype == T: return self.mul.calcMode(T) variables = self.vars self.vars = {} for varName, var in variables.items(): self.setVar(varName, Variable(var.data.astype(T), name=var.name, grad=var.grad.astype(T))) attrs = self.attrs self.attrs = {} for attrName, attr in attrs.items(): self.setAttr(attrName, attr.astype(T)) self.mul.calctype = T self.calctype = T