def buildOutGrad(self, grad): if len(self.fwds) == 0: return grad grad = [[] for _ in range( len(self.data) if isinstance(self.data, list) else 1)] for node, slots in self.fwds: if slots is not None: for slot in slots: grad[slot].append(node.grad[self.name][slot]) else: for i, gr in enumerate(node.grad[self.name]): grad[i].append(gr) for i, grads in enumerate(grad): if len(grads) > 1: gr = copy(None, grads[0]) for j in range(1, len(grads)): Blas.toVectorAddVector(gr.ravel(), grads[j].ravel()) else: gr = grads[0] grad[i] = gr if len(grad) == 1: grad = grad[0] return grad
def updateGrad(self, grad): sections = [grad.shape[self.axis] // self.times] * self.times ingrad = Utils.split(grad, sections, axis=self.axis) for i in range(1, len(ingrad)): Blas.toVectorAddVector(ingrad[0].ravel(), ingrad[i].ravel()) self.grad = ingrad[0]
def accGradParams(self, grad, scale=1.0, momentum=0.0): self.dw = backwardParamsRnn(self.inData, self.fulldata, self.W, self.reserve, self.descRnn) Blas.addVectorToVector(self.dw, self.getVar("W").grad, out=self.getVar("W").grad, alpha=scale, beta=momentum)
def calcVal(self, pred, target): diff = Blas.addVectorToVector(pred.ravel(), target.ravel(), alpha=1.0, beta=-1.0) error = Blas.vectorL1Norm(diff) / np.prod(target.shape) return error
def calcError(self, pred, target): diff = Blas.addVectorToVector(pred.ravel(), target.ravel(), alpha=1.0, beta=-1.0) self.devErr.fill(Blas.vectorL1Norm(diff) / np.prod(pred.shape[1:])) self.accumErr += self.devErr
def calcVal(self, pred, target): diff = Blas.addVectorToVector(target.ravel(), pred.ravel(), alpha=1.0, beta=-1.0) error = Blas.dot(diff, diff) / (2.0 * np.prod(target.shape)) return error
def updateGrad(self, grad): firstgrad = grad[0] self.grad = gpuarray.empty(firstgrad.shape, dtype=firstgrad.dtype, allocator=memPool) self.grad.fill(0) for gr in grad: Blas.toVectorAddVector(self.grad.ravel(), gr.ravel())
def updateData(self, data): firstdata = data[0] self.data = gpuarray.empty(firstdata.shape, dtype=firstdata.dtype, allocator=memPool) self.data.fill(0) for dat in data: Blas.toVectorAddVector(self.data.ravel(), dat.ravel())
def accGradParams(self, grad, scale=1.0, momentum=0.0): if self.affine: Blas.addVectorToVector(self.scalegrad.ravel(), self.vars["scale"].grad.ravel(), out=self.vars["scale"].grad.ravel(), alpha=scale, beta=momentum) Blas.addVectorToVector(self.biasgrad.ravel(), self.vars["bias"].grad.ravel(), out=self.vars["bias"].grad.ravel(), alpha=scale, beta=momentum)
def updateData(self, data): self.data = Blas.mulMatrixOnMatrix(data, self.W, transpB=self.transpose) if self.useBias: MatVec.addVecToMat(self.b, self.data, axis=1, out=self.data)
def visibleFromHidden(self, hidden): visible = Blas.mulMatrixOnMatrix(hidden, self.W, transpB=True) if self.useBias: addVecToMat(self.b, visible, axis=1, out=visible) self.activateNeurons(visible) return visible
def hiddenFromVisible(self, visible): hidden = Blas.mulMatrixOnMatrix(visible, self.W) if self.useBias: addVecToMat(self.c, hidden, axis=1, out=hidden) self.activateNeurons(hidden) return hidden
def calcGrad(self, pred, target): c = 1.0 / np.prod(target.shape) grad = Blas.addVectorToVector(target.ravel(), pred.ravel(), alpha=c, beta=-c) grad = grad.reshape(pred.shape) return grad
def accGradParams(self, grad, scale=1.0, momentum=0.0): if self.wmode == GroupMode.full: if self.useW: A, B = (grad, self.inData) if self.transpW else (self.inData, grad) BlasGroup.mulTensorBatch(A, B, out=self.vars["W"].grad, formatA=self.format, formatB=self.format, formatOut="gbp", transpA=True, alpha=scale, beta=momentum) if self.useBias: BlasGroup.sumOnTensorGroup(grad, out=self.vars["b"].grad, formatT=self.format) else: if self.useW: A, B = (grad, self.inData) if self.transpW else (self.inData, grad) wgrad = BlasGroup.mulTensorBatch(A, B, transpA=True, formatA=self.format, formatB=self.format, formatOut="gbp", alpha=scale, beta=momentum) Blas.sumOnMatrix(wgrad.reshape(wgrad.shape[0], -1), out=self.vars["W"].grad.ravel()) if self.useBias: Blas.sumOnMatrix(grad.reshape(grad.shape[0] * grad.shape[1], grad.shape[2]), out=self.vars["b"].grad[0])
def updateGrad(self, grad): self.grad, meansGrad = mapLRNBackward(self.inData, self.data, grad, self.means, None, N=self.N, alpha=self.alpha, beta=self.beta, K=self.K) if self.includePad: meansGrad = poolNdBackward(self.inData, self.means, meansGrad, self.workspace, size=self.size, stride=1, pad=self.pad, mode=self.mode) Blas.addVectorToVector(self.grad.ravel(), meansGrad.ravel(), out=self.grad.ravel(), beta=-1.0)
def updateData(self, data): reshape = len(data.shape) > 2 if reshape: reshape2d = Reshape( (int(np.prod(data.shape[:-1])), data.shape[-1])) reshape2d.calcMode(self.calctype) reshapeNd = Reshape(data.shape[:-1] + tuple([self.W.shape[1]])) reshapeNd.calcMode(self.calctype) data = reshape2d(data) self.data = Blas.mulMatrixOnMatrix(data, self.W, transpB=self.transpose) if self.useBias: MatVec.addVecToMat(self.b, self.data, axis=1, out=self.data) if reshape: self.data = reshapeNd(self.data)
def posPhaseGrad(self, data): hidden = self.hiddenFromVisible(data) Blas.mulMatrixOnMatrix(data, hidden, out=self.vars["W"].grad, transpA=True) if self.useBias: Blas.sumOnMatrix(data, out=self.vars["b"].grad) Blas.sumOnMatrix(hidden, out=self.vars["c"].grad) return hidden
def updateGrad(self, grad): if self.mode == PenaltyMode.l1: self.grad = gpuarray.empty(grad.shape, dtype=grad.dtype, allocator=memPool) l1penaltyKer(self.grad, grad, self.data, self.weight / grad.shape[0]) elif self.mode == PenaltyMode.l2: self.grad = Blas.addVectorToVector(grad.ravel(), self.data.ravel(), alpha=1.0, beta=-self.weight / grad.shape[0]) self.grad = self.grad.reshape(grad.shape) else: raise NotImplementedError(self.mode)
def updateGrad(self, grad): if self.useW: formatOut = self.format if self.inmode == GroupMode.full else "gbp" self.grad = BlasGroup.mulTensorBatch(grad, self.W, formatA=self.format, formatB="gbp", transpB=not self.transpW, formatOut=formatOut) if self.inmode != GroupMode.full: self.grad = Blas.sumOnMatrix( self.grad.reshape(self.groups, grad.shape[0] * self.W.shape[1])) self.grad = self.grad.reshape(grad.shape[0], 1, self.W.shape[1]) else: self.grad = grad
def accGradParams(self, grad, scale=1.0, momentum=0.0): if not self.transpose: Blas.mulMatrixOnMatrix(self.inData, grad, out=self.vars["W"].grad, transpA=True, alpha=scale, beta=momentum) else: Blas.mulMatrixOnMatrix(grad, self.inData, out=self.vars["W"].grad, transpA=True, alpha=scale, beta=momentum) if self.useBias: Blas.sumOnMatrix(grad, out=self.vars["b"].grad, alpha=scale, beta=momentum)
def negPhaseGrad(self, hidden): visible = self.visibleFromHidden(hidden) hidden = self.hiddenFromVisible(visible) Blas.mulMatrixOnMatrix(visible, hidden, out=self.vars["W"].grad, transpA=True, alpha=-1.0, beta=1.0) if self.useBias: Blas.sumOnMatrix(visible, out=self.vars["b"].grad, alpha=-1.0, beta=1.0) Blas.sumOnMatrix(hidden, out=self.vars["c"].grad, alpha=-1.0, beta=1.0) return hidden
def updateGrad(self, grad): meansGrad = poolNdBackward(self.inData, self.means, grad, self.workspace, size=self.size, stride=1, pad=self.pad, mode=self.mode) Blas.addVectorToVector(grad.ravel(), meansGrad.ravel(), out=meansGrad.ravel(), beta=-1.0) self.grad = meansGrad
def updateGrad(self, grad): self.grad = Blas.mulMatrixOnMatrix(grad, self.W, transpB=not self.transpose)
def updateData(self, data): self.means, self.workspace = poolNd(data, size=self.size, stride=1, pad=self.pad, mode=self.mode, test=not self.train) self.data = Blas.addVectorToVector(data.ravel(), self.means.ravel(), beta=-1.0).reshape(*data.shape)
def updateParams(self, learnRate): for var in self.vars.values(): Blas.toVectorAddVector(var.data.ravel(), var.grad.ravel(), alpha=learnRate)
def calcError(self, pred, target): self.devErr.fill( Blas.dot(self.grad.ravel(), self.grad.ravel()) * np.prod(self.grad.shape) * self.grad.shape[0] / 2.0) self.accumErr += self.devErr