def buildOutGrad(self, grad):
if len(self.fwds) == 0:
return grad
grad = [[] for _ in range(
len(self.data) if isinstance(self.data, list) else 1)]
for node, slots in self.fwds:
if slots is not None:
for slot in slots:
grad[slot].append(node.grad[self.name][slot])
else:
for i, gr in enumerate(node.grad[self.name]):
grad[i].append(gr)
for i, grads in enumerate(grad):
if len(grads) > 1:
gr = copy(None, grads[0])
for j in range(1, len(grads)):
Blas.toVectorAddVector(gr.ravel(), grads[j].ravel())
else:
gr = grads[0]
grad[i] = gr
if len(grad) == 1:
grad = grad[0]
return grad
def updateGrad(self, grad): sections = [grad.shape[self.axis] // self.times] * self.times ingrad = Utils.split(grad, sections, axis=self.axis) for i in range(1, len(ingrad)): Blas.toVectorAddVector(ingrad[0].ravel(), ingrad[i].ravel()) self.grad = ingrad[0]
def accGradParams(self, grad, scale=1.0, momentum=0.0):
self.dw = backwardParamsRnn(self.inData, self.fulldata, self.W,
self.reserve, self.descRnn)
Blas.addVectorToVector(self.dw,
self.getVar("W").grad,
out=self.getVar("W").grad,
alpha=scale,
beta=momentum)
def calcVal(self, pred, target):
diff = Blas.addVectorToVector(pred.ravel(),
target.ravel(),
alpha=1.0,
beta=-1.0)
error = Blas.vectorL1Norm(diff) / np.prod(target.shape)
return error
def calcError(self, pred, target):
diff = Blas.addVectorToVector(pred.ravel(),
target.ravel(),
alpha=1.0,
beta=-1.0)
self.devErr.fill(Blas.vectorL1Norm(diff) / np.prod(pred.shape[1:]))
self.accumErr += self.devErr
def calcVal(self, pred, target):
diff = Blas.addVectorToVector(target.ravel(),
pred.ravel(),
alpha=1.0,
beta=-1.0)
error = Blas.dot(diff, diff) / (2.0 * np.prod(target.shape))
return error
def updateGrad(self, grad):
firstgrad = grad[0]
self.grad = gpuarray.empty(firstgrad.shape,
dtype=firstgrad.dtype,
allocator=memPool)
self.grad.fill(0)
for gr in grad:
Blas.toVectorAddVector(self.grad.ravel(), gr.ravel())
def updateData(self, data):
firstdata = data[0]
self.data = gpuarray.empty(firstdata.shape,
dtype=firstdata.dtype,
allocator=memPool)
self.data.fill(0)
for dat in data:
Blas.toVectorAddVector(self.data.ravel(), dat.ravel())
def accGradParams(self, grad, scale=1.0, momentum=0.0):
if self.affine:
Blas.addVectorToVector(self.scalegrad.ravel(),
self.vars["scale"].grad.ravel(),
out=self.vars["scale"].grad.ravel(),
alpha=scale,
beta=momentum)
Blas.addVectorToVector(self.biasgrad.ravel(),
self.vars["bias"].grad.ravel(),
out=self.vars["bias"].grad.ravel(),
alpha=scale,
beta=momentum)
def updateData(self, data):
self.data = Blas.mulMatrixOnMatrix(data,
self.W,
transpB=self.transpose)
if self.useBias:
MatVec.addVecToMat(self.b, self.data, axis=1, out=self.data)
def visibleFromHidden(self, hidden):
visible = Blas.mulMatrixOnMatrix(hidden, self.W, transpB=True)
if self.useBias:
addVecToMat(self.b, visible, axis=1, out=visible)
self.activateNeurons(visible)
return visible
def hiddenFromVisible(self, visible):
hidden = Blas.mulMatrixOnMatrix(visible, self.W)
if self.useBias:
addVecToMat(self.c, hidden, axis=1, out=hidden)
self.activateNeurons(hidden)
return hidden
def calcGrad(self, pred, target):
c = 1.0 / np.prod(target.shape)
grad = Blas.addVectorToVector(target.ravel(),
pred.ravel(),
alpha=c,
beta=-c)
grad = grad.reshape(pred.shape)
return grad
def accGradParams(self, grad, scale=1.0, momentum=0.0):
if self.wmode == GroupMode.full:
if self.useW:
A, B = (grad, self.inData) if self.transpW else (self.inData,
grad)
BlasGroup.mulTensorBatch(A,
B,
out=self.vars["W"].grad,
formatA=self.format,
formatB=self.format,
formatOut="gbp",
transpA=True,
alpha=scale,
beta=momentum)
if self.useBias:
BlasGroup.sumOnTensorGroup(grad,
out=self.vars["b"].grad,
formatT=self.format)
else:
if self.useW:
A, B = (grad, self.inData) if self.transpW else (self.inData,
grad)
wgrad = BlasGroup.mulTensorBatch(A,
B,
transpA=True,
formatA=self.format,
formatB=self.format,
formatOut="gbp",
alpha=scale,
beta=momentum)
Blas.sumOnMatrix(wgrad.reshape(wgrad.shape[0], -1),
out=self.vars["W"].grad.ravel())
if self.useBias:
Blas.sumOnMatrix(grad.reshape(grad.shape[0] * grad.shape[1],
grad.shape[2]),
out=self.vars["b"].grad[0])
def updateGrad(self, grad):
self.grad, meansGrad = mapLRNBackward(self.inData,
self.data,
grad,
self.means,
None,
N=self.N,
alpha=self.alpha,
beta=self.beta,
K=self.K)
if self.includePad:
meansGrad = poolNdBackward(self.inData,
self.means,
meansGrad,
self.workspace,
size=self.size,
stride=1,
pad=self.pad,
mode=self.mode)
Blas.addVectorToVector(self.grad.ravel(),
meansGrad.ravel(),
out=self.grad.ravel(),
beta=-1.0)
def updateData(self, data):
reshape = len(data.shape) > 2
if reshape:
reshape2d = Reshape(
(int(np.prod(data.shape[:-1])), data.shape[-1]))
reshape2d.calcMode(self.calctype)
reshapeNd = Reshape(data.shape[:-1] + tuple([self.W.shape[1]]))
reshapeNd.calcMode(self.calctype)
data = reshape2d(data)
self.data = Blas.mulMatrixOnMatrix(data,
self.W,
transpB=self.transpose)
if self.useBias:
MatVec.addVecToMat(self.b, self.data, axis=1, out=self.data)
if reshape:
self.data = reshapeNd(self.data)
def posPhaseGrad(self, data):
hidden = self.hiddenFromVisible(data)
Blas.mulMatrixOnMatrix(data,
hidden,
out=self.vars["W"].grad,
transpA=True)
if self.useBias:
Blas.sumOnMatrix(data, out=self.vars["b"].grad)
Blas.sumOnMatrix(hidden, out=self.vars["c"].grad)
return hidden
def updateGrad(self, grad):
if self.mode == PenaltyMode.l1:
self.grad = gpuarray.empty(grad.shape,
dtype=grad.dtype,
allocator=memPool)
l1penaltyKer(self.grad, grad, self.data,
self.weight / grad.shape[0])
elif self.mode == PenaltyMode.l2:
self.grad = Blas.addVectorToVector(grad.ravel(),
self.data.ravel(),
alpha=1.0,
beta=-self.weight /
grad.shape[0])
self.grad = self.grad.reshape(grad.shape)
else:
raise NotImplementedError(self.mode)
def updateGrad(self, grad):
if self.useW:
formatOut = self.format if self.inmode == GroupMode.full else "gbp"
self.grad = BlasGroup.mulTensorBatch(grad,
self.W,
formatA=self.format,
formatB="gbp",
transpB=not self.transpW,
formatOut=formatOut)
if self.inmode != GroupMode.full:
self.grad = Blas.sumOnMatrix(
self.grad.reshape(self.groups,
grad.shape[0] * self.W.shape[1]))
self.grad = self.grad.reshape(grad.shape[0], 1,
self.W.shape[1])
else:
self.grad = grad
def accGradParams(self, grad, scale=1.0, momentum=0.0):
if not self.transpose:
Blas.mulMatrixOnMatrix(self.inData,
grad,
out=self.vars["W"].grad,
transpA=True,
alpha=scale,
beta=momentum)
else:
Blas.mulMatrixOnMatrix(grad,
self.inData,
out=self.vars["W"].grad,
transpA=True,
alpha=scale,
beta=momentum)
if self.useBias:
Blas.sumOnMatrix(grad,
out=self.vars["b"].grad,
alpha=scale,
beta=momentum)
def negPhaseGrad(self, hidden):
visible = self.visibleFromHidden(hidden)
hidden = self.hiddenFromVisible(visible)
Blas.mulMatrixOnMatrix(visible,
hidden,
out=self.vars["W"].grad,
transpA=True,
alpha=-1.0,
beta=1.0)
if self.useBias:
Blas.sumOnMatrix(visible,
out=self.vars["b"].grad,
alpha=-1.0,
beta=1.0)
Blas.sumOnMatrix(hidden,
out=self.vars["c"].grad,
alpha=-1.0,
beta=1.0)
return hidden
def updateGrad(self, grad): meansGrad = poolNdBackward(self.inData, self.means, grad, self.workspace, size=self.size, stride=1, pad=self.pad, mode=self.mode) Blas.addVectorToVector(grad.ravel(), meansGrad.ravel(), out=meansGrad.ravel(), beta=-1.0) self.grad = meansGrad
def updateGrad(self, grad):
self.grad = Blas.mulMatrixOnMatrix(grad,
self.W,
transpB=not self.transpose)
def updateData(self, data): self.means, self.workspace = poolNd(data, size=self.size, stride=1, pad=self.pad, mode=self.mode, test=not self.train) self.data = Blas.addVectorToVector(data.ravel(), self.means.ravel(), beta=-1.0).reshape(*data.shape)
def updateParams(self, learnRate):
for var in self.vars.values():
Blas.toVectorAddVector(var.data.ravel(),
var.grad.ravel(),
alpha=learnRate)
def calcError(self, pred, target):
self.devErr.fill(
Blas.dot(self.grad.ravel(), self.grad.ravel()) *
np.prod(self.grad.shape) * self.grad.shape[0] / 2.0)
self.accumErr += self.devErr
