def extractFeature(self,videodata):
n = videodata.size()
if n[0] <= 0:
return tor.zeros(1, self.NumberLabel)
N = math.floor(n[0] / self.clipM)
HP = tor.FloatTensor(N, self.d1, 1, self.scLstmDim)
HV = tor.FloatTensor(N, self.d1, 1, self.scLstmDim)
for i in range(N):
print("clip ", i, " start")
# Q = tor.FloatTensor(self.k, 1, self.scLstmDim)
for j in range(self.clipM):
o = self.linear(videodata[i * self.clipM + j])
# grnn
output = self.net(o)
## fc 2
##q k*scLstmDim
q = self.linear2(output)
## q->inn k*1*scLstmDim
## q- lastq = vector
innV = expendRank(1, q.add(- self.lastq))
innP = expendRank(1, q)
self.lastq = q
## skip-clip lstm for one frame
outtV, (self.lastHV, self.lastCV) = self.skLSTM_Vec(innV, self.lastHV, self.lastCV)
outtP, (self.lastHP, self.lastCP) = self.skLSTM_Pos(innP, self.lastHP, self.lastCP)
# print("frame ", i*clipM+j, " finished")
# get hmV,h
self.HmV = tor.add(self.lastHV, self.HmV)
self.HcV = tor.add(self.lastCV, self.HcV)
self.lastHV = self.HmV
self.lastCV = self.HcV
self.HmP = tor.add(self.lastHP, self.HmP)
self.HcP = tor.add(self.lastCP, self.HcP)
self.lastHP = self.HmP
self.lastCP = self.HcP
# Hs 8*1*8 d1*1*scLstmDim
## d3==scLstmDim hidden4==hiddenGRNN
# HP,HV N* self.d1* 1*self.scLstmDim
HP[i] = self.HmP
HV[i] = self.HmV
print("clip ", i, " finished")
return tor.add(HP, HV)
def updateState(self,r,m,lastS):
[k,b]=lastS.size()
newS=tor.zeros(k,b)
tr=expendRank(1,r)
tm=expendRank(1,m)
ts=expendRank(1,lastS)
for i in range(k):
pre=(Variable(tm[i]),Variable(ts[i]))
h_state, c_state =self.lstmcell(Variable(tr[i]), pre)
newS[i]=h_state
return newS
def train(self,train_path):
files = Path(train_path)
self.setLabels(files)
ls=0
ee=2
for label in files.glob('*'):
print("label name:", label.name)
if ee<0:
break
ee = ee - 1
for sample in label.glob('*'):
print("smaples name:", sample.name)
data=myIO.readOp3d(sample)
n = data.size()
N = math.floor(n[0] / self.clipM)
for i in range(N):
print("clip ", i, " start")
# Q = tor.FloatTensor(self.k, 1, self.b2)
for j in range(self.clipM):
o = self.linear(data[i * self.clipM + j])
# grnn
output = self.net(o)
## fc 2
##q 67* 8 --> k*b2
q = self.linear2(output)
del output
## q->inn k*1*b2
## q- lastq = vector
innV = expendRank(1, q - self.lastq)
innP = expendRank(1, q)
self.lastq = q
## skip-clip lstm for one frame
outtV, (self.lastHV, self.lastCV) = self.skLSTM_Vec(innV, self.lastHV, self.lastCV)
outtP, (self.lastHP, self.lastCP) = self.skLSTM_Pos(innP, self.lastHP, self.lastCP)
del outtP,outtV
# print("frame ", i*clipM+j, " finished")
#get hmV,h
self.HmV = tor.add(self.lastHV, self.HmV)
self.HcV = tor.add(self.lastCV, self.HcV)
self.lastHV = self.HmV
self.lastCV = self.HcV
self.HmP = tor.add(self.lastHP, self.HmP)
self.HcP = tor.add(self.lastCP, self.HcP)
self.lastHP = self.HmP
self.lastCP = self.HcP
# Hs 8*1*8 d1*1*b2
## d3==b2 b5==b3
# Om b2*1*numberlabel
Hs = tor.add(self.HmP, self.HmV)
Os = self.linear3(Hs)
# tmp numberlabel*1 *b2
tmp = Os.permute(2, 1, 0)
tmp1 = self.linear4(tmp)
# tmp1 numberlabel*1 *1
tmp1 = tmp1.permute(2, 1, 0)
# p pro numberlabel
probability = tmp1[0][0]
## feed sotfmax
probability = probability.softmax(0)
ytrue=self.initLabelMatrix(label)
ls=ls+(i/N)*tor.sum(ytrue.mul(probability.log()),-1)
del Hs,Os,tmp,tmp1
print("clip ", i, " finished")
print("current loss:",-ls)
del data
ls=-ls
print("final loss:",ls)
def fit(self,videodata,label):
n = videodata.size()
N = math.floor(n[0] / self.clipM)
Po=tor.FloatTensor(N,self.NumberLabel)
for i in range(N):
print("clip ", i, " start")
# Q = tor.FloatTensor(self.k, 1, self.b2)
for j in range(self.clipM):
o = self.linear(videodata[i * self.clipM + j])
# grnn
output = self.net(o)
## fc 2
##q 67* 8 --> k*b2
q = self.linear2(output)
del output
## q->inn k*1*b2
## q- lastq = vector
innV = expendRank(1, q - self.lastq)
innP = expendRank(1, q)
self.lastq = q
## skip-clip lstm for one frame
outtV, (self.lastHV, self.lastCV) = self.skLSTM_Vec(innV, self.lastHV, self.lastCV)
outtP, (self.lastHP, self.lastCP) = self.skLSTM_Pos(innP, self.lastHP, self.lastCP)
del outtP, outtV
# print("frame ", i*clipM+j, " finished")
# get hmV,h
self.HmV = tor.add(self.lastHV, self.HmV)
self.HcV = tor.add(self.lastCV, self.HcV)
self.lastHV = self.HmV
self.lastCV = self.HcV
self.HmP = tor.add(self.lastHP, self.HmP)
self.HcP = tor.add(self.lastCP, self.HcP)
self.lastHP = self.HmP
self.lastCP = self.HcP
# Hs 8*1*8 d1*1*b2
## d3==b2 b5==b3
# Om b2*1*numberlabel
Hs = tor.add(self.HmP, self.HmV)
Os = self.linear3(Hs)
# tmp numberlabel*1 *b2
tmp = Os.permute(2, 1, 0)
tmp1 = self.linear4(tmp)
# tmp1 numberlabel*1 *1
tmp1 = tmp1.permute(2, 1, 0)
# pro numberlabel
probability = tmp1[0][0]
## feed sotfmax
probability = probability.softmax(0)
#Po N*numberlabel
Po[i]=probability
del Hs, Os, tmp, tmp1
print("clip ", i, " finished")
ytrue=self.initLabelMatrix(label)
loss=self.criterion(probability,ytrue)
loss.backward()
return Po,loss
def bothPV():
path = "Data\\train\\tik\\"
data = myIO.readOp3d(path)
k = 67
b = 8
b2 = 8
b3 = 8
b4 = 16
b5 = b3
dim = 4
clipM = 10
d = 5
d1 = 8
d2 = 6
d3 = d1
NumberLabel = 5
weight = tor.ones(k, b)
bias = tor.zeros(k, b)
[n, kkkkkk, bbbbb] = data.size()
##net init
# fc 1
linear = nn.Linear(dim, b)
# grnn
net = grnn(weights=weight, bias=bias, epoch=5)
# fc2
linear2 = nn.Linear(b, b2)
# skip clip
skLSTM_Vec = tsl.skip_clipLSTM(1, b2, b3, d1, b4, d2, b5, d3)
skLSTM_Pos = tsl.skip_clipLSTM(1, b2, b3, d1, b4, d2, b5, d3)
# fc3 F01
linear3 = nn.Linear(b2, NumberLabel)
# fc 4 F02
linear4 = nn.Linear(b2, 1)
N = math.floor(n / clipM)
lastHV = tor.zeros(d1, 1, b2)
lastCV = tor.zeros(d1, 1, b2)
HmV = tor.zeros(d1, 1, b2)
HcV = tor.zeros(d1, 1, b2)
lastHP = tor.zeros(d1, 1, b2)
lastCP = tor.zeros(d1, 1, b2)
HmP = tor.zeros(d1, 1, b2)
HcP = tor.zeros(d1, 1, b2)
lastq = tor.zeros(k, b2)
for i in range(N):
print("clip ", i, " start")
Q = tor.FloatTensor(k, 1, b2)
for j in range(clipM):
o = linear(data[i * clipM + j])
# grnn
output = net(o)
## fc 2
##q 67* 8 --> k*b2
q = linear2(output)
## q->inn k*1*b2
## q- lastq = vector
innV = expendRank(1, q - lastq)
innP = expendRank(1, q)
lastq = q
## skip-clip lstm for one frame
outtV, (lastHV, lastCV) = skLSTM_Vec(innV, lastHV, lastCV)
outtP, (lastHP, lastCP) = skLSTM_Pos(innP, lastHP, lastCP)
# print("frame ", i*clipM+j, " finished")
HmV = tor.add(lastHV, HmV)
HcV = tor.add(lastCV, HcV)
lastHV = HmV
lastCV = HcV
HmP = tor.add(lastHP, HmP)
HcP = tor.add(lastCP, HcP)
lastHP = HmP
lastCP = HcP
# Hs 8*1*8 d1*1*b2
## d3==b2 b5==b3
# Om b2*1*numberlabel
Hs = tor.add(HmP, HmV)
Os = linear3(Hs)
# tmp numberlabel*1 *b2
tmp = Os.permute(2, 1, 0)
tmp1 = linear4(tmp)
# tmp1 numberlabel*1 *1
tmp1 = tmp1.permute(2, 1, 0)
# p pro numberlabel
probability = tmp1[0][0]
## feed sotfmax
probability = probability.softmax(0)
print("clip ", i, " finished")
print(probability)
def OnlyVec():
path = "Data\\train\\not\\"
# data = myIO.readOp3d(path)
k = 67
b = 8
b2 = 8
b3 = 8
b4 = 16
b5 = b3
b6 = 12
b7 = 16
dim = 4
clipM = 10
d = 5
d1 = 8
d2 = 6
d3 = d1
NumberLabel = 5
weight = tor.ones(k, b)
bias = tor.zeros(k, b)
#test data
data = tor.rand(11, 67, 4)
[n, kkkkkk, bbbbb] = data.size()
##net init
# fc 1
linear = nn.Linear(dim, b)
# grnn
net = grnn(weights=weight, bias=bias, epoch=5)
# fc2
linear2 = nn.Linear(b, b2)
# skip clip
skLSTM = tsl.skip_clipLSTM(1, b2, b3, d1, b4, d2, b5, d3)
# fc3 F01
linear3 = nn.Linear(b2, NumberLabel)
# fc 4 F02
linear4 = nn.Linear(b2, 1)
N = math.floor(n / clipM)
lastH = tor.zeros(d1, 1, b2)
lastC = tor.zeros(d1, 1, b2)
Hm = tor.zeros(d1, 1, b2)
Hc = tor.zeros(d1, 1, b2)
lastq=tor.zeros(k,b2)
for i in range(N):
print("clip ", i, " start")
Q = tor.FloatTensor(k, 1, b2)
for j in range(clipM):
o = linear(data[i * clipM + j])
# grnn
output = net(o)
## fc 2
##q 67* 8 --> k*b2
q = linear2(output)
## q->inn k*1*b2
## q- lastq = vector
inn = expendRank(1, q-lastq)
lastq = q
## skip-clip lstm for one frame
outt, (lastH, lastC) = skLSTM(inn, lastH, lastC)
# print("frame ", i*clipM+j, " finished")
Hm = tor.add(lastH, Hm)
Hc = tor.add(lastC, Hc)
lastH = Hm
lastC = Hc
# hm 8*1*8 d1*1*b2
## d3==b2 b5==b3
# Om b2*1*numberlabel
Om = linear3(Hm)
# tmp numberlabel*1 *b2
tmp = Om.permute(2, 1, 0)
tmp1 = linear4(tmp)
# tmp1 numberlabel*1 *1
tmp1 = tmp1.permute(2, 1, 0)
# p pro numberlabel
probability = tmp1[0][0]
probability = probability.softmax(0)
print("clip ", i, " finished")
print(probability)