def build_model(options,tparams):
"""Build up the whole computation graph
Input is the features extracted from googleNet.
"""
last_n = options['last_n'];
actionNum=options['actions'];
decay_c=options['decay_c'];
use_dropout=options['use_dropout'];
location_dim=options['locations'];
feature_dim=options['featureMaps'];
trng = RandomStreams(1234);
use_noise = theano.shared(np.float32(0.));
"""combine model"""
x=T.ftensor3('x');
n_steps=x.shape[0]; n_samples=x.shape[1];
mask = T.fmatrix('mask');
y=T.ftensor3('y'); # one hot vector,n_steps*n_samples*actionNum
_x=x.reshape([n_steps*n_samples,location_dim,feature_dim]);
feature,alpha=Saliency_Layer(tparams,_x,prefix="recog");
# feature=feature/feature.max(1,keepdims=True);
alpha=alpha.reshape([n_steps,n_samples,location_dim]);
feature=feature.reshape([n_steps,n_samples,feature_dim]);
if use_dropout: feature = dropout_layer(feature, use_noise, trng);
h,c=LSTM_layer(tparams,feature,prefix='recog',name="lstm");
f1=ff_build(tparams,h,prefix="recog",name='fullconn',active="tanh");
if use_dropout: f1 = dropout_layer(f1, use_noise, trng);
lin=ff_build(tparams,f1,prefix="recog",name='output',active="linear"); # n_steps*n_samples*actionNum
probs=T.nnet.softmax(lin.reshape([-1,actionNum]));
probs=probs.reshape([n_steps,n_samples,actionNum]);
"""compute cost"""
cost=0;
# cross entropy
entropy_cost=-y*T.log(probs);
entropy_cost = (entropy_cost.sum(2)*mask).mean(0).sum()*100;
cost+=entropy_cost;
# weight decay
weight_decay = 0.;
if decay_c > 0.:
decay_c = theano.shared(np.float32(decay_c), name='decay_c');
for kk, vv in tparams.iteritems():
weight_decay += (vv ** 2).sum()
weight_decay *= decay_c;
cost+=weight_decay;
"""Predictions"""
preds = T.sum(probs[-last_n:,:,:],axis=0);
preds = T.argmax(preds,axis=1); # n_samples
# preds=T.argmax(probs[-last_n:,:,:],axis=2);
return cost,preds,[h,c,alpha],[x,mask,y],use_noise;
def build_model(options,tparams):
"""Build up the whole computation graph
Input is the features extracted from googleNet.
"""
last_n = options['last_n'];
actionNum=options['actions'];
decay_c=options['decay_c'];
use_dropout=options['use_dropout'];
use_wta=options['use_wta'];
location_dim=options['locations'];
feature_dim=options['featureMaps'];
trng = RandomStreams(1234);
use_noise = theano.shared(np.float32(0.));
"""combine model"""
x=T.ftensor3('x');
n_steps=x.shape[0]; n_samples=x.shape[1];
mask = T.fmatrix('mask');
y=T.ftensor3('y'); # one hot vector,n_steps*n_samples*actionNum
_x=x.reshape([n_steps,n_samples,location_dim,feature_dim]);
# _x=_x/_x.max(-1,keepdims=True);
feature,alpha=SaliencyFgbg_block(tparams,_x,prefix="recog",name='saliencyFgbg');
# feature=_x.mean(2);
# feature=feature/feature.max(1,keepdims=True);
# feature=feature.reshape([n_steps,n_samples,feature_dim]);
# alpha=alpha.reshape([n_steps,n_samples,location_dim]);
feature=feature+use_noise*trng.normal(feature.shape,avg=0,std=0.0001,dtype=feature.dtype);
if use_dropout: feature = dropout_layer(feature, use_noise, trng);
if use_wta: feature=WTA_Layer(feature,4,2,ndim=options['featureMaps']);
h,c=LSTM_layer(tparams,feature,prefix='recog',name="lstm1");
if use_dropout: h = dropout_layer(h, use_noise, trng);
if use_wta: h=WTA_Layer(h,4,2,ndim=options['featureMaps']);
f0=ff_build(tparams,feature,prefix="recog",name='channel0',active="linear");
f1=ff_build(tparams,h,prefix="recog",name='channel1',active="linear");
f0=T.tanh((f0+f1)/2);
# f0=f0+use_noise*trng.normal(f0.shape,avg=0,std=0.01,dtype=f0.dtype);
if use_dropout: f0 = dropout_layer(f0, use_noise, trng);
lin=ff_build(tparams,f0,prefix="recog",name='output',active="linear"); # n_steps*n_samples*actionNum
# lin=T.switch(lin>5,5,lin);
# lin=T.switch(lin<-5,-5,lin);
probs=T.nnet.softmax(lin.reshape([-1,actionNum]));
probs=probs.reshape([n_steps,n_samples,actionNum]);
"""compute cost"""
cost=0;
# cross entropy
# _probs=T.switch(probs>0.9,1,probs);
# _probs=T.switch(probs>0.99,1,probs);
# _probs=T.switch(probs<0.1/actionNum,1,_probs);
entropy_cost=-y*T.log(probs+1e-8);
entropy_cost = (entropy_cost.sum(2)*mask).mean(0).sum()*100;
cost+=entropy_cost;
# direct cost
# direct_cost=((probs-y)**2).sum(2).mean(0).sum()*100;
# cost+=direct_cost;
# weight decay
weight_decay = 0.;
if decay_c > 0.:
decay_c = theano.shared(np.float32(decay_c), name='decay_c');
for kk, vv in tparams.iteritems():
weight_decay += (vv ** 2).sum()
weight_decay *= decay_c;
cost+=weight_decay;
# alpha cost
zero=theano.shared(np.float32(0.), name='zero');
alpha_cost=theano.ifelse.ifelse( T.gt(n_steps,1), ((alpha[1:,:,:]-alpha[:-1,:,:])**2).mean(0).sum()*100*0.1, zero);
alpha_s=T.sort(alpha);
alpha_cost+=((alpha_s[:,:,-9:].sum(-1)-1.0)**2).mean(0).sum()*100;
cost+=alpha_cost;
"""Predictions"""
preds = T.sum(probs[-last_n:,:,:],axis=0);
preds = T.argmax(preds,axis=1); # n_samples
# preds=T.argmax(probs[-last_n:,:,:],axis=2);
return cost,preds,[alpha],[x,mask,y],use_noise;
def build_model(options, tparams):
"""Build up the whole computation graph
Input is the features extracted from googleNet.
"""
last_n = options['last_n']
actionNum = options['actions']
decay_c = options['decay_c']
use_dropout = options['use_dropout']
use_wta = options['use_wta']
location_dim = options['locations']
feature_dim = options['featureMaps']
trng = RandomStreams(1234)
use_noise = theano.shared(np.float32(0.))
"""combine model"""
x = T.ftensor3('x')
n_steps = x.shape[0]
n_samples = x.shape[1]
mask = T.fmatrix('mask')
y = T.ftensor3('y')
# one hot vector,n_steps*n_samples*actionNum
_x = x.reshape([n_steps * n_samples, location_dim, feature_dim])
feature, alpha = Saliency_Layer(tparams,
_x,
prefix="recog",
name='saliency')
# feature=_x.mean(1);
# feature=feature/feature.max(1,keepdims=True);
feature = feature.reshape([n_steps, n_samples, feature_dim])
alpha = alpha.reshape([n_steps, n_samples, location_dim])
# feature=feature+use_noise*trng.normal(feature.shape,avg=0,std=0.01,dtype=feature.dtype);
if use_dropout: feature = dropout_layer(feature, use_noise, trng)
if use_wta: feature = WTA_Layer(feature, 4, 2, ndim=options['featureMaps'])
# fnn channel
f1 = ff_build(tparams,
feature,
prefix="recog",
name='fullconn',
active="linear")
# lstm channel
h, c = LSTM_layer(tparams, feature, prefix='recog', name="lstm")
if use_dropout: h = dropout_layer(h, use_noise, trng)
if use_wta: h = WTA_Layer(h, 4, 2, ndim=options['featureMaps'])
f2 = ff_build(tparams,
h,
prefix="recog",
name='fullconn_lstm',
active="linear")
# add two channels
f1 = T.tanh((f1 + f2) / 2)
if use_dropout: f1 = dropout_layer(f1, use_noise, trng)
lin = ff_build(tparams, f1, prefix="recog", name='output', active="linear")
# n_steps*n_samples*actionNum
probs = T.nnet.softmax(lin.reshape([-1, actionNum]))
probs = probs.reshape([n_steps, n_samples, actionNum])
"""compute cost"""
cost = 0
# cross entropy
entropy_cost = -y * T.log(probs)
entropy_cost = (entropy_cost.sum(2) * mask).mean(0).sum() * 100
cost += entropy_cost
# weight decay
weight_decay = 0.
if decay_c > 0.:
decay_c = theano.shared(np.float32(decay_c), name='decay_c')
for kk, vv in tparams.iteritems():
weight_decay += (vv**2).sum()
weight_decay *= decay_c
cost += weight_decay
# alpha cost
# zero=theano.shared(np.float32(0.), name='zero');
# alpha_cost=theano.ifelse.ifelse( T.gt(n_steps,1), ((alpha[1:,:,:]-alpha[:-1,:,:])**2).mean(0).sum()*100*0.1, zero);
# alpha_s=T.sort(alpha);
# alpha_cost+=((alpha_s[:,:,-9:].sum(-1)-1.0)**2).mean(0).sum()*100;
# cost+=alpha_cost;
"""Predictions"""
preds = T.sum(probs[-last_n:, :, :], axis=0)
preds = T.argmax(preds, axis=1)
# n_samples
# preds=T.argmax(probs[-last_n:,:,:],axis=2);
return cost, preds, [alpha, h, c], [x, mask, y], use_noise
def build_model(options,tparams):
"""Build up the whole computation graph
Input is the features extracted from googleNet.
"""
last_n = options['last_n'];
actionNum=options['actions'];
decay_c=options['decay_c'];
use_dropout=options['use_dropout'];
use_wta=options['use_wta'];
location_dim=options['locations'];
feature_dim=options['featureMaps'];
trng = RandomStreams(1234);
use_noise = theano.shared(np.float32(0.));
"""combine model"""
x=T.ftensor3('x');
n_steps=x.shape[0]; n_samples=x.shape[1];
mask = T.fmatrix('mask');
y=T.ftensor3('y'); # one hot vector,n_steps*n_samples*actionNum
_x=x.reshape([n_steps*n_samples,location_dim,feature_dim]);
feature=_x.mean(1);
# feature=feature/feature.max(1,keepdims=True);
feature=feature.reshape([n_steps,n_samples,feature_dim]);
feature=feature+use_noise*trng.normal(feature.shape,avg=1,std=0.05,dtype=feature.dtype); #noisy
if use_dropout: feature = dropout_layer(feature, use_noise, trng);
if use_wta: feature=WTA_Layer(feature,4,2,ndim=options['featureMaps']);
f1=ff_build(tparams,feature,prefix="recog",name='fullconn',active="tanh");
if use_dropout: f1 = dropout_layer(f1, use_noise, trng);
lin=ff_build(tparams,f1,prefix="recog",name='output',active="linear"); # n_steps*n_samples*actionNum
probs=T.nnet.softmax(lin.reshape([-1,actionNum]));
probs=probs.reshape([n_steps,n_samples,actionNum]);
"""compute cost"""
cost=0;
# cross entropy
entropy_cost=-y*T.log(probs+1e-8);
entropy_cost = (entropy_cost.sum(2)*mask).mean(0).sum()*100;
cost+=entropy_cost;
# weight decay
weight_decay = 0.;
if decay_c > 0.:
decay_c = theano.shared(np.float32(decay_c), name='decay_c');
for kk, vv in tparams.iteritems():
weight_decay += (vv ** 2).sum()
weight_decay *= decay_c;
cost+=weight_decay;
"""Predictions"""
preds = T.sum(probs[-last_n:,:,:],axis=0);
preds = T.argmax(preds,axis=1); # n_samples
# preds=T.argmax(probs[-last_n:,:,:],axis=2);
return cost,preds,[],[x,mask,y],use_noise;
def build_model(options,tparams):
"""Build up the whole computation graph
Input is the features extracted from googleNet.
"""
last_n = options['last_n'];
actionNum=options['actions'];
decay_c=options['decay_c'];
use_dropout=options['use_dropout'];
use_wta=options['use_wta'];
featureMaps=options['featureMaps'];
videosize=options['videosize'];
trng = RandomStreams(1234);
use_noise = theano.shared(np.float32(0.));
"""combine model"""
x=T.ftensor3('x');
n_steps=x.shape[0]; n_samples=x.shape[1];
mask = T.fmatrix('mask');
y=T.ftensor3('y'); # one hot vector,n_steps*n_samples*actionNum
_x=x.reshape([n_steps*n_samples,3,videosize,videosize]);
_x=_x+use_noise*trng.normal(_x.shape,avg=0,std=0.1,dtype=_x.dtype);
feature=CNN_block(_x);
feature=feature.reshape([n_steps,n_samples,featureMaps]);
if use_dropout: feature = dropout_layer(feature, use_noise, trng);
f0=ff_build(tparams,feature,prefix="recog",name='fc0',active="tanh");
f1=ff_build(tparams,f0,prefix="recog",name='fc1',active="tanh");
# if use_dropout: f0 = dropout_layer(f0, use_noise, trng);
lin=ff_build(tparams,f1,prefix="recog",name='output',active="linear"); # n_steps*n_samples*actionNum
probs=T.nnet.softmax(lin.reshape([-1,actionNum]));
probs=probs.reshape([n_steps,n_samples,actionNum]);
"""compute cost"""
cost=0;
# cross entropy
entropy_cost=-y*T.log(probs);
entropy_cost = (entropy_cost.sum(2)*mask).mean(0).sum();
cost+=entropy_cost;
# weight decay
weight_decay = 0.;
if decay_c > 0.:
decay_c = theano.shared(np.float32(decay_c), name='decay_c');
for vv in itemlist(tparams):
# if (vv.ndim==1):
weight_decay += (vv ** 2).sum()
# else:
# weight_decay+=(vv.sum()-vv.shape[0])**2;
# weight_decay += (vv ** 2).sum()
weight_decay *= decay_c;
cost+=weight_decay;
"""Predictions"""
preds = T.sum(probs[-last_n:,:,:],axis=0);
preds = T.argmax(preds,axis=1); # n_samples
# preds=T.argmax(probs[-last_n:,:,:],axis=2);
return cost,preds,[],[x,mask,y],use_noise;
def build_model(options, tparams):
"""Build up the whole computation graph
Input is the features extracted from googleNet.
"""
last_n = options['last_n']
actionNum = options['actions']
decay_c = options['decay_c']
use_dropout = options['use_dropout']
use_wta = options['use_wta']
location_dim = options['locations']
feature_dim = options['featureMaps']
trng = RandomStreams(1234)
use_noise = theano.shared(np.float32(0.))
"""combine model"""
x = T.ftensor3('x')
n_steps = x.shape[0]
n_samples = x.shape[1]
mask = T.fmatrix('mask')
y = T.ftensor3('y')
# one hot vector,n_steps*n_samples*actionNum
_x = x.reshape([n_steps, n_samples, location_dim, feature_dim])
feature = _x.mean(1)
feature = feature + use_noise * trng.normal(
feature.shape, avg=1, std=0.01, dtype=feature.dtype)
if use_dropout: feature = dropout_layer(feature, use_noise, trng)
h, c = LSTM_mcore_layer(tparams,
feature,
cores=options['cores'],
prefix='recog',
name="lstm")
if use_dropout: h = dropout_layer(h, use_noise, trng)
f0 = ff_build(tparams, h, prefix="recog", name='fullconn', active="tanh")
# f0=f0+use_noise*trng.normal(f0.shape,avg=0,std=0.01,dtype=f0.dtype);
if use_dropout: f0 = dropout_layer(f0, use_noise, trng)
lin = ff_build(tparams, f0, prefix="recog", name='output', active="linear")
# n_steps*n_samples*actionNum
probs = T.nnet.softmax(lin.reshape([-1, actionNum]))
probs = probs.reshape([n_steps, n_samples, actionNum])
"""Predictions"""
preds = T.sum(probs[-last_n:, :, :], axis=0)
preds = T.argmax(preds, axis=1)
# n_samples
# preds=T.argmax(probs[-last_n:,:,:],axis=2);
"""compute cost"""
cost = 0
# cross entropy
entropy_cost = -y * T.log(probs + 1e-8)
entropy_cost = (entropy_cost.sum(2) * mask).mean(0).sum() * 100
cost += entropy_cost
# weight decay
weight_decay = 0.
if decay_c > 0.:
decay_c = theano.shared(np.float32(decay_c), name='decay_c')
for kk, vv in tparams.iteritems():
weight_decay += (vv**2).sum()
weight_decay *= decay_c
cost += weight_decay
return cost, preds, [h, c], [x, mask, y], use_noise
def build_model(options, tparams):
"""Build up the whole computation graph
Input is the features extracted from googleNet.
"""
last_n = options['last_n']
actionNum = options['actions']
decay_c = options['decay_c']
use_dropout = options['use_dropout']
use_wta = options['use_wta']
location_dim = 4 * 4
feature_dim = 256
trng = RandomStreams(1234)
use_noise = theano.shared(np.float32(0.))
"""combine model"""
x = T.ftensor3('x')
n_steps = x.shape[0]
n_samples = x.shape[1]
mask = T.fmatrix('mask')
y = T.ftensor3('y')
# one hot vector,n_steps*n_samples*actionNum
_x = x.reshape([n_steps * n_samples, 3, 64, 64])
feature = CNN_block(_x)
# n_steps*featureMap*4*4
feature = feature.reshape([n_steps, n_samples, feature_dim, location_dim])
feature = feature.mean(-1)
feature = feature.reshape([n_steps, n_samples, feature_dim])
# feature=feature+use_noise*trng.normal(feature.shape,avg=0,std=0.01,dtype=feature.dtype);
# if use_dropout: feature = dropout_layer(feature, use_noise, trng);
# feature=Linger_layer(tparams,feature,prefix='recog',name='linger');
#LSTM
# h1,c1=LSTM_layer(tparams,feature,prefix='recog',name="lstm1");
# if use_dropout: h1 = dropout_layer(h1, use_noise, trng);
# h2,c2=LSTM_layer(tparams,h1,prefix='recog',name="lstm2");
# if use_dropout: h2 = dropout_layer(h2, use_noise, trng);
# #
# h3,c3=LSTM_layer(tparams,h2,prefix='recog',name="lstm3");
# if use_dropout: h3 = dropout_layer(h3, use_noise, trng);
# h4,c4=LSTM_layer(tparams,h3,prefix='recog',name="lstm4");
# if use_dropout: h4 = dropout_layer(h4, use_noise, trng);
f0 = ff_build(tparams,
feature,
prefix="recog",
name='channel0',
active="tanh")
# f1=ff_build(tparams,h1,prefix="recog",name='channel1',active="linear");
# f2=ff_build(tparams,h2,prefix="recog",name='channel2',active="linear");
# f3=ff_build(tparams,h3,prefix="recog",name='channel3',active="linear");
# f4=ff_build(tparams,h4,prefix="recog",name='channel4',active="linear");
# add two channels
# f0=T.tanh(f1);
# if use_dropout: f0 = dropout_layer(f0, use_noise, trng);
lin = ff_build(tparams, f0, prefix="recog", name='output', active="linear")
# n_steps*n_samples*actionNum
probs = T.nnet.softmax(lin.reshape([-1, actionNum]))
probs = probs.reshape([n_steps, n_samples, actionNum])
"""compute cost"""
cost = 0
# cross entropy
entropy_cost = -y * T.log(probs)
entropy_cost = (entropy_cost.sum(2) * mask).mean(0).sum() * 100
cost += entropy_cost
# weight decay
weight_decay = 0.
if decay_c > 0.:
decay_c = theano.shared(np.float32(decay_c), name='decay_c')
for kk, vv in tparams.iteritems():
# if (vv.ndim==1):
weight_decay += (vv**2).sum()
# else:
# weight_decay+=(vv.sum()-vv.shape[0])**2;
# weight_decay += (vv ** 2).sum()
weight_decay *= decay_c
cost += weight_decay
"""Predictions"""
preds = T.sum(probs[-last_n:, :, :], axis=0)
preds = T.argmax(preds, axis=1)
# n_samples
# preds=T.argmax(probs[-last_n:,:,:],axis=2);
return cost, preds, [], [x, mask, y], use_noise
def build_model(options,tparams):
"""Build up the whole computation graph
Input is the features extracted from googleNet.
"""
last_n = options['last_n'];
actionNum=options['actions'];
decay_c=options['decay_c'];
use_dropout=options['use_dropout'];
use_wta=options['use_wta'];
location_dim=options['locations'];
feature_dim=options['featureMaps'];
trng = RandomStreams(1234);
use_noise = theano.shared(np.float32(0.));
"""combine model"""
x=T.ftensor3('x');
n_steps=x.shape[0]; n_samples=x.shape[1];
mask = T.fmatrix('mask');
y=T.ftensor3('y'); # one hot vector,n_steps*n_samples*actionNum
_x=x.reshape([n_steps,n_samples,location_dim,feature_dim]);
feature,alpha=SaliencyLSTM_block(tparams,_x,prefix="recog",name='saliencyLSTM');
# feature=feature+use_noise*trng.normal(feature.shape,avg=0,std=0.01,dtype=feature.dtype);
if use_dropout: feature = dropout_layer(feature, use_noise, trng);
h,c=LSTM_layer(tparams,feature,prefix='recog',name="lstm");
if use_dropout: h = dropout_layer(h, use_noise, trng);
f0=ff_build(tparams,feature,prefix="recog",name='channel0',active="linear");
f1=ff_build(tparams,h,prefix="recog",name='channel1',active="linear");
f0=T.tanh((f0+f1)/2); # merge channel
# f0=f0+use_noise*trng.normal(f0.shape,avg=0,std=0.01,dtype=f0.dtype);
if use_dropout: f0 = dropout_layer(f0, use_noise, trng);
lin=ff_build(tparams,f0,prefix="recog",name='output',active="linear"); # n_steps*n_samples*actionNum
probs=T.nnet.softmax(lin.reshape([-1,actionNum]));
probs=probs.reshape([n_steps,n_samples,actionNum]);
"""compute cost"""
cost=0;
# cross entropy
# _probs=T.switch(probs>0.9,1,probs);
# _probs=T.switch(probs>0.99,1,probs);
# _probs=T.switch(probs<0.1/actionNum,1,_probs);
entropy_cost=-y*T.log(probs+1e-8); # avoid explosion
entropy_cost = (entropy_cost.sum(2)*mask).mean(0).sum()*100;
cost+=entropy_cost;
# weight decay
weight_decay = 0.;
if decay_c > 0.:
decay_c = theano.shared(np.float32(decay_c), name='decay_c');
for kk, vv in tparams.iteritems():
weight_decay += (vv ** 2).sum()
weight_decay *= decay_c;
cost+=weight_decay;
# alpha cost
zero=theano.shared(np.float32(0.), name='zero');
alpha_cost=theano.ifelse.ifelse( T.gt(n_steps,1), ((alpha[1:,:,:]-alpha[:-1,:,:])**2).mean(0).sum()*100*0.1, zero);
alpha_s=T.sort(alpha);
alpha_cost+=((alpha_s[:,:,-9:].sum(-1)-1.0)**2).mean(0).sum()*100;
cost+=alpha_cost;
"""Predictions"""
preds = T.sum(probs[-last_n:,:,:],axis=0);
preds = T.argmax(preds,axis=1); # n_samples
# preds=T.argmax(probs[-last_n:,:,:],axis=2);
return cost,preds,[alpha,h,c],[x,mask,y],use_noise;
def build_model(options,tparams):
"""Build up the whole computation graph
Input is the features extracted from googleNet.
"""
last_n = options['last_n'];
actionNum=options['actions'];
decay_c=options['decay_c'];
use_dropout=options['use_dropout'];
use_wta=options['use_wta'];
location_dim=4*4;
feature_dim=256;
trng = RandomStreams(1234);
use_noise = theano.shared(np.float32(0.));
"""combine model"""
x=T.ftensor3('x');
n_steps=x.shape[0]; n_samples=x.shape[1];
mask = T.fmatrix('mask');
y=T.ftensor3('y'); # one hot vector,n_steps*n_samples*actionNum
_x=x.reshape([n_steps*n_samples,3,64,64]);
feature=CNN_block(_x); # n_steps*featureMap*4*4
feature=feature.reshape([n_steps, n_samples, feature_dim, location_dim]);
feature=feature.mean(-1);
feature=feature.reshape([n_steps,n_samples,feature_dim]);
# feature=feature+use_noise*trng.normal(feature.shape,avg=0,std=0.01,dtype=feature.dtype);
# if use_dropout: feature = dropout_layer(feature, use_noise, trng);
# feature=Linger_layer(tparams,feature,prefix='recog',name='linger');
#LSTM
# h1,c1=LSTM_layer(tparams,feature,prefix='recog',name="lstm1");
# if use_dropout: h1 = dropout_layer(h1, use_noise, trng);
# h2,c2=LSTM_layer(tparams,h1,prefix='recog',name="lstm2");
# if use_dropout: h2 = dropout_layer(h2, use_noise, trng);
# #
# h3,c3=LSTM_layer(tparams,h2,prefix='recog',name="lstm3");
# if use_dropout: h3 = dropout_layer(h3, use_noise, trng);
# h4,c4=LSTM_layer(tparams,h3,prefix='recog',name="lstm4");
# if use_dropout: h4 = dropout_layer(h4, use_noise, trng);
f0=ff_build(tparams,feature,prefix="recog",name='channel0',active="tanh");
# f1=ff_build(tparams,h1,prefix="recog",name='channel1',active="linear");
# f2=ff_build(tparams,h2,prefix="recog",name='channel2',active="linear");
# f3=ff_build(tparams,h3,prefix="recog",name='channel3',active="linear");
# f4=ff_build(tparams,h4,prefix="recog",name='channel4',active="linear");
# add two channels
# f0=T.tanh(f1);
# if use_dropout: f0 = dropout_layer(f0, use_noise, trng);
lin=ff_build(tparams,f0,prefix="recog",name='output',active="linear"); # n_steps*n_samples*actionNum
probs=T.nnet.softmax(lin.reshape([-1,actionNum]));
probs=probs.reshape([n_steps,n_samples,actionNum]);
"""compute cost"""
cost=0;
# cross entropy
entropy_cost=-y*T.log(probs);
entropy_cost = (entropy_cost.sum(2)*mask).mean(0).sum()*100;
cost+=entropy_cost;
# weight decay
weight_decay = 0.;
if decay_c > 0.:
decay_c = theano.shared(np.float32(decay_c), name='decay_c');
for kk, vv in tparams.iteritems():
# if (vv.ndim==1):
weight_decay += (vv ** 2).sum()
# else:
# weight_decay+=(vv.sum()-vv.shape[0])**2;
# weight_decay += (vv ** 2).sum()
weight_decay *= decay_c;
cost+=weight_decay;
"""Predictions"""
preds = T.sum(probs[-last_n:,:,:],axis=0);
preds = T.argmax(preds,axis=1); # n_samples
# preds=T.argmax(probs[-last_n:,:,:],axis=2);
return cost,preds,[],[x,mask,y],use_noise;
def build_model(options, tparams):
"""Build up the whole computation graph
Input is the features extracted from googleNet.
"""
last_n = options['last_n']
actionNum = options['actions']
decay_c = options['decay_c']
use_dropout = options['use_dropout']
location_dim = options['locations']
feature_dim = options['featureMaps']
trng = RandomStreams(1234)
use_noise = theano.shared(np.float32(0.))
"""combine model"""
x = T.ftensor3('x')
n_steps = x.shape[0]
n_samples = x.shape[1]
mask = T.fmatrix('mask')
y = T.ftensor3('y')
# one hot vector,n_steps*n_samples*actionNum
_x = x.reshape([n_steps * n_samples, location_dim, feature_dim])
_x = _x + trng.normal(_x.shape, avg=0, std=0.05, dtype=_x.dtype)
#noisy
# feature,alpha=Saliency_Layer(tparams,_x,prefix="recog");
#alpha=alpha.reshape([n_steps,n_samples,location_dim]);
feature = _x.mean(1)
# feature=feature/feature.max(1,keepdims=True);
feature = feature.reshape([n_steps, n_samples, feature_dim])
ctx = Linger_layer(tparams, feature, prefix='recog', name='linger')
if use_dropout: ctx = dropout_layer(ctx, use_noise, trng)
h, c = LSTM_layer(tparams, ctx, prefix='recog', name="lstm")
if use_dropout: h = dropout_layer(h, use_noise, trng)
f1 = ff_build(tparams, h, prefix="recog", name='fullconn', active="linear")
f2 = ff_build(tparams,
ctx,
prefix="recog",
name='highway',
active="linear")
# add two channels
f1 = T.tanh((f1 + f2) / 2)
if use_dropout: f1 = dropout_layer(f1, use_noise, trng)
lin = ff_build(tparams, f1, prefix="recog", name='output', active="linear")
# n_steps*n_samples*actionNum
probs = T.nnet.softmax(lin.reshape([-1, actionNum]))
probs = probs.reshape([n_steps, n_samples, actionNum])
"""compute cost"""
cost = 0
# cross entropy
entropy_cost = -y * T.log(probs + 1e-8)
entropy_cost = (entropy_cost.sum(2) * mask).mean(0).sum() * 100
cost += entropy_cost
# weight decay
weight_decay = 0.
if decay_c > 0.:
decay_c = theano.shared(np.float32(decay_c), name='decay_c')
for kk, vv in tparams.iteritems():
weight_decay += (vv**2).sum()
weight_decay *= decay_c
cost += weight_decay
"""Predictions"""
preds = T.sum(probs[-last_n:, :, :], axis=0)
preds = T.argmax(preds, axis=1)
# n_samples
# preds=T.argmax(probs[-last_n:,:,:],axis=2);
return cost, preds, [h, c], [x, mask, y], use_noise
