class TemporalModel(Model):
def __init__(self, inputs, bs, max_time, classes, feature_dim, hidden_size, levels, N=1, pool=None, seed=12345):
self._inputs = inputs
self.N = N
self.batch_size = bs
self.classes = classes
self.max_time = max_time
self.levels = levels
self.feature_dim = feature_dim
self.pool = pool
self.dropout = True
# create a pyramid of filters
self.temporal_pyramid = []
for l in range(self.levels):
for f in range(2**l):
tf = TemporalAttentionLayer(batch_size=bs, N=N, channels=feature_dim,
name='temporal-attention-layer-'+str(l)+'-filter-'+str(f))
tf.test = True
tf.d = theano.shared(value=np.asarray([1./2**(l+1)]).astype('float32'), name='d', borrow=True,
broadcastable=[True])
tf.g = theano.shared(value=np.asarray([((1./2**l)+(2*f/2.**l))]).astype('float32'), name='g', borrow=True,
broadcastable=[True])
tf.sigma = theano.shared(value=np.asarray([5.0]).astype('float32'), name='sigma', borrow=True,
broadcastable=[True])
self.temporal_pyramid.append(tf)
input_size = feature_dim*N*(len(self.temporal_pyramid) if pool == None else 1)
self.hidden = HiddenLayer(input_size=input_size, hidden_size=hidden_size, activation=act.LeakyRelu(),
batch_size=bs, name='hidden', dropout=0.5)
self.softmax = SoftmaxLayer(input_size=hidden_size, classes=self.classes,
batch_size=bs, name='softmax', dropout=0.5)
@property
def params(self):
return self.softmax.params+self.hidden.params#+[p for f in self.temporal_filters for p in f.params]
@property
def inputs(self):
return self._inputs
@property
def outputs(self):
return self._outputs
@property
def updates(self):
return self._updates
@property
def test_algorithm(self):
if not hasattr(self, '_talgorithm'):
d = self.dropout
self.dropout = False
o = self.run(*self.inputs)
for i,ot in enumerate(self.outputs):
o[i].name = ot.name
self._talgorithm = theano.function(inputs=self.inputs,
outputs=o, on_unused_input='warn')
self.dropout = d
return self._talgorithm
def run(self, x, mask, y):
# use temporal filters
results = []
# make x to be batch x features x time
x = x.transpose([0,2,1])
for tf in self.temporal_pyramid:
# results is batch x features x N
# flatten to batch x features*N
res, (g,s2,d) = tf.run(x, mask)
if self.pool == None:
results.append(res.reshape((x.shape[0], self.feature_dim*self.N)))
else:
results.append(res.reshape((x.shape[0], 1, self.feature_dim*self.N)))
# concatenate on axis 1 to get batch x features*N*filters
x = T.concatenate(results, axis=1)
if self.pool == 'max':
x = T.max(x, axis=1)
elif self.pool == 'sum':
x = T.sum(x, axis=1)
elif self.pool == 'mean':
x = T.mean(x, axis=1)
x = self.hidden.run(x, self.dropout)
prob, pred = self.softmax.run(x, self.dropout)
loss = self.softmax.loss(prob, y)
error = self.softmax.error(pred, y)
acc = 1-error
return prob, pred, loss, error, acc
class TemporalModel(Model):
def __init__(self,
inputs,
bs,
max_time,
classes,
feature_dim,
hidden_size,
method='max',
seed=12345):
self._inputs = inputs
self.method = method
self.batch_size = bs
self.classes = classes
self.max_time = max_time
self.feature_dim = feature_dim
self.dropout = True
self.hidden = HiddenLayer(input_size=feature_dim,
hidden_size=hidden_size,
batch_size=bs,
name='hidden',
dropout=0.5,
activation=act.LeakyRelu())
self.softmax = SoftmaxLayer(input_size=hidden_size,
classes=self.classes,
batch_size=bs,
name='softmax',
dropout=0.5)
@property
def params(self):
return self.softmax.params + self.hidden.params
@property
def inputs(self):
return self._inputs
@property
def outputs(self):
return self._outputs
@property
def updates(self):
return self._updates
@property
def test_algorithm(self):
if not hasattr(self, '_talgorithm'):
d = self.dropout
self.dropout = False
o = self.run(*self.inputs)
for i, ot in enumerate(self.outputs):
o[i].name = ot.name
self._talgorithm = theano.function(inputs=self.inputs,
outputs=o,
on_unused_input='warn')
self.dropout = d
return self._talgorithm
def run(self, x, mask, y):
# get the max/mean/sum of x for each feature
# from all frame
if self.method == 'max':
m = (-100 * (1 - mask)).dimshuffle([0, 1, 'x'])
x = T.max(x + m, axis=1)
elif self.method == 'sum' or self.method == 'mean':
x = T.sum(x, axis=1)
elif self.method == 'mean':
x = x / T.sum(mask, axis=1).dimshuffle([0, 'x'])
x = x.astype(theano.config.floatX)
x = self.hidden.run(x, self.dropout)
prob, pred = self.softmax.run(x, self.dropout)
y = y.reshape((y.shape[0], ))
loss = self.softmax.loss(prob, y) + T.sum(
self.hidden.w**2) * 0.001 + T.sum(self.softmax.w**2) * 0.0001
y = T.extra_ops.to_one_hot(y, 51)
error = self.softmax.error(pred, y)
acc = 1 - error
return prob, pred, loss, error, acc
class TemporalModel(Model):
def __init__(self,
inputs,
bs,
max_time,
classes,
feature_dim,
hidden_size,
filters,
N=1,
pool=None,
lstm_dim=4096,
steps=8,
seed=12345):
self._inputs = inputs
self.N = N
self.batch_size = bs
self.classes = classes
self.max_time = max_time
self.filters = filters
self.feature_dim = feature_dim
self.pool = pool
self.dropout = True
self.steps = steps
self.temporal_filters = []
for f in range(filters):
tf = TemporalAttentionLayer(
batch_size=bs,
N=N,
channels=feature_dim,
input_hidden_size=lstm_dim,
name='temporal-attention-layer-filter-' + str(f))
self.temporal_filters.append(tf)
input_size = feature_dim * len(
self.temporal_filters) * (N if pool == None else 1)
self.lstm_in = HiddenLayer(input_size=input_size,
hidden_size=lstm_dim * 4,
batch_size=bs)
self.lstm = LSTMLayer(input_size=lstm_dim, hidden_size=lstm_dim)
self.hidden = HiddenLayer(input_size=lstm_dim,
hidden_size=hidden_size,
activation=act.relu,
batch_size=bs,
name='hidden',
dropout=0.5)
self.softmax = SoftmaxLayer(input_size=hidden_size,
classes=self.classes,
batch_size=bs,
name='softmax',
dropout=0.5)
@property
def params(self):
return self.softmax.params + self.hidden.params + self.lstm_in.params + self.lstm.params + [
p for f in self.temporal_filters for p in f.params
]
@property
def inputs(self):
return self._inputs
@property
def outputs(self):
return self._outputs
@property
def updates(self):
return self._updates
@property
def test_algorithm(self):
if not hasattr(self, '_talgorithm'):
d = self.dropout
self.dropout = False
o = self.run(*self.inputs)
for i, ot in enumerate(self.outputs):
o[i].name = ot.name
self._talgorithm = theano.function(inputs=self.inputs,
outputs=o,
on_unused_input='warn')
self.dropout = d
return self._talgorithm
def run(self, x, mask, y):
# use temporal filters
# make x to be batch x features x time
x = x.transpose([0, 2, 1])
h, c = self.lstm.get_initial_hidden(x)
outputs_info = [
dict(initial=h, taps=[-1]), # h
dict(initial=c, taps=[-1])
] # c
[h, c], _ = theano.scan(fn=self.step,
non_sequences=[x, mask],
outputs_info=outputs_info,
n_steps=self.steps)
x = self.hidden.run(h[-1], self.dropout)
prob, pred = self.softmax.run(x, self.dropout)
loss = self.softmax.loss(prob, y)
error = self.softmax.error(pred, y)
acc = 1 - error
return prob, pred, loss, error, acc
def step(self, h, c, x, mask):
results = []
for tf in self.temporal_filters:
# results is batch x features x N
# flatten to batch x features*N
res, (g, s2, d) = tf.run(x, h, mask)
if self.pool == None:
results.append(
res.reshape((x.shape[0], self.feature_dim * self.N)))
elif self.pool == 'max':
results.append(
T.max(res, axis=2).reshape((x.shape[0], self.feature_dim)))
elif self.pool == 'sum':
results.append(
T.sum(res, axis=2).reshape((x.shape[0], self.feature_dim)))
elif self.pool == 'mean':
results.append(
T.mean(res, axis=2).reshape(
(x.shape[0], self.feature_dim)))
# concatenate on axis 1 to get batch x features*N*filters
x = T.concatenate(results, axis=1)
x = self.lstm_in.run(x)
h, c = self.lstm.run(x, h, c)
return h, c
