input_dim=None, output_dim=None, timesteps=None, activation='linear'):

'''Apply y.w + b for every temporal slice y of x.

'''

activation = activations.get(activation)

if not input_dim:

# won't work with TensorFlow

input_dim = K.shape(x)[2]

if not timesteps:

# won't work with TensorFlow

timesteps = K.shape(x)[1]

if not output_dim:

# won't work with TensorFlow

output_dim = K.shape(w)[1]

if dropout is not None and 0. < dropout < 1.:

# apply the same dropout pattern at every timestep

ones = K.ones_like(K.reshape(x[:, 0, :], (-1, input_dim)))

dropout_matrix = K.dropout(ones, dropout)

expanded_dropout_matrix = K.repeat(dropout_matrix, timesteps)

x = K.in_train_phase(x * expanded_dropout_matrix, x)

# collapse time dimension and batch dimension together

x = K.reshape(x, (-1, input_dim))

x = K.dot(x, w)

if b:

x = x + b

# reshape to 3D tensor

x = K.reshape(activation(x), (-1, timesteps, output_dim))

def __init__(self, weights=None,

return_sequences=False, go_backwards=False, stateful=False,

unroll=False, consume_less='cpu',

input_dim=None, input_length=None, **kwargs):

self.return_sequences = return_sequences

self.initial_weights = weights

self.go_backwards = go_backwards

self.stateful = stateful

self.unroll = unroll

self.consume_less = consume_less

self.supports_masking = True

self.input_spec = [InputSpec(ndim=4)]

self.input_dim = input_dim

self.input_length = input_length

if self.input_dim:

kwargs['input_shape'] = (self.input_length, self.input_dim)

super(ARecurrent, self).__init__(**kwargs)

def get_output_shape_for(self, input_shape):

if self.return_sequences:

return (input_shape[0], input_shape[1], self.output_dim)

else:

return (input_shape[0], self.output_dim)

def compute_mask(self, input, mask):

if self.return_sequences:

return mask

else:

return None

def step(self, x, states):

raise NotImplementedError

def get_constants(self, x):

return []

def get_initial_states(self, x):

# build an all-zero tensor of shape (samples, output_dim)

initial_state = K.zeros_like(x[:,:,0,:]) # (samples, timesteps, prev_timesteps, input_dim)

initial_state = K.sum(initial_state, axis=1) # (samples, prev_timesteps, input_dim)

reducer = K.zeros((self.input_dim, self.output_dim))

initial_state = K.dot(initial_state, reducer) # (samples, output_dim)

initial_states = [initial_state for _ in range(len(self.states))]

return initial_states

def preprocess_input(self, x):

return x

def call(self, x, mask=None):

# input shape: (nb_samples, time (padded with zeros), input_dim)

# note that the .build() method of subclasses MUST define

# self.input_spec with a complete input shape.

input_shape = self.input_spec[0].shape

if K._BACKEND == 'tensorflow':

if not input_shape[1]:

raise Exception('When using TensorFlow, you should define '

'explicitly the number of timesteps of '

'your sequences.\n'

'If your first layer is an Embedding, '

'make sure to pass it an "input_length" '

'argument. Otherwise, make sure '

'the first layer has '

'an "input_shape" or "batch_input_shape" '

'argument, including the time axis. '

'Found input shape at layer ' + self.name +

': ' + str(input_shape))

if self.stateful:

initial_states = self.states

else:

initial_states = self.get_initial_states(x)

constants = self.get_constants(x)

preprocessed_input = self.preprocess_input(x)

last_output, outputs, states = K.rnn(self.step, preprocessed_input,

initial_states,

go_backwards=self.go_backwards,

mask=mask,

constants=constants,

unroll=self.unroll,

input_length=input_shape[1])

if self.stateful:

self.updates = []

for i in range(len(states)):

self.updates.append((self.states[i], states[i]))

if self.return_sequences:

return outputs

else:

return last_output

def get_config(self):

config = {'return_sequences': self.return_sequences,

'go_backwards': self.go_backwards,

'stateful': self.stateful,

'unroll': self.unroll,

'consume_less': self.consume_less}

if self.stateful:

config['batch_input_shape'] = self.input_spec[0].shape

else:

config['input_dim'] = self.input_dim

config['input_length'] = self.input_length

base_config = super(ARecurrent, self).get_config()

def __init__(self, output_dim,

init='glorot_uniform', inner_init='orthogonal',

forget_bias_init='one', activation='tanh',

inner_activation='hard_sigmoid',

W_regularizer=None, U_regularizer=None, b_regularizer=None,

dropout_W=0., dropout_U=0., **kwargs):

self.output_dim = output_dim

self.init = initializations.get(init)

self.inner_init = initializations.get(inner_init)

self.forget_bias_init = initializations.get(forget_bias_init)

self.activation = activations.get(activation)

self.inner_activation = activations.get(inner_activation)

self.W_regularizer = regularizers.get(W_regularizer)

self.U_regularizer = regularizers.get(U_regularizer)

self.b_regularizer = regularizers.get(b_regularizer)

self.dropout_W, self.dropout_U = dropout_W, dropout_U

if self.dropout_W or self.dropout_U:

self.uses_learning_phase = True

super(ALSTM, self).__init__(**kwargs)

def build(self, input_shape):

#assert self.output_dim == input_shape[-1]

self.input_spec = [InputSpec(shape=input_shape)]

self.middle_length = input_shape[2]

input_dim = input_shape[3]

# Attention

self.W_a = self.init((input_dim + self.output_dim, self.output_dim),

name='{}_W_a'.format(self.name))

self.b_a = K.zeros((self.output_dim,), name='{}_b_a'.format(self.name))

# Regular LSTM

self.input_dim = input_dim

if self.stateful:

self.reset_states()

else:

# initial states: 2 all-zero tensors of shape (output_dim)

self.states = [None, None]

self.W_i = self.init((input_dim, self.output_dim),

name='{}_W_i'.format(self.name))

self.U_i = self.inner_init((self.output_dim, self.output_dim),

name='{}_U_i'.format(self.name))

self.b_i = K.zeros((self.output_dim,), name='{}_b_i'.format(self.name))

self.W_f = self.init((input_dim, self.output_dim),

name='{}_W_f'.format(self.name))

self.U_f = self.inner_init((self.output_dim, self.output_dim),

name='{}_U_f'.format(self.name))

self.b_f = self.forget_bias_init((self.output_dim,),

name='{}_b_f'.format(self.name))

self.W_c = self.init((input_dim, self.output_dim),

name='{}_W_c'.format(self.name))

self.U_c = self.inner_init((self.output_dim, self.output_dim),

name='{}_U_c'.format(self.name))

self.b_c = K.zeros((self.output_dim,), name='{}_b_c'.format(self.name))

self.W_o = self.init((input_dim, self.output_dim),

name='{}_W_o'.format(self.name))

self.U_o = self.inner_init((self.output_dim, self.output_dim),

name='{}_U_o'.format(self.name))

self.b_o = K.zeros((self.output_dim,), name='{}_b_o'.format(self.name))

self.regularizers = []

if self.W_regularizer:

self.W_regularizer.set_param(K.concatenate([self.W_i,

self.W_f,

self.W_c,

self.W_o]))

self.regularizers.append(self.W_regularizer)

if self.U_regularizer:

self.U_regularizer.set_param(K.concatenate([self.U_i,

self.U_f,

self.U_c,

self.U_o]))

self.regularizers.append(self.U_regularizer)

if self.b_regularizer:

self.b_regularizer.set_param(K.concatenate([self.b_i,

self.b_f,

self.b_c,

self.b_o]))

self.regularizers.append(self.b_regularizer)

self.trainable_weights = [self.W_i, self.U_i, self.b_i,

self.W_c, self.U_c, self.b_c,

self.W_f, self.U_f, self.b_f,

self.W_o, self.U_o, self.b_o,

self.W_a, self.b_a]

if self.initial_weights is not None:

self.set_weights(self.initial_weights)

del self.initial_weights

def reset_states(self):

assert self.stateful, 'Layer must be stateful.'

input_shape = self.input_spec[0].shape

if not input_shape[0]:

raise Exception('If a RNN is stateful, a complete ' +

'input_shape must be provided (including batch size).')

if hasattr(self, 'states'):

K.set_value(self.states[0],

np.zeros((input_shape[0], self.output_dim)))

K.set_value(self.states[1],

np.zeros((input_shape[0], self.output_dim)))

else:

self.states = [K.zeros((input_shape[0], self.output_dim)),

K.zeros((input_shape[0], self.output_dim))]

def preprocess_input(self, x):

return x

def step(self, x, states):

# ALTM

h_tm1 = states[0]

exp_h_tm1 = h_tm1.dimshuffle((0, 'x', 1))

exp_h_tm1 = T.extra_ops.repeat(exp_h_tm1, x.shape[1], axis=1)

con = K.concatenate((x,exp_h_tm1), axis=-1)

tdense = time_distributed_dense(con, self.W_a, self.b_a, None, self.input_dim+self.output_dim, self.output_dim, x.shape[1])

d_sum = K.sum(tdense, axis=-1)

sm = K.softmax(d_sum)

sm = sm.dimshuffle((0, 1, 'x'))

sm = T.extra_ops.repeat(sm, self.input_dim, axis=-1)

new_x = sm * x

new_x = K.sum(new_x, axis=1)

# LSTM

#h_tm1 = states[0]

c_tm1 = states[1]

B_U = states[2]

B_W = states[3]

x_i = K.dot(new_x * B_W[0], self.W_i) + self.b_i

x_f = K.dot(new_x * B_W[1], self.W_f) + self.b_f

x_c = K.dot(new_x * B_W[2], self.W_c) + self.b_c

x_o = K.dot(new_x * B_W[3], self.W_o) + self.b_o

i = self.inner_activation(x_i + K.dot(h_tm1 * B_U[0], self.U_i))

f = self.inner_activation(x_f + K.dot(h_tm1 * B_U[1], self.U_f))

c = f * c_tm1 + i * self.activation(x_c + K.dot(h_tm1 * B_U[2], self.U_c))

o = self.inner_activation(x_o + K.dot(h_tm1 * B_U[3], self.U_o))

h = o * self.activation(c)

return h, [h, c]

def get_constants(self, x):

constants = []

constants.append([K.cast_to_floatx(1.) for _ in range(4)])

constants.append([K.cast_to_floatx(1.) for _ in range(4)])

return constants

def get_config(self):

config = {"output_dim": self.output_dim,

"init": self.init.__name__,

"inner_init": self.inner_init.__name__,

"forget_bias_init": self.forget_bias_init.__name__,

"activation": self.activation.__name__,

"inner_activation": self.inner_activation.__name__,

"W_regularizer": self.W_regularizer.get_config() if self.W_regularizer else None,

"U_regularizer": self.U_regularizer.get_config() if self.U_regularizer else None,

"b_regularizer": self.b_regularizer.get_config() if self.b_regularizer else None,

"dropout_W": self.dropout_W,

"dropout_U": self.dropout_U}

base_config = super(ALSTM, self).get_config()

def __init__(self, n, **kwargs):

self.n = n

self.input_spec = [InputSpec(ndim=3)]

super(RepeatTimeDistributedVector, self).__init__(**kwargs)

def get_output_shape_for(self, input_shape):

return (input_shape[0], self.n, input_shape[1], input_shape[2])

def call(self, x, mask=None):

x = x.dimshuffle((0, 'x', 1, 2))

return T.extra_ops.repeat(x,self.n, axis=1)

def get_config(self):

config = {'n': self.n}

base_config = super(RepeatTimeDistributedVector, self).get_config()

def __init__(self, levels=2, init='glorot_uniform', weights=None, **kwargs):

self.levels = levels

self.init = initializations.get(init)

self.initial_weights = weights

super(HierarchicalSoftmax, self).__init__(**kwargs)

def build(self, input_shape):

self.output_dim = input_shape[1]

self.level_size = np.ceil(np.power(self.output_dim,1/self.levels))

self.W_shape = (self.output_dim, self.level_size)

self.W_list = []

self.b_list = []

for i in range(self.levels):

self.W_list.append(self.init(self.W_shape, name='W_{}'.format(i)))

self.b_list.append(K.zeros((self.level_size,), name='b_{}'.format(i)))

self.trainable_weights = self.W_list + self.b_list

if self.initial_weights is not None:

self.set_weights(self.initial_weights)

del self.initial_weights

def call(self, x, mask=None):

h_levels = []

for i in range(self.levels):

h_levels.append(K.softmax(K.dot(x, self.W_list[i]) + self.b_list[i]))

def _path_probas(idx):

results = []

for i in range(self.levels-1):

lev1_vec = h_levels[0][idx] if i == 0 else results[-1]

lev2_vec = h_levels[i+1][idx]

result, _ = theano.scan(fn=lambda k, array_: k * array_,

sequences=lev1_vec,

non_sequences=lev2_vec)

results.append(result.flatten())

return K.concatenate(results)

output, _ = theano.scan(fn=_path_probas, sequences=T.arange(x.shape[0]))

output = output[:, :self.output_dim]

return output

def get_config(self):

config = {'init': self.init.__name__}

base_config = super(HierarchicalSoftmax, self).get_config()