def build(self, input_shape):
if isinstance(input_shape, list):
input_shape = input_shape[0]
batch_size = input_shape[0] if self.stateful else None
self.timestep_dim = input_shape[1]
self.input_dim = input_shape[2]
self.input_spec[0] = InputSpec(shape=(batch_size, None,
self.input_dim))
self.states = [None, None]
if self.stateful:
self.reset_states()
self.kernel = self.add_weight(shape=(self.input_dim, self.units * 4),
name='kernel',
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.recurrent_kernel = self.add_weight(
shape=(self.units, self.units * 4),
name='recurrent_kernel',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
# add attention kernel
self.attention_kernel = self.add_weight(
shape=(self.input_dim, self.units * 4),
name='attention_kernel',
initializer=self.attention_initializer,
regularizer=self.attention_regularizer,
constraint=self.attention_constraint)
# add attention weights
# weights for attention model
self.attention_weights = self.add_weight(
shape=(self.input_dim, self.units),
name='attention_W',
initializer=self.attention_initializer,
regularizer=self.attention_regularizer,
constraint=self.attention_constraint)
self.attention_recurrent_weights = self.add_weight(
shape=(self.units, self.units),
name='attention_U',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
if self.use_bias:
if self.unit_forget_bias:
def bias_initializer(shape, *args, **kwargs):
return K.concatenate([
self.bias_initializer((self.units, ), *args, **kwargs),
initializers.Ones()((self.units, ), *args, **kwargs),
self.bias_initializer((self.units * 2, ), *args,
**kwargs),
])
else:
bias_initializer = self.bias_initializer
self.bias = self.add_weight(shape=(self.units * 4, ),
name='bias',
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
self.attention_bias = self.add_weight(
shape=(self.units, ),
name='attention_b',
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
self.attention_recurrent_bias = self.add_weight(
shape=(self.units, 1),
name='attention_v',
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
else:
self.bias = None
self.attention_bias = None
self.attention_recurrent_bias = None
self.kernel_i = self.kernel[:, :self.units]
self.kernel_f = self.kernel[:, self.units:self.units * 2]
self.kernel_c = self.kernel[:, self.units * 2:self.units * 3]
self.kernel_o = self.kernel[:, self.units * 3:]
self.recurrent_kernel_i = self.recurrent_kernel[:, :self.units]
self.recurrent_kernel_f = self.recurrent_kernel[:,
self.units:self.units *
2]
self.recurrent_kernel_c = self.recurrent_kernel[:, self.units *
2:self.units * 3]
self.recurrent_kernel_o = self.recurrent_kernel[:, self.units * 3:]
self.attention_i = self.attention_kernel[:, :self.units]
self.attention_f = self.attention_kernel[:, self.units:self.units * 2]
self.attention_c = self.attention_kernel[:,
self.units * 2:self.units * 3]
self.attention_o = self.attention_kernel[:, self.units * 3:]
if self.use_bias:
self.bias_i = self.bias[:self.units]
self.bias_f = self.bias[self.units:self.units * 2]
self.bias_c = self.bias[self.units * 2:self.units * 3]
self.bias_o = self.bias[self.units * 3:]
else:
self.bias_i = None
self.bias_f = None
self.bias_c = None
self.bias_o = None
self.built = True
def build(self, input_shape):
self.input_spec = [InputSpec(shape=input_shape)]
self.conv_layers = {
c: [] for c in ["i", "f", "c", "o", "a", "ahat"]
} # LSTM: (i)nput, (f)orget, (c)ell, (o) output; a, a_hat
self.e_up_layers = []
self.e_down_layers = []
self.e_layers = []
for l in range(self.nb_layers):
for c in ["i", "f", "c", "o"]:
act = (
self.LSTM_activation
if c == "c"
else self.LSTM_inner_activation
)
self.conv_layers[c].append(
Conv2D(
self.R_stack_sizes[l],
self.R_filt_sizes[l],
padding="same",
activation=act,
data_format=self.data_format,
)
)
act = "relu" if l == 0 else self.A_activation
self.conv_layers["ahat"].append(
Conv2D(
self.stack_sizes[l],
self.Ahat_filt_sizes[l],
padding="same",
activation=act,
data_format=self.data_format,
)
)
if l < self.nb_layers - 1:
self.conv_layers["a"].append(
Conv2D(
self.stack_sizes[l + 1],
self.A_filt_sizes[l],
padding="same",
activation=self.A_activation,
data_format=self.data_format,
)
)
self.upsample = UpSampling2D(data_format=self.data_format)
self.pool = MaxPooling2D(data_format=self.data_format)
self.trainable_weights = []
nb_row, nb_col = (
(input_shape[-2], input_shape[-1])
if self.data_format == "channels_first"
else (input_shape[-3], input_shape[-2])
)
for c in sorted(self.conv_layers.keys()):
for l in range(len(self.conv_layers[c])):
ds_factor = 2 ** l
if c == "ahat":
nb_channels = self.R_stack_sizes[l]
elif c == "a":
nb_channels = 2 * self.R_stack_sizes[l]
else:
nb_channels = (
self.stack_sizes[l] * 2 + self.R_stack_sizes[l]
)
if l < self.nb_layers - 1:
nb_channels += self.R_stack_sizes[l + 1]
in_shape = (
input_shape[0],
nb_channels,
nb_row // ds_factor,
nb_col // ds_factor,
)
if self.data_format == "channels_last":
in_shape = (
in_shape[0],
in_shape[2],
in_shape[3],
in_shape[1],
)
if c == "ahat":
# print("layer: %d, shape: %s" % (l, in_shape))
self.e_down_layers.append(Subtract())
self.e_up_layers.append(Subtract())
self.e_layers.append(Concatenate())
with K.name_scope("layer_e_down_" + str(l)):
self.e_down_layers[-1].build([in_shape, in_shape])
with K.name_scope("layer_e_up_" + str(l)):
self.e_up_layers[-1].build([in_shape, in_shape])
with K.name_scope("layer_e_" + str(l)):
self.e_layers[-1].build([in_shape, in_shape])
with K.name_scope("layer_" + c + "_" + str(l)):
self.conv_layers[c][l].build(in_shape)
self.trainable_weights += self.conv_layers[c][
l
].trainable_weights
self.states = [None] * self.nb_layers * 3
if self.extrap_start_time is not None:
self.t_extrap = K.variable(
self.extrap_start_time,
int if K.backend() != "tensorflow" else "int32",
)
self.states += [None] * 2 # [previous frame prediction, timestep]
def __init__(self, padding=(1, 1), **kwargs):
self.padding = tuple(padding)
self.input_spec = [InputSpec(ndim=4)]
super(ReflectionPadding2D, self).__init__(**kwargs)
def __init__(self,
stack_sizes,
R_stack_sizes,
A_filt_sizes,
Ahat_filt_sizes,
R_filt_sizes,
pixel_max=1.,
error_activation='relu',
A_activation='relu',
LSTM_activation='tanh',
LSTM_inner_activation='hard_sigmoid',
output_mode='error',
extrap_start_time=None,
data_format=K.image_data_format(),
**kwargs):
self.stack_sizes = stack_sizes
self.nb_layers = len(stack_sizes)
assert len(
R_stack_sizes
) == self.nb_layers, 'len(R_stack_sizes) must equal len(stack_sizes)'
self.R_stack_sizes = R_stack_sizes
assert len(A_filt_sizes) == (
self.nb_layers -
1), 'len(A_filt_sizes) must equal len(stack_sizes) - 1'
self.A_filt_sizes = A_filt_sizes
assert len(
Ahat_filt_sizes
) == self.nb_layers, 'len(Ahat_filt_sizes) must equal len(stack_sizes)'
self.Ahat_filt_sizes = Ahat_filt_sizes
assert len(R_filt_sizes) == (
self.nb_layers), 'len(R_filt_sizes) must equal len(stack_sizes)'
self.R_filt_sizes = R_filt_sizes
self.pixel_max = pixel_max
self.error_activation = activations.get(error_activation)
self.A_activation = activations.get(A_activation)
self.LSTM_activation = activations.get(LSTM_activation)
self.LSTM_inner_activation = activations.get(LSTM_inner_activation)
default_output_modes = ['prediction', 'error', 'all']
layer_output_modes = [
layer + str(n) for n in range(self.nb_layers)
for layer in ['R', 'E', 'A', 'Ahat']
]
assert output_mode in default_output_modes + layer_output_modes, 'Invalid output_mode: ' + str(
output_mode)
self.output_mode = output_mode
if self.output_mode in layer_output_modes:
self.output_layer_type = self.output_mode[:-1]
self.output_layer_num = int(self.output_mode[-1])
else:
self.output_layer_type = None
self.output_layer_num = None
self.extrap_start_time = extrap_start_time
assert data_format in {
'channels_last', 'channels_first'
}, 'data_format must be in {channels_last, channels_first}'
self.data_format = data_format
self.channel_axis = -3 if data_format == 'channels_first' else -1
self.row_axis = -2 if data_format == 'channels_first' else -3
self.column_axis = -1 if data_format == 'channels_first' else -2
super(PredNet, self).__init__(**kwargs)
self.input_spec = [InputSpec(ndim=5)]
def __init__(self, padding=2, flip=False, **kwargs):
super(PaddingCrop, self).__init__(**kwargs)
self.padding = padding
self.flip = flip
self.input_spec = [InputSpec(ndim=4)]
def __init__(self, *args, n_channels=3, mono=False, **kwargs):
super().__init__(*args, **kwargs)
self.input_spec = [InputSpec(ndim=4), InputSpec(ndim=4)]
from keras import backend as K
def __init__(self, mask_size, **kwargs):
super(MaskLayer, self).__init__(**kwargs)
self.mask_size = mask_size
self.input_spec = [InputSpec(ndim=4)]
def __init__(self, **kwargs):
super(Folding, self).__init__(**kwargs)
self.input_spec = InputSpec(ndim=3)
def __init__(self,
units,
activation='tanh',
recurrent_activation='hard_sigmoid',
attention_activation='tanh',
use_bias=True,
kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal',
attention_initializer='orthogonal',
bias_initializer='zeros',
unit_forget_bias=True,
kernel_regularizer=None,
recurrent_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
attention_regularizer=None,
kernel_constraint=None,
recurrent_constraint=None,
bias_constraint=None,
attention_constraint=None,
dropout=0.,
recurrent_dropout=0.,
return_attention=False,
implementation=1,
**kwargs):
super(AttentionLSTMCell, self).__init__(**kwargs)
self.input_spec = [InputSpec(ndim=3)]
self.units = units
self.activation = activations.get(activation)
self.recurrent_activation = activations.get(recurrent_activation)
self.attention_activation = activations.get(attention_activation)
self.use_bias = use_bias
self.kernel_initializer = initializers.get(kernel_initializer)
self.recurrent_initializer = initializers.get(recurrent_initializer)
self.attention_initializer = initializers.get(attention_initializer)
self.bias_initializer = initializers.get(bias_initializer)
self.unit_forget_bias = unit_forget_bias
self.kernel_regularizer = regularizers.get(kernel_regularizer)
self.recurrent_regularizer = regularizers.get(recurrent_regularizer)
self.bias_regularizer = regularizers.get(bias_regularizer)
self.activity_regularizer = regularizers.get(activity_regularizer)
self.attention_regularizer = regularizers.get(attention_regularizer)
self.kernel_constraint = constraints.get(kernel_constraint)
self.recurrent_constraint = constraints.get(recurrent_constraint)
self.bias_constraint = constraints.get(bias_constraint)
self.attention_constraint = constraints.get(attention_constraint)
self.dropout = min(1., max(0., dropout))
self.recurrent_dropout = min(1., max(0., recurrent_dropout))
self.return_attention = return_attention
self._dropout_mask = None
self._recurrent_dropout_mask = None
self.implementation = implementation
self.state_spec = [
InputSpec(shape=(None, self.units)),
InputSpec(shape=(None, self.units))
]
self.state_size = (self.units, self.units)
def __init__(self,
filters,
kernel_size,
input_components=None,
output_components=None,
strides=(1, 1),
padding='valid',
data_format=None,
dilation_rate=(1, 1),
activation=None,
use_bias=True,
pre_kernel_initializer='glorot_uniform',
kernel_initializer='glorot_uniform',
post_kernel_initializer='glorot_uniform',
bias_initializer='zeros',
pre_kernel_regularizer=None,
kernel_regularizer=None,
post_kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
pre_kernel_constraint=None,
kernel_constraint=None,
post_kernel_constraint=None,
bias_constraint=None,
**kwargs):
super(FactorizedConv2DTucker, self).__init__(**kwargs)
rank = 2
self.rank = rank
self.input_components = input_components
self.output_components = output_components
self.filters = filters
self.output_components = output_components
self.kernel_size = conv_utils.normalize_tuple(kernel_size, rank,
'kernel_size')
self.strides = conv_utils.normalize_tuple(strides, rank, 'strides')
self.padding = conv_utils.normalize_padding(padding)
self.data_format = K.normalize_data_format(data_format)
self.dilation_rate = conv_utils.normalize_tuple(
dilation_rate, rank, 'dilation_rate')
self.activation = activations.get(activation)
self.use_bias = use_bias
self.pre_kernel_initializer = initializers.get(pre_kernel_initializer)
self.kernel_initializer = initializers.get(kernel_initializer)
self.post_kernel_initializer = initializers.get(
post_kernel_initializer)
self.bias_initializer = initializers.get(bias_initializer)
self.pre_kernel_regularizer = regularizers.get(pre_kernel_regularizer)
self.kernel_regularizer = regularizers.get(kernel_regularizer)
self.post_kernel_regularizer = regularizers.get(
post_kernel_regularizer)
self.bias_regularizer = regularizers.get(bias_regularizer)
self.activity_regularizer = regularizers.get(activity_regularizer)
self.pre_kernel_constraint = constraints.get(pre_kernel_constraint)
self.kernel_constraint = constraints.get(kernel_constraint)
self.post_kernel_constraint = constraints.get(post_kernel_constraint)
self.bias_constraint = constraints.get(bias_constraint)
self.input_spec = InputSpec(ndim=rank + 2) # batch, H, W, C
def __init__(self, k=1, **kwargs):
super(KMaxPooling, self).__init__(**kwargs)
self.input_spec = InputSpec(ndim=3)
self.k = k
def __init__(self, hard_selection=False, **kwargs):
self.input_spec = [InputSpec(ndim=3)]
self.hard_selection = hard_selection
super(DotProductKnowledgeSelector, self).__init__(**kwargs)
def __init__(self, rate, **kwargs):
super(TimeStepDropout, self).__init__(rate, **kwargs)
self.input_spec = InputSpec(ndim=3)
def __init__(self, k=1, **kwargs):
super().__init__(**kwargs)
self.input_spec = InputSpec(ndim=3)
self.k = k
def __init__(self):
super(ImageOpt, self).__init__()
self.input_spec = [InputSpec(ndim=3)]
def __init__(self, size=(2, 2), data_format=None, **kwargs):
super(BilinearUpSampling2D, self).__init__(**kwargs)
self.data_format = conv_utils.normalize_data_format(data_format)
self.size = conv_utils.normalize_tuple(size, 2, 'size')
self.input_spec = InputSpec(ndim=4)
def __init__(self, k=8, sorted=True, **kwargs):
super().__init__(**kwargs)
self.input_spec = InputSpec(ndim=3)
self.k = k
self.sorted = sorted
def build(self, input_shape):
# input_shape = [num_samples, timesteps, num_nodes, input_dim]
if isinstance(input_shape, list):
input_shape = input_shape[0]
batch_size = input_shape[0] if self.stateful else None
self.input_dim = input_shape[3]
self.input_spec[0] = InputSpec(shape=(batch_size, None, self.num_nodes,
self.input_dim))
self.states = [
None, None
] # initial states: two zero tensors of shape (num_samples, units)
if self.stateful:
self.reset_states()
kernel_shape = (self.num_nodes,
(self.poly_degree + 1) * self.input_dim,
self.units * 4)
self.kernel_shape = kernel_shape # output_shape = [input_dim * K, output_dim * 4]
recurrent_kernel_shape = (self.num_nodes,
(self.poly_degree + 1) * self.units,
self.units * 4)
self.recurrent_kernel_shape = recurrent_kernel_shape # output_shape = [output_dim * K, output_dim * 4]
self.kernel = self.add_weight(shape=kernel_shape,
initializer=self.kernel_initializer,
name='kernel',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.recurrent_kernel = self.add_weight(
shape=recurrent_kernel_shape,
initializer=self.recurrent_initializer,
name='recurrent_kernel',
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
if self.use_bias:
self.bias = self.add_weight(shape=(
self.num_nodes,
self.units * 4,
),
initializer=self.bias_initializer,
name='bias',
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
if self.unit_forget_bias:
bias_value = np.zeros((
self.num_nodes,
self.units * 4,
))
bias_value[:, self.units:self.units * 2] = 1.
K.set_value(self.bias, bias_value)
else:
self.bias = None
self.kernel_i = self.kernel[:, :, :self.units]
self.kernel_f = self.kernel[:, :, self.units:self.units * 2]
self.kernel_c = self.kernel[:, :, self.units * 2:self.units * 3]
self.kernel_o = self.kernel[:, :, self.units * 3:]
self.recurrent_kernel_i = self.recurrent_kernel[:, :, :self.units]
self.recurrent_kernel_f = self.recurrent_kernel[:, :,
self.units:self.units *
2]
self.recurrent_kernel_c = self.recurrent_kernel[:, :, self.units *
2:self.units * 3]
self.recurrent_kernel_o = self.recurrent_kernel[:, :, self.units * 3:]
if self.use_bias:
self.bias_i = self.bias[:, :self.units]
self.bias_f = self.bias[:, self.units:self.units * 2]
self.bias_c = self.bias[:, self.units * 2:self.units * 3]
self.bias_o = self.bias[:, self.units * 3:]
else:
self.bias_i = None
self.bias_f = None
self.bias_c = None
self.bias_o = None
self.built = True
def build(self, input_shape):
input_shape = input_shape[1]
B, L, dim = input_shape
super(NormalAttention, self).build((B, L, 3 * dim))
self.input_spec = [InputSpec(min_ndim=2, axes={-1: dim})] * 2
def build(self, input_shape):
self.input_spec = [InputSpec(shape=input_shape)]
self.conv_layers = {c: [] for c in ['i', 'f', 'c', 'o', 'a', 'ahat']}
for l in range(self.nb_layers):
for c in ['i', 'f', 'c', 'o']:
act = self.LSTM_activation if c == 'c' else self.LSTM_inner_activation
self.conv_layers[c].append(
Conv2D(self.R_stack_sizes[l],
self.R_filt_sizes[l],
padding='same',
activation=act,
data_format=self.data_format))
act = 'relu' if l == 0 else self.A_activation
self.conv_layers['ahat'].append(
Conv2D(self.stack_sizes[l],
self.Ahat_filt_sizes[l],
padding='same',
activation=act,
data_format=self.data_format))
if l < self.nb_layers - 1:
self.conv_layers['a'].append(
Conv2D(self.stack_sizes[l + 1],
self.A_filt_sizes[l],
padding='same',
activation=self.A_activation,
data_format=self.data_format))
self.upsample = UpSampling2D(data_format=self.data_format)
self.pool = MaxPooling2D(data_format=self.data_format)
self.trainable_weights = []
nb_row, nb_col = (
input_shape[-2],
input_shape[-1]) if self.data_format == 'channels_first' else (
input_shape[-3], input_shape[-2])
for c in sorted(self.conv_layers.keys()):
for l in range(len(self.conv_layers[c])):
ds_factor = 2**l
if c == 'ahat':
nb_channels = self.R_stack_sizes[l]
elif c == 'a':
nb_channels = 2 * self.stack_sizes[l]
else:
nb_channels = self.stack_sizes[l] * 2 + self.R_stack_sizes[
l]
if l < self.nb_layers - 1:
nb_channels += self.R_stack_sizes[l + 1]
in_shape = (input_shape[0], nb_channels, nb_row // ds_factor,
nb_col // ds_factor)
if self.data_format == 'channels_last':
in_shape = (in_shape[0], in_shape[2], in_shape[3],
in_shape[1])
with K.name_scope('layer_' + c + '_' + str(l)):
self.conv_layers[c][l].build(in_shape)
self.trainable_weights += self.conv_layers[c][
l].trainable_weights
self.states = [None] * self.nb_layers * 3
if self.extrap_start_time is not None:
self.t_extrap = K.variable(
self.extrap_start_time,
int if K.backend() != 'tensorflow' else 'int32')
self.states += [None] * 2 # [previous frame prediction, timestep]
def __init__(self, **kwargs):
super(NormalAttention, self).__init__(units=1, **kwargs)
self.supports_masking = True
self.input_spec = [self.input_spec, InputSpec(min_ndim=2)]
def build(self, input_shape):
if isinstance(input_shape, list):
input_shape = input_shape[0]
batch_size = input_shape[0] if self.stateful else None
self.input_dim = input_shape[2]
self.input_spec[0] = InputSpec(shape=(batch_size, None,
self.input_dim))
self.states = [None, None]
if self.stateful:
self.reset_states()
input_dim = input_shape[2]
self.input_dim = input_dim
self.tt_output_shape[0] *= 4
num_inputs = int(np.prod(input_shape[2::]))
if np.prod(self.tt_input_shape) != num_inputs:
raise ValueError("The size of the input tensor (i.e. product "
"of the elements in tt_input_shape) should "
"equal to the number of input neurons %d." %
(num_inputs))
if self.tt_input_shape.shape[0] != self.tt_output_shape.shape[0]:
raise ValueError("The number of input and output dimensions "
"should be the same.")
if self.tt_ranks.shape[0] != self.tt_output_shape.shape[0] + 1:
raise ValueError("The number of the TT-ranks should be "
"1 + the number of the dimensions.")
if self.debug:
print 'tt_input_shape = ' + str(self.tt_input_shape)
print 'tt_output_shape = ' + str(self.tt_output_shape)
print 'tt_ranks = ' + str(self.tt_ranks)
np.random.seed(self.init_seed)
total_length = np.sum(self.tt_input_shape * self.tt_output_shape *
self.tt_ranks[1:] * self.tt_ranks[:-1])
local_cores_arr = np.random.randn(total_length)
self.kernel = self.add_weight((total_length, ),
initializer=self.kernel_initializer,
name='kernel',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
if self.use_bias:
if self.unit_forget_bias:
def bias_initializer(shape, *args, **kwargs):
return K.concatenate([
self.bias_initializer((self.units, ), *args, **kwargs),
initializers.Ones()((self.units, ), *args, **kwargs),
self.bias_initializer((self.units * 2, ), *args,
**kwargs),
])
else:
bias_initializer = self.bias_initializer
self.bias = self.add_weight(shape=(self.units * 4, ),
name='bias',
initializer=bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
else:
self.bias = None
self.inds = np.zeros(self.num_dim).astype('int32')
self.shapes = np.zeros((self.num_dim, 2)).astype('int32')
self.cores = [None] * self.num_dim
for k in range(self.num_dim - 1, -1, -1):
self.shapes[k] = (self.tt_input_shape[k] * self.tt_ranks[k + 1],
self.tt_ranks[k] * self.tt_output_shape[k])
self.cores[k] = self.kernel[self.inds[k]:self.inds[k] +
np.prod(self.shapes[k])]
if 0 < k:
self.inds[k - 1] = self.inds[k] + np.prod(self.shapes[k])
if self.debug:
print 'self.shapes = ' + str(self.shapes)
self.TT_size = total_length
self.full_size = (np.prod(self.tt_input_shape) *
np.prod(self.tt_output_shape))
self.compress_factor = 1. * self.TT_size / self.full_size
print 'Compression factor = ' + str(self.TT_size) + ' / ' \
+ str(self.full_size) + ' = ' + str(self.compress_factor)
self.recurrent_kernel = self.add_weight(
shape=(self.units, self.units * 4),
name='recurrent_kernel',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.inds = np.zeros(self.num_dim).astype('int32')
self.shapes = np.zeros((self.num_dim, 2)).astype('int32')
self.cores = [None] * (self.num_dim)
for k in range(self.num_dim - 1, -1, -1):
self.shapes[k] = (self.tt_input_shape[k] * self.tt_ranks[k + 1],
self.tt_ranks[k] * self.tt_output_shape[k])
self.cores[k] = self.kernel[self.inds[k]:self.inds[k] +
np.prod(self.shapes[k])]
if 0 < k:
self.inds[k - 1] = self.inds[k] + np.prod(self.shapes[k])
self.compress_factor = 1.*(local_cores_arr.size) / \
(np.prod(self.tt_input_shape)*np.prod(self.tt_output_shape))
print 'Compressrion factor = ' + str(self.compress_factor)
self.built = True
def build(self, input_shape):
self.input_spec = [InputSpec(shape=input_shape)]
super(ReflectionPadding2D, self).build(input_shape)
def __init__(self, **kwargs):
super(GlobalArgminPooling1D, self).__init__(**kwargs)
self.input_spec = InputSpec(ndim=3)
def build(self, input_shape):
# pylint: disable=attribute-defined-outside-init
"""
This is used by Keras to verify things, but also to build the weights.
The only differences from the Keras GRU (which we copied exactly
other than the below) are:
- We generate weights with dimension input_dim[2] - 1, rather than
dimension input_dim[2].
- There are a few variables which are created in non-'gpu' modes which
are not required, and actually raise errors in Theano if you include them in
the trainable weights(as Theano will alert you if you try to compute a gradient
of a loss wrt a constant). These are commented out but left in for clarity below.
"""
self.input_spec = [InputSpec(shape=input_shape)]
# Here we make all the weights with a dimension one smaller
# than the input, as we remove the attention weights.
self.input_dim = input_shape[2] - 1
if self.stateful:
self.reset_states()
else:
# initial states: all-zero tensor of shape (output_dim)
self.states = [None]
if self.consume_less == 'gpu':
self.W = self.init((self.input_dim, 3 * self.output_dim),
name='{}_W'.format(self.name))
self.U = self.inner_init((self.output_dim, 3 * self.output_dim),
name='{}_U'.format(self.name))
self.b = K.variable(np.hstack(
(np.zeros(self.output_dim), np.zeros(self.output_dim),
np.zeros(self.output_dim))),
name='{}_b'.format(self.name))
self.trainable_weights = [self.W, self.U, self.b]
else:
self.W_z = self.init((self.input_dim, self.output_dim),
name='{}_W_z'.format(self.name))
self.U_z = self.inner_init((self.output_dim, self.output_dim),
name='{}_U_z'.format(self.name))
self.b_z = K.zeros((self.output_dim, ),
name='{}_b_z'.format(self.name))
self.W_r = self.init((self.input_dim, self.output_dim),
name='{}_W_r'.format(self.name))
self.U_r = self.inner_init((self.output_dim, self.output_dim),
name='{}_U_r'.format(self.name))
self.b_r = K.zeros((self.output_dim, ),
name='{}_b_r'.format(self.name))
self.W_h = self.init((self.input_dim, self.output_dim),
name='{}_W_h'.format(self.name))
self.U_h = self.inner_init((self.output_dim, self.output_dim),
name='{}_U_h'.format(self.name))
self.b_h = K.zeros((self.output_dim, ),
name='{}_b_h'.format(self.name))
# self.W_z, self.U_z, self.b_z - we don't need these parameters anymore.
self.trainable_weights = [
self.W_r, self.U_r, self.b_r, self.W_h, self.U_h, self.b_h
]
self.W = K.concatenate([self.W_z, self.W_r, self.W_h])
self.U = K.concatenate([self.U_z, self.U_r, self.U_h])
self.b = K.concatenate([self.b_z, self.b_r, self.b_h])
self.regularizers = []
if self.W_regularizer:
self.W_regularizer.set_param(self.W)
self.regularizers.append(self.W_regularizer)
if self.U_regularizer:
self.U_regularizer.set_param(self.U)
self.regularizers.append(self.U_regularizer)
if self.b_regularizer:
self.b_regularizer.set_param(self.b)
self.regularizers.append(self.b_regularizer)
if self.initial_weights is not None:
self.set_weights(self.initial_weights)
del self.initial_weights
self.built = True
def __init__(
self,
stack_sizes,
R_stack_sizes,
A_filt_sizes,
Ahat_filt_sizes,
R_filt_sizes,
pixel_max=1.0,
error_activation="relu",
A_activation="relu",
LSTM_activation="tanh",
LSTM_inner_activation="hard_sigmoid",
output_mode="error",
extrap_start_time=None,
data_format=K.image_data_format(),
**kwargs
):
self.stack_sizes = stack_sizes
self.nb_layers = len(
stack_sizes
) # the number of layers (w/ each layer containing R, A_hat, A, E)
# now let's assert that arguments are consistent with the number of layer
assert (
len(R_stack_sizes) == self.nb_layers
), "len(R_stack_sizes) must equal len(stack_sizes)"
self.R_stack_sizes = R_stack_sizes
assert len(A_filt_sizes) == (
self.nb_layers - 1
), "len(A_filt_sizes) must equal len(stack_sizes) - 1"
self.A_filt_sizes = A_filt_sizes
assert (
len(Ahat_filt_sizes) == self.nb_layers
), "len(Ahat_filt_sizes) must equal len(stack_sizes)"
self.Ahat_filt_sizes = Ahat_filt_sizes
assert len(R_filt_sizes) == (
self.nb_layers
), "len(R_filt_sizes) must equal len(stack_sizes)"
self.R_filt_sizes = R_filt_sizes
self.pixel_max = pixel_max
self.error_activation = activations.get(error_activation)
self.A_activation = activations.get(A_activation)
self.LSTM_activation = activations.get(
LSTM_activation
) # act func for output (c)
self.LSTM_inner_activation = activations.get(
LSTM_inner_activation
) # act func for f,i,o gates
default_output_modes = [
"prediction",
"error",
"all",
]
layer_output_modes = [
layer + str(n)
for n in range(self.nb_layers)
for layer in ["R", "E", "A", "Ahat"]
]
assert (
output_mode in default_output_modes + layer_output_modes
), "Invalid output_mode: " + str(output_mode)
self.output_mode = output_mode
if self.output_mode in layer_output_modes:
self.output_layer_type = self.output_mode[:-1]
self.output_layer_num = int(self.output_mode[-1])
else:
self.output_layer_type = None
self.output_layer_num = None
self.extrap_start_time = extrap_start_time
assert data_format in {
"channels_last",
"channels_first",
}, "data_format must be in {channels_last, channels_first}"
self.data_format = data_format
self.channel_axis = -3 if data_format == "channels_first" else -1
self.row_axis = -2 if data_format == "channels_first" else -3
self.column_axis = -1 if data_format == "channels_first" else -2
super(PredNet, self).__init__(**kwargs)
self.input_spec = [InputSpec(ndim=5)]
def build(self, input_shape):
"""
See Appendix 2 of Bahdanau 2014, arXiv:1409.0473
for model details that correspond to the matrices here.
"""
self.batch_size, self.timesteps, self.input_dim = input_shape
if self.stateful:
super(AttentionDecoder, self).reset_states()
self.states = [None, None] # y, s
"""
Matrices for creating the context vector
"""
self.V_a = self.add_weight(shape=(self.units, ),
name='V_a',
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.W_a = self.add_weight(shape=(self.units, self.units),
name='W_a',
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.U_a = self.add_weight(shape=(self.input_dim, self.units),
name='U_a',
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.b_a = self.add_weight(shape=(self.units, ),
name='b_a',
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
"""
Matrices for the r (reset) gate
"""
self.C_r = self.add_weight(shape=(self.input_dim, self.units),
name='C_r',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.U_r = self.add_weight(shape=(self.units, self.units),
name='U_r',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.W_r = self.add_weight(shape=(self.output_dim, self.units),
name='W_r',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.b_r = self.add_weight(shape=(self.units, ),
name='b_r',
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
"""
Matrices for the z (update) gate
"""
self.C_z = self.add_weight(shape=(self.input_dim, self.units),
name='C_z',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.U_z = self.add_weight(shape=(self.units, self.units),
name='U_z',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.W_z = self.add_weight(shape=(self.output_dim, self.units),
name='W_z',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.b_z = self.add_weight(shape=(self.units, ),
name='b_z',
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
"""
Matrices for the proposal
"""
self.C_p = self.add_weight(shape=(self.input_dim, self.units),
name='C_p',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.U_p = self.add_weight(shape=(self.units, self.units),
name='U_p',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.W_p = self.add_weight(shape=(self.output_dim, self.units),
name='W_p',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.b_p = self.add_weight(shape=(self.units, ),
name='b_p',
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
"""
Matrices for making the final prediction vector
"""
self.C_o = self.add_weight(shape=(self.input_dim, self.output_dim),
name='C_o',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.U_o = self.add_weight(shape=(self.units, self.output_dim),
name='U_o',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.W_o = self.add_weight(shape=(self.output_dim, self.output_dim),
name='W_o',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.b_o = self.add_weight(shape=(self.output_dim, ),
name='b_o',
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
# For creating the initial state:
self.W_s = self.add_weight(shape=(self.input_dim, self.units),
name='W_s',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.input_spec = [
InputSpec(shape=(self.batch_size, self.timesteps, self.input_dim))
]
self.built = True
def build(self, input_shape):
self.input_spec = [InputSpec(shape=input_shape)]
def build(self, input_shape):
self.batch_size, self.timesteps, self.input_dim = input_shape
if self.stateful:
super(AttentionDecoder, self).reset_states()
self.states = [None, None] # y, s
self.V_a = self.add_weight(shape=(self.units, ),
name='V_a',
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.W_a = self.add_weight(shape=(self.units, self.units),
name='W_a',
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.U_a = self.add_weight(shape=(self.input_dim, self.units),
name='U_a',
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.b_a = self.add_weight(shape=(self.units, ),
name='b_a',
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
self.C_r = self.add_weight(shape=(self.input_dim, self.units),
name='C_r',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.U_r = self.add_weight(shape=(self.units, self.units),
name='U_r',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.W_r = self.add_weight(shape=(self.output_dim, self.units),
name='W_r',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.b_r = self.add_weight(shape=(self.units, ),
name='b_r',
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
self.C_z = self.add_weight(shape=(self.input_dim, self.units),
name='C_z',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.U_z = self.add_weight(shape=(self.units, self.units),
name='U_z',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.W_z = self.add_weight(shape=(self.output_dim, self.units),
name='W_z',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.b_z = self.add_weight(shape=(self.units, ),
name='b_z',
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
self.C_p = self.add_weight(shape=(self.input_dim, self.units),
name='C_p',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.U_p = self.add_weight(shape=(self.units, self.units),
name='U_p',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.W_p = self.add_weight(shape=(self.output_dim, self.units),
name='W_p',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.b_p = self.add_weight(shape=(self.units, ),
name='b_p',
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
self.C_o = self.add_weight(shape=(self.input_dim, self.output_dim),
name='C_o',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.U_o = self.add_weight(shape=(self.units, self.output_dim),
name='U_o',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.W_o = self.add_weight(shape=(self.output_dim, self.output_dim),
name='W_o',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.b_o = self.add_weight(shape=(self.output_dim, ),
name='b_o',
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
self.W_s = self.add_weight(shape=(self.input_dim, self.units),
name='W_s',
initializer=self.recurrent_initializer,
regularizer=self.recurrent_regularizer,
constraint=self.recurrent_constraint)
self.input_spec = [
InputSpec(shape=(self.batch_size, self.timesteps, self.input_dim))
]
self.built = True
