def arch_016(input_shape,
n_labels,
weight_input_shape,
l2w,
rnn_unit='simplernn'):
#IRNN
if l2w is None:
regularizer = None
else:
regularizer = l2(l2w)
inputs = Input(shape=input_shape[1:])
x = RepeatVector(input_shape[0])(inputs)
x = get_rnn_unit(rnn_unit,
128,
x,
activation='relu',
l2w=regularizer,
recurrent_dropout=0.25,
recurrent_initializer=Identity(1.))
outputs = Dense(n_labels, activation='sigmoid')(x)
weight_input = Input(shape=weight_input_shape)
return Model(input=[inputs, weight_input], output=[outputs]), weight_input
def dummy_model(input_shape, nb_actions, use_bias=True):
model = Sequential()
model.add(Flatten())
# model.add(Flatten(input_shape= input_shape))
model.add(
Dense(nb_actions,
kernel_initializer=Identity(gain=-1),
use_bias=use_bias,
bias_initializer=Constant(1)))
return model
def build(self, input_shape):
super(LogarithmLayer, self).build(input_shape)
self.kernel = self.add_weight(name='kernel',
shape=(input_shape[1], self.output_dim),
initializer=Identity(1. /
input_shape[1]),
trainable=True)
self.bias = self.add_weight(name='bias',
shape=(self.output_dim, ),
initializer='glorot_uniform',
trainable=True)
self.built = True
def build(self, input_shape):
assert len(input_shape) >= 2
input_dim = input_shape[-1]
dim = input_shape[-1]
shape = (dim, )
shape = tuple([1] * (len(input_shape) - 1) + [input_shape[-1]])
self.C = self.add_weight(shape=(input_dim, input_dim),
initializer=Zeros(),
name='C',
trainable=False)
self.U = self.add_weight(shape=(input_dim, input_dim),
initializer=Identity(),
name='U',
trainable=False)
self.input_spec = InputSpec(min_ndim=2, axes={-1: input_dim})
self.built = True
def InitLoader(init, init_scale, hidden_units):
if init == 'chain':
kernel_initializer = NonnormalSourceInit(scale=0.9)
recurrent_initializer = ChainInit(diag_gain=0.0,
offdiag_gain=init_scale)
elif init == 'fbchain':
kernel_initializer = NonnormalSourceInit(scale=0.9)
recurrent_initializer = FbChainInit(diag_gain=0.0,
offdiag_gain=init_scale)
elif init == 'orthogonal':
kernel_initializer = RandomNormal(stddev=0.9 / np.sqrt(hidden_units))
recurrent_initializer = Orthogonal(gain=init_scale)
elif init == 'identity':
kernel_initializer = RandomNormal(stddev=0.9 / np.sqrt(hidden_units))
recurrent_initializer = Identity(gain=init_scale)
else:
raise ValueError('Initializer not defined.')
return kernel_initializer, recurrent_initializer
def irnn_same_param(input_shape,
n_labels,
weight_input_shape,
l2w,
rnn_unit='simplernn'):
#IRNN
if l2w is None:
regularizer = None
else:
regularizer = l2(l2w)
n_features = input_shape[1]
if rnn_unit == 'lstm':
m = np.floor(
np.roots([4, 4 * (n_features + 1) + n_labels,
n_labels - 200000])[1])
elif rnn_unit == 'gru':
m = np.floor(
np.roots([3, 3 * (n_features + 1) + n_labels,
n_labels - 200000])[1])
elif rnn_unit == 'simplernn':
m = np.floor(
np.roots([1, 1 * (n_features + 1) + n_labels,
n_labels - 200000])[1])
mem_units = int(m)
inputs = Input(shape=input_shape[1:])
x = RepeatVector(input_shape[0])(inputs)
x = get_rnn_unit(rnn_unit,
mem_units,
x,
activation='relu',
l2w=regularizer,
recurrent_dropout=0.25,
recurrent_initializer=Identity(1.))
outputs = Dense(n_labels, activation='sigmoid')(x)
weight_input = Input(shape=weight_input_shape)
return Model(input=[inputs, weight_input], output=[outputs]), weight_input
def __init__(self,
momentum=0.99,
epsilon=1e-3,
moving_mean_initializer='zeros',
decomposition='cholesky',
group=16,
renorm=False,
center=True,
scale=True,
moving_cov_initializer=Identity(),
beta_initializer='zeros',
gamma_initializer='ones',
beta_regularizer=None,
gamma_regularizer=None,
beta_constraint=None,
gamma_constraint=None,
**kwargs):
assert decomposition in [
'cholesky', 'zca', 'zca-cor', 'pca', 'pca-cor'
]
super(DecorrelatedBatchNorm, self).__init__(**kwargs)
self.supports_masking = True
self.momentum = momentum
self.epsilon = epsilon
self.moving_mean_initializer = initializers.get(
moving_mean_initializer)
self.moving_cov_initializer = initializers.get(moving_cov_initializer)
self.axis = -1
self.renorm = renorm
self.group = group
self.decomposition = decomposition
self.center = center
self.scale = scale
self.beta_initializer = initializers.get(beta_initializer)
self.gamma_initializer = initializers.get(gamma_initializer)
self.beta_regularizer = regularizers.get(beta_regularizer)
self.gamma_regularizer = regularizers.get(gamma_regularizer)
self.beta_constraint = constraints.get(beta_constraint)
self.gamma_constraint = constraints.get(gamma_constraint)
def build(self, input_shape):
weights = []
for i in range(self.depth):
if self.init_type == 'Gaussian':
weights.append(
self.add_weight(
name=str(i),
shape=(d, d),
# initializer=RandomNormal(stddev=self.const*np.sqrt(self.d)**((self.depth-1)/self.depth)/self.d),
initializer=RandomNormal(
stddev=self.init_size**(1. / self.depth) *
(1. / np.sqrt(self.d))),
trainable=True))
elif self.init_type == 'Identity':
weights.append(
self.add_weight(name=str(i),
shape=(d, d),
initializer=Identity(
gain=self.init_size**(1. /
self.depth)),
trainable=True))
self.weight_list = weights
super(DeepLinear, self).build(input_shape)
for word, i in tokenizer.word_index.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
embedding_layer = Embedding(len(tokenizer.word_index) + 1,
300,
weights=[embedding_matrix],
input_length=MAX_LEN,
trainable=False)
sum_embeddings_layer = keras.layers.core.Lambda(lambda x: K.sum(x, axis=1),
output_shape=(300, ))
# 'simple way'
init_kernel = RandomNormal(stddev=0.001)
init_recurrent = Identity(0.01)
rnn_layer = SimpleRNN(units=300, activation='relu', kernel_initializer=init_kernel, \
recurrent_initializer=init_recurrent, implementation=1)
#############
lstm_layer = LSTM(units=300,
dropout=0.5,
recurrent_dropout=0.5,
implementation=2)
translate_layer = Dense(units=100, activation='relu')
premise = Input(shape=(MAX_LEN, ), dtype='int32')
hypothesis = Input(shape=(MAX_LEN, ), dtype='int32')
prem = embedding_layer(premise)
hypo = embedding_layer(hypothesis)
#prem = sum_embeddings_layer(prem)
#hypo = sum_embeddings_layer(hypo)
def build_rel_ext_model(num_classes, embed_matrix, pos_classes_num,
ent_class_num, dep_classes_num):
words_indices_input = Input(shape=(None, ), name='words_indices_input')
pos_tag_indices_input = Input(shape=(None, ), name='pos_tag_indices_input')
ent_tag_indices_input = Input(shape=(None, ), name='ent_tag_indices_input')
dep_tag_indices_input = Input(shape=(None, ), name='dep_tag_indices_input')
branch_1_indices = Input(shape=(None, ),
name='branch_1_indices',
dtype='int32')
branch_2_indices = Input(shape=(None, ),
name='branch_2_indices',
dtype='int32')
word_embeddings_out = Embedding(
embed_matrix.shape[0],
embed_matrix.shape[1],
weights=[embed_matrix],
trainable=False,
name='word_embeddings')(words_indices_input)
pos_embeddings_out = Embedding(
pos_classes_num,
pos_classes_num,
trainable=True,
embeddings_initializer=Identity())(pos_tag_indices_input)
ent_embeddings_out = Embedding(
ent_class_num,
ent_class_num,
trainable=True,
embeddings_initializer=Identity())(ent_tag_indices_input)
dep_embeddings_out = Embedding(
dep_classes_num,
dep_classes_num,
trainable=True,
embeddings_initializer=Identity())(dep_tag_indices_input)
concatenated_out = concatenate(
[word_embeddings_out, pos_embeddings_out, ent_embeddings_out],
name='concat')
bilstm_out = Bidirectional(LSTM(300,
dropout=0.5,
recurrent_dropout=0.25,
return_sequences=True),
name='bilstm')(concatenated_out)
bilstm_and_depededecies = concatenate([bilstm_out, dep_embeddings_out],
name='bilsm_and_depends')
gather_layer = Lambda(gather, output_shape=output_shape_of_gather)
branch_1_input = gather_layer([bilstm_and_depededecies, branch_1_indices])
branch1_lstm_out = Bidirectional(LSTM(300,
dropout=0.5,
recurrent_dropout=0.25,
return_sequences=False),
name='branch_1_bilstm')(branch_1_input)
branch_2_input = gather_layer([bilstm_and_depededecies, branch_2_indices])
branch2_lstm_out = Bidirectional(LSTM(300,
dropout=0.5,
recurrent_dropout=0.25,
return_sequences=False),
name='branch_2_bilstm')(branch_2_input)
concatenated_branch_out = concatenate([branch1_lstm_out, branch2_lstm_out],
name='branch_concat')
fc_out = Dense(num_classes, activation='softmax')(concatenated_branch_out)
model = Model([
words_indices_input, pos_tag_indices_input, ent_tag_indices_input,
dep_tag_indices_input, branch_1_indices, branch_2_indices
], fc_out)
model.compile(optimizer='nadam', loss='sparse_categorical_crossentropy')
model.name = 'relation_classifier'
return model
def build_whole_autoencoder(self):
i = Input(shape=(self.number_features(), ), name='value_input')
m = Input(shape=(self.number_features(), ), name='mask_input')
#create bias only layer for input
self.bias_only_layer = Dense(units=self.number_features,
kernel_initializer=Identity())
maskLayer = MaskCreateLayerWithModality(
self.hyperparams["initial_corruption_rate"], self.modalities)(m)
r = [
Multiply()([self.bias_only_layer(i), m]),
]
x = [
Multiply()([self.bias_only_layer(i), maskLayer]),
]
ri = Input(shape=(self.IR_size, ))
# build encoder and encoder end of the autoencoder
for encode_layer in self.encode_layers:
if encode_layer is not self.IR_layer:
newX = encode_layer(
Dropout(self.hyperparams["dropout_rate"])(x[-1]))
newR = encode_layer(
Dropout(self.hyperparams["dropout_rate"])(r[-1]))
else:
newX = encode_layer(x[-1])
newR = encode_layer(r[-1])
#apply activation and add to input for next layer if we aren't the final part of the encoder
if encode_layer is not self.IR_layer:
newX = Activation(self.activation_function)(newX)
newR = Activation(self.activation_function)(newR)
x.append(Concatenate()([newX, x[-1]]))
r.append(Concatenate()([newR, r[-1]]))
# if we are the final part, don't add activation or concatenation
else:
x.append(newX)
r.append(newR)
# build decoder end
#rx = Activation(self.activation_function)(x[-1])
#rit = Activation(self.activation_function)(ri)
y = [
x[-1],
]
d = [
ri,
]
for decode_layer in self.decode_layers:
if decode_layer is not self.decode_layers[-1]:
newY = decode_layer(
Dropout(self.hyperparams["dropout_rate"])(y[-1]))
newD = decode_layer(
Dropout(self.hyperparams["dropout_rate"])(d[-1]))
else:
newY = decode_layer(y[-1])
newD = decode_layer(d[-1])
# apply activation and add to input for next layer if we aren't the final part of the decoder
if decode_layer != self.decode_layers[-1]:
newY = Activation(self.activation_function)(newY)
newD = Activation(self.activation_function)(newD)
y.append(Concatenate()([newY, y[-1]]))
d.append(Concatenate()([newD, d[-1]]))
# if we are the final part, don't add activation or concatenation
else:
y.append(newY)
d.append(newD)
y[-1] = Activation("sigmoid")(y[-1])
d[-1] = Activation("sigmoid")(d[-1])
# put on masking layer for faster training (A2 only)
y.append(Subtract()([i, y[-1]]))
a1out = Multiply()([y[-1], maskLayer])
a2out = Lambda(lambda x: 2.0 * x)(
Multiply()([y[-1], Subtract()([m, maskLayer])]))
zeros = Add()([a1out, a2out])
self.encoder = Model(inputs=[i, m], outputs=r[-1])
self.decoder = Model(inputs=[ri], outputs=d[-1])
self.autoencoder = Model(inputs=[i, m], outputs=[zeros])
#pop the trainable kernel off of the bias
self.bias_only_layer.non_trainable_weights.append(
self.bias_only_layer.trainable_weights.pop(0))