def one_hidden_layer_complex_nn(input_size = 300, output_size = 2):
input_1 = Input(shape = (input_size,input_size))
input_2 = Input(shape = (input_size,input_size))
# conv = ComplexConv1D(
# 32, 512, strides=16,
# activation='relu')(inputs)
# pool = AveragePooling1D(pool_size=4, strides=2)(conv)
# pool = Permute([2, 1])(pool)
# flattened = Flatten()(pool)
input_11 = Flatten()(input_1)
input_22 = Flatten()(input_2)
predictions = ComplexDense(units = output_size, activation='sigmoid', bias_initializer=Constant(value=0))([input_11, input_22])
# predictions = ComplexDense(
# output_size,
# activation='sigmoid',
# bias_initializer=Constant(value=-5))(dense)
predictions = GetReal()(predictions)
model = Model(inputs=[input_1, input_2], outputs=predictions)
model.compile(optimizer=Adam(lr=1e-4),
loss='binary_crossentropy',
metrics=['accuracy'])
return model
def createModel(
dropout_rate=0.5,
optimizer='adam',
init_criterion="he",
projection=True,
):
# projection= True,max_sequence_length=56,nb_classes=2,dropout_rate=0.5,embedding_trainable=True,random_init=False
max_sequence_length = 56
nb_classes = 2
embedding_trainable = True
# can be searched by grid
random_init = False
embedding_dimension = lookup_table.shape[1]
sequence_input = Input(shape=(max_sequence_length, ), dtype='int32')
phase_embedding = Dropout(dropout_rate)(phase_embedding_layer(
max_sequence_length,
lookup_table.shape[0],
embedding_dimension,
trainable=embedding_trainable)(sequence_input))
amplitude_embedding = Dropout(dropout_rate)(amplitude_embedding_layer(
np.transpose(lookup_table),
max_sequence_length,
trainable=embedding_trainable,
random_init=random_init)(sequence_input))
[seq_embedding_real, seq_embedding_imag
] = ComplexMultiply()([phase_embedding, amplitude_embedding])
if (projection):
[sentence_embedding_real, sentence_embedding_imag
] = ComplexMixture()([seq_embedding_real, seq_embedding_imag])
sentence_embedding_real = Flatten()(sentence_embedding_real)
sentence_embedding_imag = Flatten()(sentence_embedding_imag)
else:
[sentence_embedding_real, sentence_embedding_imag
] = ComplexSuperposition()([seq_embedding_real, seq_embedding_imag])
# output = Complex1DProjection(dimension = embedding_dimension)([sentence_embedding_real, sentence_embedding_imag])
predictions = ComplexDense(units=nb_classes,
init_criterion=init_criterion,
activation='sigmoid',
bias_initializer=Constant(value=-1))([
sentence_embedding_real,
sentence_embedding_imag
])
output = GetReal()(predictions)
model = Model(sequence_input, output)
model.compile(loss="binary_crossentropy",
optimizer=optimizer,
metrics=['accuracy'])
return model
def run_complex_embedding_network_superposition(lookup_table,
max_sequence_length,
nb_classes=2,
random_init=True,
embedding_trainable=True):
embedding_dimension = lookup_table.shape[1]
sequence_input = Input(shape=(max_sequence_length, ), dtype='int32')
phase_embedding = phase_embedding_layer(
max_sequence_length,
lookup_table.shape[0],
embedding_dimension,
trainable=embedding_trainable)(sequence_input)
amplitude_embedding = amplitude_embedding_layer(
np.transpose(lookup_table),
max_sequence_length,
trainable=embedding_trainable,
random_init=random_init)(sequence_input)
[seq_embedding_real, seq_embedding_imag
] = ComplexMultiply()([phase_embedding, amplitude_embedding])
[sentence_embedding_real, sentence_embedding_imag
] = ComplexSuperposition()([seq_embedding_real, seq_embedding_imag])
# output = Complex1DProjection(dimension = embedding_dimension)([sentence_embedding_real, sentence_embedding_imag])
predictions = ComplexDense(units=nb_classes,
activation='sigmoid',
bias_initializer=Constant(value=-1))([
sentence_embedding_real,
sentence_embedding_imag
])
output = GetReal()(predictions)
model = Model(sequence_input, output)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
return model
def run_complex_embedding_network_mixture(lookup_table,
max_sequence_length,
nb_classes=2,
random_init=True,
embedding_trainable=True):
embedding_dimension = lookup_table.shape[1]
sequence_input = Input(shape=(max_sequence_length, ), dtype='int32')
phase_embedding = phase_embedding_layer(
max_sequence_length,
lookup_table.shape[0],
embedding_dimension,
trainable=embedding_trainable)(sequence_input)
amplitude_embedding = amplitude_embedding_layer(
np.transpose(lookup_table),
max_sequence_length,
trainable=embedding_trainable,
random_init=random_init)(sequence_input)
[seq_embedding_real, seq_embedding_imag
] = ComplexMultiply()([phase_embedding, amplitude_embedding])
[sentence_embedding_real, sentence_embedding_imag
] = ComplexMixture()([seq_embedding_real, seq_embedding_imag])
sentence_embedding_real = Flatten()(sentence_embedding_real)
sentence_embedding_imag = Flatten()(sentence_embedding_imag)
# output = Complex1DProjection(dimension = embedding_dimension)([sentence_embedding_real, sentence_embedding_imag])
predictions = ComplexDense(units=nb_classes,
activation='sigmoid',
bias_initializer=Constant(value=-1))([
sentence_embedding_real,
sentence_embedding_imag
])
output = GetReal()(predictions)
model = Model(sequence_input, output)
return model
def createModel(dropout_rate=0.5,
optimizer='adam',
learning_rate=0.1,
init_criterion="he",
projection=True,
activation="relu"):
# projection= True,max_sequence_length=56,nb_classes=2,dropout_rate=0.5,embedding_trainable=True,random_init=False
embedding_trainable = True
# can be searched by grid
random_init = False
embedding_dimension = lookup_table.shape[1]
sequence_input = Input(shape=(max_sequence_length, ), dtype='int32')
phase_embedding = Dropout(dropout_rate)(phase_embedding_layer(
max_sequence_length,
lookup_table.shape[0],
embedding_dimension,
trainable=embedding_trainable)(sequence_input))
amplitude_embedding = Dropout(dropout_rate)(amplitude_embedding_layer(
np.transpose(lookup_table),
max_sequence_length,
trainable=embedding_trainable,
random_init=random_init)(sequence_input))
[seq_embedding_real, seq_embedding_imag
] = ComplexMultiply()([phase_embedding, amplitude_embedding])
if (projection):
[sentence_embedding_real, sentence_embedding_imag
] = ComplexMixture()([seq_embedding_real, seq_embedding_imag])
sentence_embedding_real = Flatten()(sentence_embedding_real)
sentence_embedding_imag = Flatten()(sentence_embedding_imag)
else:
[sentence_embedding_real, sentence_embedding_imag
] = ComplexSuperposition()([seq_embedding_real, seq_embedding_imag])
# output = Complex1DProjection(dimension = embedding_dimension)([sentence_embedding_real, sentence_embedding_imag])
predictions = ComplexDense(units=nb_classes,
init_criterion=init_criterion,
activation='sigmoid',
bias_initializer=Constant(value=-1))([
sentence_embedding_real,
sentence_embedding_imag
])
output = GetReal()(predictions)
from keras import optimizers
if optimizer == "Nadam":
optimizer = optimizers.Nadam(lr=learning_rate, clipvalue=0.5)
else:
optimizer = optimizers.Adam(lr=learning_rate, clipvalue=0.5)
# optimizer = optimizers.SGD(lr=learning_rate, momentum=momentum)
# optimizer=Adadelta(lr=1.0, rho=0.95, epsilon=1e-09)
model = Model(sequence_input, output)
if nb_classes == 2:
loss = "binary_crossentropy"
else:
loss = "categorical_crossentropy"
model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])
return model