def run_complex_embedding_network(lookup_table, max_sequence_length):
embedding_dimension = lookup_table.shape[1]
sequence_input = Input(shape=(max_sequence_length, ), dtype='int32')
phase_embedding = phase_embedding_layer(
max_sequence_length, lookup_table.shape[0])(sequence_input)
amplitude_embedding = amplitude_embedding_layer(
np.transpose(lookup_table), max_sequence_length,
trainable=True)(sequence_input)
[seq_embedding_real, seq_embedding_imag
] = ComplexMultiply()([phase_embedding, amplitude_embedding])
[sentence_embedding_real, sentence_embedding_imag
] = ComplexAverage()([seq_embedding_real, seq_embedding_imag])
output = Complex1DProjection(dimension=embedding_dimension)(
[sentence_embedding_real, sentence_embedding_imag])
model = Model(sequence_input, output)
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
return model
def createModel(
dropout_rate=0.5,
optimizer='adam',
init_criterion="he",
projection=True,
):
# projection= True,max_sequence_length=56,nb_classes=2,dropout_rate=0.5,embedding_trainable=True,random_init=False
max_sequence_length = 56
nb_classes = 2
embedding_trainable = True
# can be searched by grid
random_init = False
embedding_dimension = lookup_table.shape[1]
sequence_input = Input(shape=(max_sequence_length, ), dtype='int32')
phase_embedding = Dropout(dropout_rate)(phase_embedding_layer(
max_sequence_length,
lookup_table.shape[0],
embedding_dimension,
trainable=embedding_trainable)(sequence_input))
amplitude_embedding = Dropout(dropout_rate)(amplitude_embedding_layer(
np.transpose(lookup_table),
max_sequence_length,
trainable=embedding_trainable,
random_init=random_init)(sequence_input))
[seq_embedding_real, seq_embedding_imag
] = ComplexMultiply()([phase_embedding, amplitude_embedding])
if (projection):
[sentence_embedding_real, sentence_embedding_imag
] = ComplexMixture()([seq_embedding_real, seq_embedding_imag])
sentence_embedding_real = Flatten()(sentence_embedding_real)
sentence_embedding_imag = Flatten()(sentence_embedding_imag)
else:
[sentence_embedding_real, sentence_embedding_imag
] = ComplexSuperposition()([seq_embedding_real, seq_embedding_imag])
# output = Complex1DProjection(dimension = embedding_dimension)([sentence_embedding_real, sentence_embedding_imag])
predictions = ComplexDense(units=nb_classes,
init_criterion=init_criterion,
activation='sigmoid',
bias_initializer=Constant(value=-1))([
sentence_embedding_real,
sentence_embedding_imag
])
output = GetReal()(predictions)
model = Model(sequence_input, output)
model.compile(loss="binary_crossentropy",
optimizer=optimizer,
metrics=['accuracy'])
return model
def run_complex_embedding_network_superposition(lookup_table,
max_sequence_length,
nb_classes=2,
random_init=True,
embedding_trainable=True):
embedding_dimension = lookup_table.shape[1]
sequence_input = Input(shape=(max_sequence_length, ), dtype='int32')
phase_embedding = phase_embedding_layer(
max_sequence_length,
lookup_table.shape[0],
embedding_dimension,
trainable=embedding_trainable)(sequence_input)
amplitude_embedding = amplitude_embedding_layer(
np.transpose(lookup_table),
max_sequence_length,
trainable=embedding_trainable,
random_init=random_init)(sequence_input)
[seq_embedding_real, seq_embedding_imag
] = ComplexMultiply()([phase_embedding, amplitude_embedding])
[sentence_embedding_real, sentence_embedding_imag
] = ComplexSuperposition()([seq_embedding_real, seq_embedding_imag])
# output = Complex1DProjection(dimension = embedding_dimension)([sentence_embedding_real, sentence_embedding_imag])
predictions = ComplexDense(units=nb_classes,
activation='sigmoid',
bias_initializer=Constant(value=-1))([
sentence_embedding_real,
sentence_embedding_imag
])
output = GetReal()(predictions)
model = Model(sequence_input, output)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
return model
def run_complex_embedding_network_mixture(lookup_table,
max_sequence_length,
nb_classes=2,
random_init=True,
embedding_trainable=True):
embedding_dimension = lookup_table.shape[1]
sequence_input = Input(shape=(max_sequence_length, ), dtype='int32')
phase_embedding = phase_embedding_layer(
max_sequence_length,
lookup_table.shape[0],
embedding_dimension,
trainable=embedding_trainable)(sequence_input)
amplitude_embedding = amplitude_embedding_layer(
np.transpose(lookup_table),
max_sequence_length,
trainable=embedding_trainable,
random_init=random_init)(sequence_input)
[seq_embedding_real, seq_embedding_imag
] = ComplexMultiply()([phase_embedding, amplitude_embedding])
[sentence_embedding_real, sentence_embedding_imag
] = ComplexMixture()([seq_embedding_real, seq_embedding_imag])
sentence_embedding_real = Flatten()(sentence_embedding_real)
sentence_embedding_imag = Flatten()(sentence_embedding_imag)
# output = Complex1DProjection(dimension = embedding_dimension)([sentence_embedding_real, sentence_embedding_imag])
predictions = ComplexDense(units=nb_classes,
activation='sigmoid',
bias_initializer=Constant(value=-1))([
sentence_embedding_real,
sentence_embedding_imag
])
output = GetReal()(predictions)
model = Model(sequence_input, output)
return model
def createModel(dropout_rate=0.5,
optimizer='adam',
learning_rate=0.1,
init_criterion="he",
projection=True,
activation="relu"):
# projection= True,max_sequence_length=56,nb_classes=2,dropout_rate=0.5,embedding_trainable=True,random_init=False
embedding_trainable = True
# can be searched by grid
random_init = False
embedding_dimension = lookup_table.shape[1]
sequence_input = Input(shape=(max_sequence_length, ), dtype='int32')
phase_embedding = Dropout(dropout_rate)(phase_embedding_layer(
max_sequence_length,
lookup_table.shape[0],
embedding_dimension,
trainable=embedding_trainable)(sequence_input))
amplitude_embedding = Dropout(dropout_rate)(amplitude_embedding_layer(
np.transpose(lookup_table),
max_sequence_length,
trainable=embedding_trainable,
random_init=random_init)(sequence_input))
[seq_embedding_real, seq_embedding_imag
] = ComplexMultiply()([phase_embedding, amplitude_embedding])
if (projection):
[sentence_embedding_real, sentence_embedding_imag
] = ComplexMixture()([seq_embedding_real, seq_embedding_imag])
sentence_embedding_real = Flatten()(sentence_embedding_real)
sentence_embedding_imag = Flatten()(sentence_embedding_imag)
else:
[sentence_embedding_real, sentence_embedding_imag
] = ComplexSuperposition()([seq_embedding_real, seq_embedding_imag])
# output = Complex1DProjection(dimension = embedding_dimension)([sentence_embedding_real, sentence_embedding_imag])
predictions = ComplexDense(units=nb_classes,
init_criterion=init_criterion,
activation='sigmoid',
bias_initializer=Constant(value=-1))([
sentence_embedding_real,
sentence_embedding_imag
])
output = GetReal()(predictions)
from keras import optimizers
if optimizer == "Nadam":
optimizer = optimizers.Nadam(lr=learning_rate, clipvalue=0.5)
else:
optimizer = optimizers.Adam(lr=learning_rate, clipvalue=0.5)
# optimizer = optimizers.SGD(lr=learning_rate, momentum=momentum)
# optimizer=Adadelta(lr=1.0, rho=0.95, epsilon=1e-09)
model = Model(sequence_input, output)
if nb_classes == 2:
loss = "binary_crossentropy"
else:
loss = "categorical_crossentropy"
model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])
return model