def __init__(self,
embed_dim,
from_inputs_features=None,
pos_divisor=10000,
keep_ndim=True,
fix_range=None,
embeddings=['sin', 'cos'],
**kwargs):
"""
embed_dim: Te output embedding will have embed_dim floats for sin and cos (separately).
from_inputs_features: If not specified, will use range(of the length of the input sequence) to generate
(integer) positions that will be embedded by sins and coss.
If specified, it needs to be a list of coordinates to the last dimension of the input vector,
which will be taken as inputs into the positional ebedding.
Then the output size will be len(from_inputs_features)*embed_dim*len(embeddings)
Has no effect when fix<-range is set.
pos_divisor: the division constant in the calculation.
keep_ndim: if True, the output will have all embedded features concatenated/flattened into one dimension and so
the input dimensions number is preserved.
fix_range: if set, will produce a sequence of a fixed range (does not read from sequence length)
and also disables from_inputs_features.
embeddings: a list of 'sin', 'cos', 'lin' functions to be applied
"""
Layer.__init__(self, **kwargs)
self.pos_divisor = pos_divisor
self.embed_dim = embed_dim
self.keep_ndim = keep_ndim
self.from_inputs_features = from_inputs_features
self.fix_range = fix_range
self.embeddings = embeddings
def __init__(self,
mask_value=0,
include_self=True,
flatten_indices_features=False,
**kwargs):
Layer.__init__(self, **kwargs)
self.mask_value = mask_value
self.include_self = include_self
self.flatten_indices_features = flatten_indices_features
def __init__(self, usesoftmax=True, num_units=None, num_heads=8, dropout_rate=0, activation='relu', causality=False,
usequerymasks=True, **kwargs):
self.activation = activation
self.num_units = num_units
self.num_heads = num_heads
self.dropout_rate = dropout_rate
self.causality = causality
self.usesoftmax = usesoftmax
self.usequerymasks = usequerymasks
Layer.__init__(self, **kwargs)
def get_config(self):
config = {
'size': self.size,
'initializer': initializers.serialize(self.initializer),
'regularizer': regularizers.serialize(self.regularizer)
}
base_config = Layer.get_config(self)
return dict(list(base_config.items()) + list(config.items()))
def fix_model_duplicate_layers_by_name(model: Layer,
substitutions=None,
return_substitutions=False) -> Layer:
"""Fix duplicates in loaded model by name."""
if not substitutions:
substitutions = {}
if hasattr(model, 'layers'):
for idx, layer in enumerate(model.layers):
if layer.name in substitutions:
model.layers[idx] = substitutions[layer.name]
else:
model.layers[idx] = fix_model_duplicate_layers_by_name(
layer, substitutions=substitutions)
substitutions[layer.name] = layer
if return_substitutions:
return model, substitutions
else:
return model
def __init__(self,
size,
initializer='glorot_uniform',
regularizer=None,
name=None,
**kwargs):
self.size = tuple(size)
self.initializer = initializers.get(initializer)
self.regularizer = regularizers.get(regularizer)
if not name:
prefix = 'shared_weight'
name = prefix + '_' + str(K.get_uid(prefix))
Layer.__init__(self, name=name, **kwargs)
with K.name_scope(self.name):
self.kernel = self.add_weight(shape=self.size,
initializer=self.initializer,
name='kernel',
regularizer=self.regularizer)
self.trainable = True
self.built = True
# self.sparse = sparse
input_tensor = self.kernel * 1.0
self.is_placeholder = False
input_tensor._keras_shape = self.size
input_tensor._uses_learning_phase = False
input_tensor._keras_history = (self, 0, 0)
Node(self,
inbound_layers=[],
node_indices=[],
tensor_indices=[],
input_tensors=[input_tensor],
output_tensors=[input_tensor],
input_masks=[None],
output_masks=[None],
input_shapes=[self.size],
output_shapes=[self.size])
def conv2d3d(layer: Layer):
if type(layer).__name__ != 'Conv2D':
raise TypeError("not conv2d layer")
config = layer.get_config()
ks = config['kernel_size']
config['kernel_size'] = (ks[0], ) + ks
st = config['strides']
config['strides'] = (st[0], ) + st
dl = config['dilation_rate']
config['dilation_rate'] = (dl[0], ) + dl
result = Conv3D(**config)
return result
def simple_joint_qa_predicate_model(conf, res, **kwargs):
### INPUT LIST####
inputs = []
### setup encoders ###
word_embedding_layer = Embedding(input_dim=conf.word_vocab_size,
output_dim=conf.word_embedding_size,
weights=[res.word_embeddings.W],
name="word_embeddings")
if conf.predicate_encoder_embedding_type == "word":
question_token_input = Input(shape=(None, ),
dtype='int32',
name='question_token_input')
inputs.append(question_token_input)
token_embeddings = word_embedding_layer(question_token_input)
resulting_embeddings = token_embeddings
elif conf.predicate_encoder_embedding_type == "char":
question_char_input = Input(shape=(None, None),
dtype='int32',
name='question_char_input')
inputs.append(question_char_input)
char_sequence_encoder = get_text_encoder(
vocab_size=conf.char_vocab_size,
embedding_size=conf.char_embedding_size,
kernel_size=conf.question_char_kernel_size,
depth=conf.question_char_cnn_depth,
dropout=conf.dropout,
layer_type=conf.layer_type)
token_char_embeddings = BetterTimeDistributed(char_sequence_encoder)(
question_char_input)
resulting_embeddings = token_char_embeddings
### combination OF WORD and CHAR embeddings
else:
question_char_input = Input(shape=(None, None),
dtype='int32',
name='question_char_input')
question_token_input = Input(shape=(None, ),
dtype='int32',
name='question_token_input')
## add to inputs
inputs.append(question_char_input)
inputs.append(question_token_input)
char_sequence_encoder = get_text_encoder(
vocab_size=conf.char_vocab_size,
embedding_size=conf.char_embedding_size,
kernel_size=conf.question_char_kernel_size,
depth=conf.question_char_cnn_depth,
dropout=conf.dropout,
layer_type=conf.layer_type)
token_char_embeddings = BetterTimeDistributed(char_sequence_encoder)(
question_char_input)
token_embeddings = word_embedding_layer(question_token_input)
## concatenate
resulting_embeddings = concatenate(
[token_embeddings, token_char_embeddings])
if "att" in conf.predicate_encoder_embedding_type:
relative_position_input = Input(shape=(None, ),
dtype='int32',
name='relative_position_input')
inputs.append(relative_position_input)
relative_position_embeddings = Embedding(
input_dim=conf.subject_position_max_distance * 2 + 1,
output_dim=conf.distance_embedding_size,
name="distance_embeddings")(relative_position_input)
text_embedding = apply_sequence_attention_model(
conf.layer_type,
input_sequence=resulting_embeddings,
relative_position_embeddings=relative_position_embeddings,
embedding_size=conf.predicate_embedding_size,
depth=conf.predicate_embedding_depth,
kernel_size=conf.predicate_embedding_kernel_size,
dropout=conf.dropout)
else:
text_embedding = apply_sequence_model(
conf.layer_type,
input_sequence=resulting_embeddings,
embedding_size=conf.predicate_embedding_size,
depth=conf.predicate_embedding_depth,
kernel_size=conf.predicate_embedding_kernel_size,
dropout=conf.dropout)
### add DROPOUT
text_embedding = Dropout(conf.dropout)(text_embedding)
#### Different outputs based on predicate_model_type ####
if conf.predicate_model_type == "predict_all_predicates":
answer_scores = Dense(conf.predicate_vocab_size,
activation="softmax")(text_embedding)
answer_scores = Layer(name="answer_scores")(answer_scores)
# loss_function = "categorical_crossentropy"
# metrics = "accuracy"
elif conf.predicate_model_type == "predict_graph_embedding":
answer_scores = Dense(conf.graph_embedding_size,
activation="selu")(text_embedding)
answer_scores = Dense(conf.graph_embedding_size,
activation="linear")(answer_scores)
answer_scores = Layer(name="answer_scores")(answer_scores)
# loss_function = "cosine_proximity"
# metrics = "cosine_proximity"
else:
#### BINARY classification for candidate predicate
graph_embedding_layer = Embedding(input_dim=conf.graph_vocab_size,
output_dim=conf.graph_embedding_size,
weights=[res.graph_embeddings.W],
name="graph_embeddings")
predicate_graph_embedding_input = Input(
shape=(None, ),
dtype='int32',
name='predicate_graph_embedding_input')
inputs.append(predicate_graph_embedding_input)
predicate_graph_embedding = graph_embedding_layer(
predicate_graph_embedding_input)
predicate_graph_embedding = ElementAt(0)(predicate_graph_embedding)
concatenated = concatenate([predicate_graph_embedding, text_embedding])
concatenated_layer_output = Dense(200, activation="selu")(concatenated)
answer_scores = Dense(1,
activation="sigmoid")(concatenated_layer_output)
answer_scores = Layer(name="answer_scores")(answer_scores)
# loss_function = "binary_crossentropy"
# metrics = "accuracy"
outputs = [answer_scores]
model = Model(inputs=inputs, outputs=outputs)
# if conf.gpus > 1:
# model = multi_gpu_model(model, gpus=conf.gpus)
model.compile(Adam(),
conf.predicate_model_loss_function,
metrics=[conf.predicate_model_metrics])
return model
def simple_joint_qa(conf, res, **kwargs):
### setup inputs ###
# word_vocab_size = conf., word_embedding_size, word_embedding_weights, char_vocab_size, char_embedding_size,
# match_embedding_size,
### INPUT LIST####
inputs = []
question_char_input = Input(shape=(None, ),
dtype='int32',
name='question_char_input')
question_token_input = Input(shape=(None, ),
dtype='int32',
name='question_token_input')
candidate_subject_labels_input = Input(
shape=(None, None),
dtype='int32',
name='candidate_subject_labels_input')
candidate_predicate_labels_input = Input(
shape=(None, None),
dtype='int32',
name='candidate_predicate_labels_input')
## add to inputs
inputs.append(question_char_input)
inputs.append(question_token_input)
inputs.append(candidate_subject_labels_input)
inputs.append(candidate_predicate_labels_input)
### setup encoders ###
word_embedding_layer = Embedding(input_dim=conf.word_vocab_size,
output_dim=conf.word_embedding_size,
weights=[res.word_embeddings.W],
name="word_embeddings")
char_sequence_encoder = get_text_encoder(
vocab_size=conf.char_vocab_size,
embedding_size=conf.char_embedding_size,
kernel_size=conf.question_char_kernel_size,
depth=conf.question_char_cnn_depth,
dropout=conf.dropout,
layer_type=conf.layer_type)
predicate_token_sequence_encoder = get_text_encoder(
vocab_size=conf.word_vocab_size,
embedding_size=conf.word_embedding_size,
kernel_size=conf.predicate_token_kernel_size,
depth=conf.predicate_token_cnn_depth,
embedding_layer=word_embedding_layer,
dropout=conf.dropout,
layer_type=conf.layer_type)
question_token_sequence_encoder = get_text_encoder(
vocab_size=conf.word_vocab_size,
embedding_size=conf.word_embedding_size,
kernel_size=conf.question_token_kernel_size,
depth=conf.question_token_cnn_depth,
embedding_layer=word_embedding_layer,
dropout=conf.dropout,
layer_type=conf.layer_type)
### encode inputs ###
question_char_embedding = char_sequence_encoder(question_char_input)
question_token_embedding = question_token_sequence_encoder(
question_token_input)
subject_label_embeddings = BetterTimeDistributed(char_sequence_encoder)(
candidate_subject_labels_input)
predicate_label_embeddings = BetterTimeDistributed(
predicate_token_sequence_encoder)(candidate_predicate_labels_input)
repeated_question_char_embedding = RepeatToMatch()(
[question_char_embedding, subject_label_embeddings])
repeated_question_token_embedding = RepeatToMatch()(
[question_token_embedding, predicate_label_embeddings])
### compute partial matches ###
## use graph embeddings
if conf.use_graph_embeddings:
candidate_subject_graph_embedding_input = Input(
shape=(None, ),
dtype='int32',
name='candidate_subject_graph_embeddings_input')
candidate_predicate_graph_embedding_input = Input(
shape=(None, ),
dtype='int32',
name='candidate_predicate_graph_embeddings_input')
## add to inputs
inputs.append(candidate_subject_graph_embedding_input)
inputs.append(candidate_predicate_graph_embedding_input)
graph_embedding_layer = Embedding(input_dim=conf.graph_vocab_size,
output_dim=conf.graph_embedding_size,
weights=[res.graph_embeddings.W],
name="graph_embeddings")
subject_graph_embeddings = graph_embedding_layer(
candidate_subject_graph_embedding_input)
predicate_graph_embeddings = graph_embedding_layer(
candidate_predicate_graph_embedding_input)
subject_match_embeddings = concatenate([
repeated_question_char_embedding, subject_label_embeddings,
subject_graph_embeddings
])
predicate_match_embeddings = concatenate([
repeated_question_token_embedding, predicate_label_embeddings,
predicate_graph_embeddings
])
else:
subject_match_embeddings = concatenate(
[repeated_question_char_embedding, subject_label_embeddings])
predicate_match_embeddings = concatenate(
[repeated_question_token_embedding, predicate_label_embeddings])
subject_match_embeddings = BetterTimeDistributed(
Dense(conf.match_embedding_size,
activation="selu",
kernel_initializer="he_normal"))(subject_match_embeddings)
subject_match_embeddings = Dropout(conf.dropout)(subject_match_embeddings)
predicate_match_embeddings = BetterTimeDistributed(
Dense(conf.match_embedding_size,
activation="selu",
kernel_initializer="he_normal"))(predicate_match_embeddings)
predicate_match_embeddings = Dropout(
conf.dropout)(predicate_match_embeddings)
if conf.use_predicate_and_subject_outputs:
### SUBJECT output
subject_match_scores = BetterTimeDistributed(
Dense(1, activation="linear"))(subject_match_embeddings)
subject_match_scores = Squeeze()(subject_match_scores)
subject_match_scores = Activation("sigmoid")(subject_match_scores)
subject_answer_scores = Layer(
name="subject_answer_scores")(subject_match_scores)
### PREDICATE output
predicate_match_scores = BetterTimeDistributed(
Dense(1, activation="linear"))(predicate_match_embeddings)
predicate_match_scores = Squeeze()(predicate_match_scores)
predicate_match_scores = Activation("sigmoid")(predicate_match_scores)
predicate_answer_scores = Layer(
name="predicate_answer_scores")(predicate_match_scores)
### compute overall matches ###
overall_match_embedding = concatenate(
[subject_match_embeddings, predicate_match_embeddings])
overall_match_embedding = BetterTimeDistributed(
Dense(conf.match_embedding_size,
activation="selu",
kernel_initializer="he_normal"))(overall_match_embedding)
overall_match_embedding = Dropout(
conf.dropout)(overall_match_embedding)
overall_match_scores = BetterTimeDistributed(
Dense(1, activation="linear"))(overall_match_embedding)
overall_match_scores = Squeeze()(overall_match_scores)
overall_match_scores = Activation("softmax")(overall_match_scores)
answer_scores = Layer(name="answer_scores")(overall_match_scores)
#### 3 OUTPUTS -: pair, subject and predicate alone
outputs = [
answer_scores, subject_answer_scores, predicate_answer_scores
]
model = Model(inputs=inputs, outputs=outputs)
model.compile(Adam(), {
"answer_scores": "categorical_crossentropy",
"predicate_answer_scores": "binary_crossentropy",
"subject_answer_scores": "binary_crossentropy"
},
metrics=["accuracy", "crossentropy"],
loss_weights={
"answer_scores": conf.answer_loss_weight,
"predicate_answer_scores":
conf.predicate_answer_loss_weight,
"subject_answer_scores":
conf.subject_answer_loss_weight
})
#### SINGLE OUTPUT : CandidatePAIR Scores
else:
### compute overall matches ###
overall_match_embedding = concatenate(
[subject_match_embeddings, predicate_match_embeddings])
overall_match_embedding = BetterTimeDistributed(
Dense(conf.match_embedding_size,
activation="selu",
kernel_initializer="he_normal"))(overall_match_embedding)
overall_match_embedding = Dropout(
conf.dropout)(overall_match_embedding)
overall_match_scores = BetterTimeDistributed(
Dense(1, activation="linear"))(overall_match_embedding)
overall_match_scores = Squeeze()(overall_match_scores)
overall_match_scores = Activation("softmax")(overall_match_scores)
answer_scores = Layer(name="answer_scores")(overall_match_scores)
outputs = [answer_scores]
model = Model(inputs=inputs, outputs=outputs)
model.compile(Adam(),
"categorical_crossentropy",
metrics=["accuracy", "crossentropy"])
return model
def input2d3d(layer: Layer, t_filters):
config = layer.get_config()
result = Input()