def create_tff_dataset(client_idxs, in_data, slot_data, intent_data,
num_clients):
"""Generates the dataset format required for tensorflow federated.
Args:
client_idxs: A list of clients for each client.
in_data: The input data.
slot_data: The slot labels.
intent_data: The intent labels.
num_clients: Number of clients.
Returns:
A dictionary of client ids mapped to the client dataset and an
additional validation dataset in case of personalization.
"""
train_fed_data = OrderedDict()
slot_inputs, slot_targets = slot_data[:, :-1], slot_data[:, 1:]
padding_masks, look_ahead_masks, intent_masks = create_masks(
in_data, slot_targets)
for i in range(num_clients):
client_idx = np.array(client_idxs[i])
train_idxs = client_idx
client_data = gather_indices(train_idxs, in_data, slot_inputs,
padding_masks, look_ahead_masks,
intent_masks, slot_targets, intent_data)
train_fed_data[str(i)] = client_data
return train_fed_data
def create_tff_dataset(client_idxs,
in_data,
out_data,
num_clients,
p13n_train_ratio=1.0):
"""Generates the dataset format required for tensorflow federated.
Args:
client_idxs: A list of clients for each client.
in_data: The input data.
out_data: The annotated outputs.
num_clients: Number of clients.
p13n_train_ratio: To be used for personalization experiments and indicates
the ration of train and eval splits.
Returns:
A dictionary of client ids mapped to the client dataset and an
additional validation dataset in case of personalization.
"""
train_fed_data = OrderedDict()
# Validation dataset for personalization experiments
valid_fed_data = OrderedDict()
out_inputs, out_targets = out_data[:, :-1], out_data[:, 1:]
padding_masks, look_ahead_masks, pointer_masks = create_masks(
in_data, out_targets)
for i in range(num_clients):
client_idx = np.array(client_idxs[i])
# For personalization split the instance into training and validation
# splits.
if p13n_train_ratio != 1.0:
np.random.shuffle(client_idx)
train_idxs = client_idx[:int(p13n_train_ratio * len(client_idx))]
eval_idxs = client_idx[int(p13n_train_ratio * len(client_idx)):]
else:
train_idxs = client_idx
client_data = gather_indices(train_idxs, in_data, out_inputs,
padding_masks, look_ahead_masks,
pointer_masks, out_targets)
train_fed_data[str(i)] = client_data
# Create the validation dataset for personalization
if p13n_train_ratio != 1.0:
client_data = gather_indices(train_idxs, in_data, out_inputs,
padding_masks, look_ahead_masks,
pointer_masks, out_targets)
valid_fed_data[str(i)] = client_data
return train_fed_data, valid_fed_data
def evaluate(model, dataset, slot_vocab, max_len=48):
"""Evaluates the performance of the model on the given dataset and
prints out the metrics.
Args:
model: The model to be evaluated.
dataset: The dataset on which the model is to be evaluated.
slot_vocab: The vocabulary of slot labels.
max_len: The number of outputs to be generated by the decoder.
"""
pred_intents = []
pred_slots = []
gt_intents = []
gt_slots = []
for inputs, slots, intents in dataset:
decoder_input = [slot_vocab['vocab'][SOS_TOKEN]] * inputs.shape[0]
output = tf.expand_dims(decoder_input, 1)
for i in range(max_len - 1):
padding_mask, look_ahead_mask, intent_mask = create_masks(
inputs, output)
predictions, p_intent = model(
(inputs, output, padding_mask, look_ahead_mask, intent_mask))
# select the last word from the seq_len dimension
predictions = predictions[:, -1:, :] # (batch_size, 1, vocab_size)
predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)
output = tf.concat([output, predicted_id], axis=-1)
if i == max_len - 2:
pred_intents.append(tf.argmax(p_intent, axis=-1).numpy())
pred_slots.append(output.numpy())
gt_slots.append(slots.numpy())
gt_intents.append(intents.numpy().squeeze())
pred_slots = np.vstack(pred_slots)
pred_intents = np.hstack(pred_intents)
gt_slots = np.vstack(gt_slots)
gt_intents = np.hstack(gt_intents)
intent_acc, semantic_acc, f1_score, _, _ = compute_metrics(
gt_slots, gt_intents, pred_slots, pred_intents, slot_vocab)
print("Intent Acc {:.4f}, Semantic Acc {:.2f}, F1 score {:.2f}".format(
intent_acc, semantic_acc, f1_score))
return semantic_acc, intent_acc, f1_score
def evaluate(model, dataset, out_vocab, max_len=66):
"""Evaluates the performance of the model on the given dataset and
prints out the metrics.
Args:
model: The model to be evaluated.
dataset: The dataset on which the model is to be evaluated.
out_vocab: The vocabulary of output labels.
max_len: The number of outputs to be generated by the decoder.
"""
pred_outputs = []
gt_outputs = []
for input_batch, gt_output in dataset:
decoder_input = [out_vocab['vocab'][SOS_TOKEN]] * input_batch.shape[0]
output = tf.expand_dims(decoder_input, 1)
for _ in range(max_len - 1):
padding_mask, look_ahead_mask, pointer_mask = create_masks(
input_batch, output)
pred_output = model((input_batch, output, padding_mask,
look_ahead_mask, pointer_mask),
training=False)
# select the last word from the seq_len dimension
pred_output = pred_output[:, -1:, :] # (batch_size, 1, vocab_size)
pred_output = tf.cast(tf.argmax(pred_output, axis=-1), tf.int32)
output = tf.concat([output, pred_output], axis=-1)
pred_outputs.append(output.numpy())
gt_outputs.append(gt_output.numpy().squeeze())
pred_outputs = np.vstack(pred_outputs)
gt_outputs = np.vstack(gt_outputs)
accuracy = compute_acc(gt_outputs, pred_outputs)
print("Accuracy {:.4f}".format(accuracy))
return accuracy
def train_step(inputs, outputs):
dec_inputs = outputs[:, :-1]
dec_target = outputs[:, 1:]
padding_mask, look_ahead_mask, pointer_mask = create_masks(
inputs, dec_target)
with tf.GradientTape() as tape:
y_pred = model((inputs, dec_inputs, padding_mask, look_ahead_mask,
pointer_mask),
training=True)
loss = loss_objective(dec_target, y_pred)
gradients = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(gradients, model.trainable_variables))
train_loss(loss)
def evaluate(sentence, tokenizer, model):
START_TOKEN, END_TOKEN = [tokenizer.vocab_size], [tokenizer.vocab_size + 1]
sentence = preprocess_sentence(sentence)
sentence = tf.expand_dims(START_TOKEN + tokenizer.encode(sentence) +
END_TOKEN,
axis=0)
output = tf.expand_dims(START_TOKEN, 0)
enc_padding_mask, look_ahead_mask, dec_padding_mask = create_masks(
sentence, output)
for _ in range(MAX_LENGTH):
#inputs, dec_inputs, enc_padding_mask, look_ahead_mask, dec_padding_mask, training
predictions = model(inputs=sentence,
dec_inputs=output,
enc_padding_mask=enc_padding_mask,
look_ahead_mask=look_ahead_mask,
dec_padding_mask=dec_padding_mask,
training=False)
#print('model output shape: {}'.format(tf.shape(predictions)))
# select the last word from the seq_len dimension
predictions = predictions[:, -1:, :]
#print(tf.shape(predictions))
predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)
#print(tf.shape(predicted_id))
#print('model prediction ID: {}'.format(predicted_id))
# return the result if the predicted_id is equal to the end token
if tf.equal(predicted_id, END_TOKEN[0]) is None:
break
# concatenated the predicted_id to the output which is given to the decoder
# as its input.
output = tf.concat([output, predicted_id], axis=-1)
return tf.squeeze(output, axis=0)
def train_step(inp, tar, model, optimizer, train_loss, train_accuracy):
inputs = inp['inputs']
print(type(tar))
dec_inputs = tar['outputs'][:, :-1]
outputs = tar['outputs'][:, 1:]
enc_padding_mask, look_ahead_mask, dec_padding_mask = create_masks(
inputs, dec_inputs)
#inputs, dec_inputs, enc_padding_mask, look_ahead_mask, dec_padding_mask, training
with tf.GradientTape() as tape:
predictions = model(inputs=inputs,
dec_inputs=dec_inputs,
enc_padding_mask=enc_padding_mask,
look_ahead_mask=look_ahead_mask,
dec_padding_mask=dec_padding_mask,
training=True)
loss = loss_function(outputs, predictions)
gradients = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(gradients, model.trainable_variables))
train_loss(loss)
train_accuracy(outputs, predictions)
def write_predictions_to_file(model,
dataset,
out_path,
input_path,
out_vocab,
max_len=66):
"""Decodes the outputs of the decoder of the model and writes the predictions to a file
Args:
model: The model to be evaluated.
dataset: The dataset on which the model is to be evaluated.
out_path: The path of the file to with the predictions have to be saved.
input_path: The path to the file containing the input queries.
out_vocab: The vocabulary of output labels.
max_len: The number of outputs to be generated by the decoder.
"""
pred_outputs = []
inputs = []
# Generate the predictions
for input_batch, _ in dataset:
decoder_input = [out_vocab['vocab'][SOS_TOKEN]] * input_batch.shape[0]
output = tf.expand_dims(decoder_input, 1)
for i in range(max_len - 1):
padding_mask, look_ahead_mask, pointer_mask = create_masks(
input_batch, output)
pred_output = model(input_batch, output, padding_mask,
look_ahead_mask, pointer_mask)
# select the last word from the seq_len dimension
pred_output = pred_output[:, -1:, :] # (batch_size, 1, vocab_size)
pred_output = tf.cast(tf.argmax(pred_output, axis=-1), tf.int32)
output = tf.concat([output, pred_output], axis=-1)
pred_outputs.append(output.numpy())
pred_outputs = np.vstack(pred_outputs)
decoded_preds = []
# Generate the input tokens to be used to decode the pointers
input_fd = open(input_path, 'r')
for inp in input_fd:
i = [token.strip() for token in inp.split(' ')]
inputs.append(i)
# Decode the precitions
for input_batch, pred in zip(inputs, pred_outputs):
output_decoded = []
for output_token in pred:
if output_token == 3:
break
decoded_token = out_vocab['rev'][output_token]
if decoded_token.startswith('@ptr'):
output_decoded.append(input_batch[int(
decoded_token.split('_')[-1])])
else:
if ']' in decoded_token:
decoded_token = ']'
output_decoded.append(decoded_token)
decoded_preds.append(' '.join(output_decoded[1:]))
# Write decoded predictions to file
with open(out_path, 'w+') as out:
for output in decoded_preds:
out.write(output + '\n')
