def maybe_print_logs(self, input_values, output_values, training_step):
x, len_x = input_values['source_tensors']
y, len_y = input_values['target_tensors']
samples = output_values[0]
x_sample = x[0]
len_x_sample = len_x[0]
y_sample = y[0]
len_y_sample = len_y[0]
deco_print(
"Train Source[0]: " + array_to_string(
x_sample[:len_x_sample],
vocab=self.get_data_layer().params['source_idx2seq'],
delim=self.get_data_layer().params["delimiter"],
),
offset=4,
)
deco_print(
"Train Target[0]: " + array_to_string(
y_sample[:len_y_sample],
vocab=self.get_data_layer().params['target_idx2seq'],
delim=self.get_data_layer().params["delimiter"],
),
offset=4,
)
deco_print(
"Train Prediction[0]: " + array_to_string(
samples[0, :],
vocab=self.get_data_layer().params['target_idx2seq'],
delim=self.get_data_layer().params["delimiter"],
),
offset=4,
)
return {}
def maybe_print_logs(self, input_values, output_values):
x, len_x = input_values['source_tensors']
y, len_y = input_values['target_tensors']
samples = output_values[0]
x_sample = x[0]
len_x_sample = len_x[0]
y_sample = y[0]
len_y_sample = len_y[0]
deco_print(
"Train Source[0]: " + array_to_string(
x_sample[:len_x_sample],
vocab=self.get_data_layer().params['source_idx2seq'],
delim=self.get_data_layer().params["delimiter"],
),
offset=4,
)
deco_print(
"Train Target[0]: " + array_to_string(
y_sample[:len_y_sample],
vocab=self.get_data_layer().params['target_idx2seq'],
delim=self.get_data_layer().params["delimiter"],
),
offset=4,
)
deco_print(
"Train Prediction[0]: " + array_to_string(
samples[0, :],
vocab=self.get_data_layer().params['target_idx2seq'],
delim=self.get_data_layer().params["delimiter"],
),
offset=4,
)
return {}
def evaluate(self, input_values, output_values):
ex, elen_x = input_values['source_tensors']
ey, elen_y = input_values['target_tensors']
x_sample = ex[0]
len_x_sample = elen_x[0]
y_sample = ey[0]
len_y_sample = elen_y[0]
deco_print(
"*****EVAL Source[0]: " + array_to_string(
x_sample[:len_x_sample],
vocab=self.get_data_layer().corp.dictionary.idx2word,
delim=self.get_data_layer().params["delimiter"],
),
offset=4,
)
deco_print(
"*****EVAL Target[0]: " + array_to_string(
y_sample[:len_y_sample],
vocab=self.get_data_layer().corp.dictionary.idx2word,
delim=self.get_data_layer().params["delimiter"],
),
offset=4,
)
samples = output_values[0][0]
deco_print(
"*****EVAL Prediction[0]: " + array_to_string(
samples,
vocab=self.get_data_layer().corp.dictionary.idx2word,
delim=self.get_data_layer().params["delimiter"],
),
offset=4,
)
def evaluate(self, input_values, output_values):
ex, elen_x = input_values['source_tensors']
ey, elen_y = input_values['target_tensors']
x_sample = ex[0]
len_x_sample = elen_x[0]
y_sample = ey[0]
len_y_sample = elen_y[0]
deco_print(
"*****EVAL Source[0]: " + array_to_string(
x_sample[:len_x_sample],
vocab=self.get_data_layer().params['source_idx2seq'],
delim=self.get_data_layer().params["delimiter"],
),
offset=4,
)
deco_print(
"*****EVAL Target[0]: " + array_to_string(
y_sample[:len_y_sample],
vocab=self.get_data_layer().params['target_idx2seq'],
delim=self.get_data_layer().params["delimiter"],
),
offset=4,
)
samples = output_values[0]
deco_print(
"*****EVAL Prediction[0]: " + array_to_string(
samples[0, :],
vocab=self.get_data_layer().params['target_idx2seq'],
delim=self.get_data_layer().params["delimiter"],
),
offset=4,
)
preds, targets = [], []
if self.params.get('eval_using_bleu', True):
preds.extend([
transform_for_bleu(
sample,
vocab=self.get_data_layer().params['target_idx2seq'],
ignore_special=True,
delim=self.get_data_layer().params["delimiter"],
bpe_used=self.params.get('bpe_used', False),
) for sample in samples
])
targets.extend([[
transform_for_bleu(
yi,
vocab=self.get_data_layer().params['target_idx2seq'],
ignore_special=True,
delim=self.get_data_layer().params["delimiter"],
bpe_used=self.params.get('bpe_used', False),
)
] for yi in ey])
return preds, targets
def maybe_print_logs(self, input_values, output_values, training_step):
x, len_x = input_values['source_tensors']
y, len_y = input_values['target_tensors']
x_sample = x[0]
len_x_sample = len_x[0]
y_sample = y[0]
len_y_sample = len_y[0]
deco_print(
"Train Source[0]: " + array_to_string(
x_sample[:len_x_sample],
vocab=self.get_data_layer().corp.dictionary.idx2word,
delim=self.delimiter,
),
offset=4,
)
if self._lm_phase:
deco_print(
"Train Target[0]: " + array_to_string(
y_sample[:len_y_sample],
vocab=self.get_data_layer().corp.dictionary.idx2word,
delim=self.delimiter,
),
offset=4,
)
else:
deco_print(
"TRAIN Target[0]: " + str(np.argmax(y_sample)),
offset=4,
)
samples = output_values[0][0]
deco_print(
"TRAIN Prediction[0]: " + str(samples),
offset=4,
)
labels = np.argmax(y, 1)
preds = np.argmax(output_values[0], axis=-1)
print('Labels', labels)
print('Preds', preds)
deco_print(
"Accuracy: {:.4f}".format(metrics.accuracy(labels, preds)),
offset=4,
)
if self._print_f1:
deco_print(
"Precision: {:.4f} | Recall: {:.4f} | F1: {:.4f}".format(
metrics.precision(labels, preds),
metrics.recall(labels, preds),
metrics.f1(labels, preds)),
offset=4,
)
return {}
def evaluate(self, input_values, output_values):
ex, elen_x = input_values['source_tensors']
ey, elen_y = input_values['target_tensors']
x_sample = ex[0]
len_x_sample = elen_x[0]
y_sample = ey[0]
len_y_sample = elen_y[0]
deco_print(
"*****EVAL Source[0]: " + array_to_string(
x_sample[:len_x_sample],
vocab=self.get_data_layer().params['source_idx2seq'],
delim=self.get_data_layer().params["delimiter"],
),
offset=4,
)
deco_print(
"*****EVAL Target[0]: " + array_to_string(
y_sample[:len_y_sample],
vocab=self.get_data_layer().params['target_idx2seq'],
delim=self.get_data_layer().params["delimiter"],
),
offset=4,
)
samples = output_values[0]
deco_print(
"*****EVAL Prediction[0]: " + array_to_string(
samples[0, :],
vocab=self.get_data_layer().params['target_idx2seq'],
delim=self.get_data_layer().params["delimiter"],
),
offset=4,
)
preds, targets = [], []
if self.params.get('eval_using_bleu', True):
preds.extend([transform_for_bleu(
sample,
vocab=self.get_data_layer().params['target_idx2seq'],
ignore_special=True,
delim=self.get_data_layer().params["delimiter"],
bpe_used=self.params.get('bpe_used', False),
) for sample in samples])
targets.extend([[transform_for_bleu(
yi,
vocab=self.get_data_layer().params['target_idx2seq'],
ignore_special=True,
delim=self.get_data_layer().params["delimiter"],
bpe_used=self.params.get('bpe_used', False),
)] for yi in ey])
return preds, targets
def maybe_print_logs(self, input_values, output_values):
x, len_x, y, len_y = input_values
samples = output_values[0]
if not self.on_horovod:
x_sample = x[0][0]
len_x_sample = len_x[0][0]
y_sample = y[0][0]
len_y_sample = len_y[0][0]
else:
x_sample = x[0]
len_x_sample = len_x[0]
y_sample = y[0]
len_y_sample = len_y[0]
deco_print(
"Train Source[0]: " + array_to_string(
x_sample[:len_x_sample],
vocab=self.data_layer.params['source_idx2seq'],
delim=self.data_layer.params["delimiter"],
),
offset=4,
)
deco_print(
"Train Target[0]: " + array_to_string(
y_sample[:len_y_sample],
vocab=self.data_layer.params['target_idx2seq'],
delim=self.data_layer.params["delimiter"],
),
offset=4,
)
deco_print(
"Train Prediction[0]: " + array_to_string(
samples[0, :],
vocab=self.data_layer.params['target_idx2seq'],
delim=self.data_layer.params["delimiter"],
),
offset=4,
)
return {}
def infer(self, input_values, output_values):
vocab = self.get_data_layer().corp.dictionary.idx2word
seed_tokens = self.params['encoder_params']['seed_tokens']
for i in range(len(seed_tokens)):
print(output_values[0][i].shape)
print('Seed:', vocab[seed_tokens[i]] + '\n')
deco_print(
"Output: " + array_to_string(
output_values[0][i],
vocab=self.get_data_layer().corp.dictionary.idx2word,
delim=self.get_data_layer().params["delimiter"],
),
offset=4,
)
def infer(self, input_values, output_values):
if self._lm_phase:
vocab = self.get_data_layer().corp.dictionary.idx2word
seed_tokens = self.params['encoder_params']['seed_tokens']
for i in range(len(seed_tokens)):
print('Seed:', vocab[seed_tokens[i]] + '\n')
deco_print(
"Output: " + array_to_string(
output_values[0][i],
vocab=self.get_data_layer().corp.dictionary.idx2word,
delim=self.delimiter,
),
offset=4,
)
return []
else:
ex, elen_x = input_values['source_tensors']
ey, elen_y = None, None
if 'target_tensors' in input_values:
ey, elen_y = input_values['target_tensors']
n_samples = len(ex)
results = []
for i in range(n_samples):
current_x = array_to_string(
ex[i][:elen_x[i]],
vocab=self.get_data_layer().corp.dictionary.idx2word,
delim=self.delimiter,
),
current_pred = np.argmax(output_values[0][i])
curret_y = None
if ey is not None:
current_y = np.argmax(ey[i])
results.append((current_x[0], current_pred, current_y))
return results
def maybe_evaluate(self, inputs_per_batch, outputs_per_batch):
preds, targets = [], []
for input_values, output_values in zip(inputs_per_batch,
outputs_per_batch):
ex, elen_x, ey, elen_y = input_values
##################
if not self.on_horovod:
x_sample = ex[0][0]
len_x_sample = elen_x[0][0]
y_sample = ey[0][0]
len_y_sample = elen_y[0][0]
else:
x_sample = ex[0]
len_x_sample = elen_x[0]
y_sample = ey[0]
len_y_sample = elen_y[0]
deco_print(
"*****EVAL Source[0]: " + array_to_string(
x_sample[:len_x_sample],
vocab=self.data_layer.params['source_idx2seq'],
delim=self.data_layer.params["delimiter"],
),
offset=4,
)
deco_print(
"*****EVAL Target[0]: " + array_to_string(
y_sample[:len_y_sample],
vocab=self.data_layer.params['target_idx2seq'],
delim=self.data_layer.params["delimiter"],
),
offset=4,
)
samples = output_values[0]
deco_print(
"*****EVAL Prediction[0]: " + array_to_string(
samples[0, :],
vocab=self.data_layer.params['target_idx2seq'],
delim=self.data_layer.params["delimiter"],
),
offset=4,
)
samples = output_values
##################
if self.params.get('eval_using_bleu', True):
preds.extend([
transform_for_bleu(
si,
vocab=self.data_layer.params['target_idx2seq'],
ignore_special=True,
delim=self.data_layer.params["delimiter"],
bpe_used=self.params.get('bpe_used', False),
) for sample in samples for si in sample
])
targets.extend([[
transform_for_bleu(
yii,
vocab=self.data_layer.params['target_idx2seq'],
ignore_special=True,
delim=self.data_layer.params["delimiter"],
bpe_used=self.params.get('bpe_used', False),
)
] for yi in ey for yii in yi])
if self.params.get('eval_using_bleu', True):
eval_bleu = calculate_bleu(preds, targets)
deco_print("Eval BLUE score: {}".format(eval_bleu), offset=4)
return {'Eval_BLEU_Score': eval_bleu}
return {}
def evaluate(self, input_values, output_values):
ex, elen_x = input_values['source_tensors']
ey, elen_y = input_values['target_tensors']
x_sample = ex[0]
len_x_sample = elen_x[0]
y_sample = ey[0]
len_y_sample = elen_y[0]
return_values = {}
if self._lm_phase:
flip = random.random()
if flip <= 0.9:
return return_values
deco_print(
"*****EVAL Source[0]: " + array_to_string(
x_sample[:len_x_sample],
vocab=self.get_data_layer().corp.dictionary.idx2word,
delim=self.delimiter,
),
offset=4,
)
samples = np.argmax(output_values[0][0], axis=-1)
deco_print(
"*****EVAL Target[0]: " + array_to_string(
y_sample[:len_y_sample],
vocab=self.get_data_layer().corp.dictionary.idx2word,
delim=self.delimiter,
),
offset=4,
)
deco_print(
"*****EVAL Prediction[0]: " + array_to_string(
samples,
vocab=self.get_data_layer().corp.dictionary.idx2word,
delim=self.delimiter,
),
offset=4,
)
else:
deco_print(
"*****EVAL Source[0]: " + array_to_string(
x_sample[:len_x_sample],
vocab=self.get_data_layer().corp.dictionary.idx2word,
delim=self.delimiter,
),
offset=4,
)
samples = output_values[0][0]
deco_print(
"EVAL Target[0]: " + str(np.argmax(y_sample)),
offset=4,
)
deco_print(
"EVAL Prediction[0]: " + str(samples),
offset=4,
)
labels = np.argmax(ey, 1)
preds = np.argmax(output_values[0], axis=-1)
print('Labels', labels)
print('Preds', preds)
return_values['accuracy'] = metrics.accuracy(labels, preds)
if self._print_f1:
return_values['true_pos'] = metrics.true_positives(
labels, preds)
return_values['pred_pos'] = np.sum(preds)
return_values['actual_pos'] = np.sum(labels)
return return_values
