def __to_class_index(class_str: str, dataset: DatasetBase):
"""
Converts from class name to class index
"""
if not class_str:
return len(dataset.class_names())
return dataset.class_names().index(class_str)
def __get_set(dataset: DatasetBase, set_type: str):
"""
param set_type: can be either test for test set, dev/val for validation set
"""
if set_type == 'test':
return dataset.testing_examples()
elif set_type == 'dev' or set_type == 'val':
return dataset.validation_examples()
else:
raise ValueError('Unexpected set type {}'.format(set_type))
def __get_in_ref_out(self, dataset: DatasetBase, set_type):
if set_type == 'valid':
x_test, y_test = dataset.validation_examples()
elif set_type == 'test':
x_test, y_test = dataset.testing_examples()
else:
raise ValueError('Unknown set type: {}'.format(set_type))
y_hat_test = self.predict_class(x_test)
y_hat_test = [
LiwcCountsClassifier.__to_class_index(cat_str, dataset)
for cat_str in y_hat_test
]
return x_test, y_test, y_hat_test
def evaluate_detailed(self, dataset: DatasetBase, set_type):
_, y_test, y_hat_test = self.__get_in_ref_out(dataset, set_type)
return metrics.classification_report(
y_test,
y_hat_test,
target_names=dataset.class_names() + ['None'],
output_dict=True)
def evaluate_detailed(self, dataset: DatasetBase, set_type: str = 'test'):
y_test, y_hat_test = self.get_test_ref_out_pair(dataset, set_type)
return metrics.classification_report(
y_test,
y_hat_test,
target_names=dataset.class_names(),
output_dict=True)
def train(self, dataset: DatasetBase, epochs):
x_train, y_train = dataset.training_examples()
cat_to_words_dict = WordEmbeddingsMatching.__group_words_to_cats(
x_train, y_train)
cat_to_centroids_and_dist_dict = {
cat_str: self.__compute_centroid_and_distance(word_set)
for cat_str, word_set in cat_to_words_dict.items()
}
def evaluate(self, dataset: DatasetBase, set_type):
if set_type == 'valid':
x_test, y_test_one_hot = dataset.validation_examples()
elif set_type == 'test':
x_test, y_test_one_hot = dataset.testing_examples()
else:
raise ValueError('Unknown set type: {}'.format(set_type))
y_hat_test_prob = self.predict(x_test)
y_test = [np.argmax(np.array(r)) for r in y_test_one_hot]
y_hat_test = [np.argmax(np.array(r)) for r in y_hat_test_prob]
fscore = metrics.f1_score(y_test, y_hat_test, average='macro')
accuracy = metrics.accuracy_score(y_test, y_hat_test)
print(self._model.evaluate(x_test, y_test_one_hot))
result_dict = {'f-score': fscore, 'accuracy': accuracy}
return result_dict
def precision_recall_curves(self, dataset: DatasetBase, set_type='test'):
class_count = dataset.class_count()
y_out_prob = self.get_test_out_prob(dataset, set_type)
y_ref = self.get_test_ref_labels(dataset, set_type)
y_ref_onehot = np.array([
SequenceClassificationModel._to_one_hot(index, class_count)
for index in y_ref
])
precision_dict, recall_dict, threshold_dict, average_precision_dict = SequenceClassificationModel.precision_recall_curve_per_class(
y_ref_onehot, y_out_prob, dataset.class_names())
precision_dict['micro'], recall_dict['micro'], threshold_dict[
'micro'], average_precision_dict[
'micro'] = SequenceClassificationModel.precision_recall_curve_micro_average(
y_ref_onehot, y_out_prob, class_count)
precision_dict['macro'], recall_dict['macro'], threshold_dict[
'macro'], average_precision_dict[
'macro'] = SequenceClassificationModel.precision_recall_curve_macro_average(
y_ref_onehot, y_out_prob, class_count)
return precision_dict, recall_dict, threshold_dict, average_precision_dict
def train(self, dataset: DatasetBase, epochs):
self.reset_cached_test_output()
start = time.process_time()
optimizer = tf.keras.optimizers.Adam(learning_rate=5e-5)
self._model.compile(optimizer=optimizer, loss=self._model.compute_loss)
history = self._model.fit(
dataset.training_examples(),
epochs=epochs,
validation_data=dataset.validation_examples(),
verbose=run_verbosity)
duration = time.process_time() - start
self._model_params_dict['learning_rate'] = 5e-5
self._model_params_dict['epochs'] = epochs
self._model_params_dict['training timestamp'] = datetime.now(
).strftime('%Y-%m-%d %H:%M:%S')
self._model_params_dict['training time'] = duration
self._model_params_dict['class count'] = dataset.class_count()
return history
def train(self, dataset: DatasetBase, epochs):
if self.__class_count == 1:
loss = tf.keras.losses.BinaryCrossentropy()
metrics = [tf.keras.metrics.BinaryCrossentropy()]
elif self.__class_count >= 2:
loss = tf.keras.losses.CategoricalCrossentropy()
metrics = [tf.keras.metrics.CategoricalCrossentropy()]
else:
raise ValueError(
'Unexpected value for class count. Provided is {}'.format(
self.__class_count))
self._model.compile(optimizer=tf.keras.optimizers.Adam(),
loss=loss,
metrics=metrics)
x_train, y_train = dataset.training_examples()
x_valid, y_valid = dataset.validation_examples()
history = self._model.fit(x_train,
y_train,
batch_size=64,
epochs=epochs,
validation_data=(x_valid, y_valid))
return history