def setup_sectlabel_bow_glove_infer(request, clf_datasets_manager,
tmpdir_factory):
track_for_best = request.param
sample_proportion = 0.5
datasets_manager = clf_datasets_manager
word_embedder = WordEmbedder(embedding_type="glove_6B_50")
bow_encoder = BOW_Encoder(embedder=word_embedder)
classifier = SimpleClassifier(
encoder=bow_encoder,
encoding_dim=word_embedder.get_embedding_dimension(),
num_classes=2,
classification_layer_bias=True,
datasets_manager=datasets_manager,
)
train_metric = PrecisionRecallFMeasure(datasets_manager=datasets_manager)
validation_metric = PrecisionRecallFMeasure(
datasets_manager=datasets_manager)
test_metric = PrecisionRecallFMeasure(datasets_manager=datasets_manager)
optimizer = torch.optim.Adam(params=classifier.parameters())
batch_size = 1
save_dir = tmpdir_factory.mktemp("experiment_1")
num_epochs = 1
save_every = 1
log_train_metrics_every = 10
engine = Engine(
model=classifier,
datasets_manager=datasets_manager,
optimizer=optimizer,
batch_size=batch_size,
save_dir=save_dir,
num_epochs=num_epochs,
save_every=save_every,
log_train_metrics_every=log_train_metrics_every,
train_metric=train_metric,
validation_metric=validation_metric,
test_metric=test_metric,
track_for_best=track_for_best,
sample_proportion=sample_proportion,
)
engine.run()
model_filepath = pathlib.Path(save_dir).joinpath("best_model.pt")
infer = ClassificationInference(
model=classifier,
model_filepath=str(model_filepath),
datasets_manager=datasets_manager,
)
return infer
def __init__(
self,
model: nn.Module,
model_filepath: str,
datasets_manager: DatasetsManager,
tokens_namespace: str = "tokens",
normalized_probs_namespace: str = "normalized_probs",
):
super(ClassificationInference, self).__init__(
model=model,
model_filepath=model_filepath,
datasets_manager=datasets_manager,
)
self.batch_size = 32
self.tokens_namespace = tokens_namespace
self.normalized_probs_namespace = normalized_probs_namespace
self.label_namespace = self.datasets_manager.label_namespaces[0]
self.labelname2idx_mapping = self.datasets_manager.get_label_idx_mapping(
label_namespace=self.label_namespace)
self.idx2labelname_mapping = self.datasets_manager.get_idx_label_mapping(
label_namespace=self.label_namespace)
self.load_model()
self.metrics_calculator = PrecisionRecallFMeasure(
datasets_manager=datasets_manager)
self.output_analytics = None
# create a dataframe with all the information
self.output_df = None
def test_classifier_produces_correct_precision(self,
setup_simple_classifier):
iter_dict, simple_classifier, batch_size, num_classes = setup_simple_classifier
output = simple_classifier(iter_dict,
is_training=True,
is_validation=False,
is_test=False)
idx2labelname_mapping = {
0: "good class",
1: "bad class",
2: "average_class"
}
metrics_calc = PrecisionRecallFMeasure(
idx2labelname_mapping=idx2labelname_mapping)
metrics_calc.calc_metric(iter_dict=iter_dict,
model_forward_dict=output)
metrics = metrics_calc.get_metric()
precision = metrics["precision"]
# NOTE: topk returns the last value in the dimension incase
# all the values are equal.
expected_precision = {1: 0, 2: 0}
assert len(precision) == 2
for class_label, precision_value in precision.items():
assert precision_value == expected_precision[class_label]
def setup_data_one_true_class_missing():
"""
The batch of instances during training might not have all
true classes. What happens in that case??
The test case here captures the situation
:return:
"""
predicted_probs = torch.FloatTensor([[0.8, 0.1, 0.2], [0.2, 0.5, 0.3]])
idx2labelname_mapping = {
0: "good class",
1: "bad class",
2: "average_class"
}
labels = torch.LongTensor([0, 2]).view(-1, 1)
expected_precision = {0: 1.0, 1: 0.0, 2: 0.0}
expected_recall = {0: 1.0, 1: 0.0, 2: 0.0}
expected_fscore = {0: 1.0, 1: 0.0, 2: 0.0}
accuracy = PrecisionRecallFMeasure(
idx2labelname_mapping=idx2labelname_mapping)
return (
predicted_probs,
labels,
accuracy,
{
"expected_precision": expected_precision,
"expected_recall": expected_recall,
"expected_fscore": expected_fscore,
},
)
def setup_data_to_test_length():
predicted_probs = torch.FloatTensor([[0.1, 0.8, 0.2], [0.2, 0.3, 0.5]])
labels = torch.LongTensor([0, 2]).view(-1, 1)
idx2labelname_mapping = {
0: "good class",
1: "bad class",
2: "average_class"
}
accuracy = PrecisionRecallFMeasure(
idx2labelname_mapping=idx2labelname_mapping)
expected_length = 3
return predicted_probs, labels, accuracy, expected_length
def __init__(
self, model: nn.Module, model_filepath: str, dataset: BaseTextClassification
):
super(ClassificationInference, self).__init__(
model=model, model_filepath=model_filepath, dataset=dataset
)
self.batch_size = 32
self.labelname2idx_mapping = self.dataset.get_classname2idx()
self.idx2labelname_mapping = {
idx: label_name for label_name, idx in self.labelname2idx_mapping.items()
}
self.load_model()
self.metrics_calculator = PrecisionRecallFMeasure(
idx2labelname_mapping=self.idx2labelname_mapping
)
self.output_analytics = None
# create a dataframe with all the information
self.output_df = None
def setup_data_for_all_zeros(clf_dataset_manager):
predicted_probs = torch.FloatTensor([[0.9, 0.1], [0.3, 0.7]])
datasets_manager = clf_dataset_manager
labels = torch.LongTensor([1, 0]).view(-1, 1)
expected_precision = {0: 0.0, 1: 0.0}
expected_recall = {0: 0.0, 1: 0.0}
expected_fmeasure = {0: 0.0, 1: 0.0}
expected_macro_precision = 0.0
expected_macro_recall = 0.0
expected_macro_fscore = 0.0
expected_num_tps = {0: 0.0, 1: 0.0}
expected_num_fps = {0: 1.0, 1: 1.0}
expected_num_fns = {0: 1.0, 1: 1.0}
expected_micro_precision = 0.0
expected_micro_recall = 0.0
expected_micro_fscore = 0.0
prf_metric = PrecisionRecallFMeasure(datasets_manager=datasets_manager)
return (
predicted_probs,
labels,
prf_metric,
datasets_manager,
{
"expected_precision": expected_precision,
"expected_recall": expected_recall,
"expected_fscore": expected_fmeasure,
"expected_macro_precision": expected_macro_precision,
"expected_macro_recall": expected_macro_recall,
"expected_macro_fscore": expected_macro_fscore,
"expected_num_tps": expected_num_tps,
"expected_num_fps": expected_num_fps,
"expected_num_fns": expected_num_fns,
"expected_micro_precision": expected_micro_precision,
"expected_micro_recall": expected_micro_recall,
"expected_micro_fscore": expected_micro_fscore,
},
)
def setup_data_basecase(clf_dataset_manager):
dataset_manager = clf_dataset_manager
prf_metric = PrecisionRecallFMeasure(dataset_manager)
predicted_probs = torch.FloatTensor([[0.1, 0.9], [0.7, 0.3]])
labels = torch.LongTensor([1, 0]).view(-1, 1)
expected_precision = {0: 1.0, 1: 1.0}
expected_recall = {0: 1.0, 1: 1.0}
expected_fmeasure = {0: 1.0, 1: 1.0}
expected_macro_precision = 1.0
expected_macro_recall = 1.0
expected_macro_fscore = 1.0
expected_num_tps = {0: 1.0, 1: 1.0}
expected_num_fps = {0: 0.0, 1: 0.0}
expected_num_fns = {0: 0.0, 1: 0.0}
expected_micro_precision = 1.0
expected_micro_recall = 1.0
expected_micro_fscore = 1.0
return (
predicted_probs,
labels,
prf_metric,
dataset_manager,
{
"expected_precision": expected_precision,
"expected_recall": expected_recall,
"expected_fscore": expected_fmeasure,
"expected_macro_precision": expected_macro_precision,
"expected_macro_recall": expected_macro_recall,
"expected_macro_fscore": expected_macro_fscore,
"expected_num_tps": expected_num_tps,
"expected_num_fps": expected_num_fps,
"expected_num_fns": expected_num_fns,
"expected_micro_precision": expected_micro_precision,
"expected_micro_recall": expected_micro_recall,
"expected_micro_fscore": expected_micro_fscore,
},
)
def setup_data_basecase():
predicted_probs = torch.FloatTensor([[0.1, 0.9], [0.7, 0.3]])
labels = torch.LongTensor([1, 0]).view(-1, 1)
idx2labelname_mapping = {0: "good class", 1: "bad class"}
expected_precision = {0: 1.0, 1: 1.0}
expected_recall = {0: 1.0, 1: 1.0}
expected_fmeasure = {0: 1.0, 1: 1.0}
expected_macro_precision = 1.0
expected_macro_recall = 1.0
expected_macro_fscore = 1.0
expected_num_tps = {0: 1.0, 1: 1.0}
expected_num_fps = {0: 0.0, 1: 0.0}
expected_num_fns = {0: 0.0, 1: 0.0}
expected_micro_precision = 1.0
expected_micro_recall = 1.0
expected_micro_fscore = 1.0
accuracy = PrecisionRecallFMeasure(
idx2labelname_mapping=idx2labelname_mapping)
return (
predicted_probs,
labels,
accuracy,
{
"expected_precision": expected_precision,
"expected_recall": expected_recall,
"expected_fscore": expected_fmeasure,
"expected_macro_precision": expected_macro_precision,
"expected_macro_recall": expected_macro_recall,
"expected_macro_fscore": expected_macro_fscore,
"expected_num_tps": expected_num_tps,
"expected_num_fps": expected_num_fps,
"expected_num_fns": expected_num_fns,
"expected_micro_precision": expected_micro_precision,
"expected_micro_recall": expected_micro_recall,
"expected_micro_fscore": expected_micro_fscore,
},
)
combine_strategy=COMBINE_STRATEGY,
device=torch.device(DEVICE),
)
encoding_dim = (2 * HIDDEN_DIMENSION if BIDIRECTIONAL
and COMBINE_STRATEGY == "concat" else HIDDEN_DIMENSION)
model = SimpleClassifier(
encoder=encoder,
encoding_dim=encoding_dim,
num_classes=NUM_CLASSES,
classification_layer_bias=True,
)
optimizer = optim.Adam(params=model.parameters(), lr=LEARNING_RATE)
metric = PrecisionRecallFMeasure(
idx2labelname_mapping=train_dataset.idx2classname)
engine = Engine(
model=model,
train_dataset=train_dataset,
validation_dataset=validation_dataset,
test_dataset=test_dataset,
optimizer=optimizer,
batch_size=BATCH_SIZE,
save_dir=MODEL_SAVE_DIR,
num_epochs=NUM_EPOCHS,
save_every=SAVE_EVERY,
log_train_metrics_every=LOG_TRAIN_METRICS_EVERY,
tensorboard_logdir=TENSORBOARD_LOGDIR,
device=torch.device(DEVICE),
metric=metric,
encoder = BOW_Encoder(embedder=embedder,
aggregation_type=args.word_aggregation,
device=args.device)
model = SimpleClassifier(
encoder=encoder,
encoding_dim=1024,
num_classes=data_manager.num_labels["label"],
classification_layer_bias=True,
datasets_manager=data_manager,
device=args.device,
)
optimizer = optim.Adam(params=model.parameters(), lr=args.lr)
train_metric = PrecisionRecallFMeasure(datasets_manager=data_manager)
dev_metric = PrecisionRecallFMeasure(datasets_manager=data_manager)
test_metric = PrecisionRecallFMeasure(datasets_manager=data_manager)
engine = Engine(
model=model,
datasets_manager=data_manager,
optimizer=optimizer,
batch_size=args.bs,
save_dir=args.model_save_dir,
num_epochs=args.epochs,
save_every=args.save_every,
log_train_metrics_every=args.log_train_metrics_every,
device=args.device,
train_metric=train_metric,
validation_metric=dev_metric,
hidden_dim=HIDDEN_DIMENSION,
combine_strategy=COMBINE_STRATEGY,
bidirectional=BIDIRECTIONAL,
device=torch.device(DEVICE),
)
classiier_encoding_dim = 2 * HIDDEN_DIMENSION if BIDIRECTIONAL else HIDDEN_DIMENSION
model = SimpleClassifier(
encoder=encoder,
encoding_dim=classiier_encoding_dim,
num_classes=NUM_CLASSES,
classification_layer_bias=True,
)
optimizer = optim.Adam(params=model.parameters(), lr=LEARNING_RATE)
metric = PrecisionRecallFMeasure(train_dataset.idx2classname)
engine = Engine(
model=model,
train_dataset=train_dataset,
validation_dataset=validation_dataset,
test_dataset=test_dataset,
optimizer=optimizer,
batch_size=BATCH_SIZE,
save_dir=MODEL_SAVE_DIR,
num_epochs=NUM_EPOCHS,
save_every=SAVE_EVERY,
log_train_metrics_every=LOG_TRAIN_METRICS_EVERY,
device=torch.device(DEVICE),
metric=metric,
use_wandb=True,
def setup_engine_test_with_simple_classifier(request, tmpdir_factory):
MAX_NUM_WORDS = 1000
MAX_LENGTH = 50
vocab_store_location = tmpdir_factory.mktemp("tempdir").join("vocab.json")
DEBUG = True
BATCH_SIZE = 1
NUM_TOKENS = 3
EMB_DIM = 300
train_dataset = SectLabelDataset(
filename=SECT_LABEL_FILE,
dataset_type="train",
max_num_words=MAX_NUM_WORDS,
max_instance_length=MAX_LENGTH,
word_vocab_store_location=vocab_store_location,
debug=DEBUG,
word_embedding_type="random",
word_embedding_dimension=EMB_DIM,
)
validation_dataset = SectLabelDataset(
filename=SECT_LABEL_FILE,
dataset_type="valid",
max_num_words=MAX_NUM_WORDS,
max_instance_length=MAX_LENGTH,
word_vocab_store_location=vocab_store_location,
debug=DEBUG,
word_embedding_type="random",
word_embedding_dimension=EMB_DIM,
)
test_dataset = SectLabelDataset(
filename=SECT_LABEL_FILE,
dataset_type="test",
max_num_words=MAX_NUM_WORDS,
max_instance_length=MAX_LENGTH,
word_vocab_store_location=vocab_store_location,
debug=DEBUG,
word_embedding_type="random",
word_embedding_dimension=EMB_DIM,
)
VOCAB_SIZE = MAX_NUM_WORDS + len(train_dataset.word_vocab.special_vocab)
NUM_CLASSES = train_dataset.get_num_classes()
NUM_EPOCHS = 1
embedding = Embedding.from_pretrained(torch.zeros([VOCAB_SIZE, EMB_DIM]))
labels = torch.LongTensor([1])
metric = PrecisionRecallFMeasure(
idx2labelname_mapping=train_dataset.idx2classname)
embedder = VanillaEmbedder(embedding_dim=EMB_DIM, embedding=embedding)
encoder = BOW_Encoder(emb_dim=EMB_DIM,
embedder=embedder,
dropout_value=0,
aggregation_type="sum")
tokens = np.random.randint(0,
VOCAB_SIZE - 1,
size=(BATCH_SIZE, NUM_TOKENS))
tokens = torch.LongTensor(tokens)
model = SimpleClassifier(
encoder=encoder,
encoding_dim=EMB_DIM,
num_classes=NUM_CLASSES,
classification_layer_bias=False,
)
optimizer = optim.SGD(model.parameters(), lr=0.01)
engine = Engine(
model,
train_dataset,
validation_dataset,
test_dataset,
optimizer=optimizer,
batch_size=BATCH_SIZE,
save_dir=tmpdir_factory.mktemp("model_save"),
num_epochs=NUM_EPOCHS,
save_every=1,
log_train_metrics_every=10,
metric=metric,
track_for_best=request.param,
)
options = {
"MAX_NUM_WORDS": MAX_NUM_WORDS,
"MAX_LENGTH": MAX_LENGTH,
"BATCH_SIZE": BATCH_SIZE,
"NUM_TOKENS": NUM_TOKENS,
"EMB_DIM": EMB_DIM,
"VOCAB_SIZE": VOCAB_SIZE,
"NUM_CLASSES": NUM_CLASSES,
"NUM_EPOCHS": NUM_EPOCHS,
}
return engine, tokens, labels, options