def build_sectlabel_bow_elmo_model(dirname: str):
exp_dirpath = pathlib.Path(dirname)
DATA_PATH = pathlib.Path(DATA_DIR)
train_file = DATA_PATH.joinpath("sectLabel.train")
dev_file = DATA_PATH.joinpath("sectLabel.dev")
test_file = DATA_PATH.joinpath("sectLabel.test")
data_manager = TextClassificationDatasetManager(
train_filename=str(train_file),
dev_filename=str(dev_file),
test_filename=str(test_file),
)
embedder = BowElmoEmbedder(layer_aggregation="last")
encoder = BOW_Encoder(aggregation_type="sum", embedder=embedder)
model = SimpleClassifier(
encoder=encoder,
encoding_dim=1024,
num_classes=data_manager.num_labels["label"],
classification_layer_bias=True,
datasets_manager=data_manager,
)
infer_client = ClassificationInference(
model=model,
model_filepath=str(exp_dirpath.joinpath("checkpoints",
"best_model.pt")),
datasets_manager=data_manager,
)
return infer_client
def _get_data(self):
data_manager = TextClassificationDatasetManager(
train_filename=self.data_dir.joinpath("scicite.train"),
dev_filename=self.data_dir.joinpath("scicite.dev"),
test_filename=self.data_dir.joinpath("scicite.test"),
)
return data_manager
def build_dataset(self):
train_filename = DATA_DIR.joinpath("genericSect.train")
dev_filename = DATA_DIR.joinpath("genericSect.dev")
test_filename = DATA_DIR.joinpath("genericSect.test")
data_manager = TextClassificationDatasetManager(
train_filename=train_filename,
dev_filename=dev_filename,
test_filename=test_filename,
)
return data_manager
def _get_data(self):
train_filename = self.data_dir.joinpath("sectLabel.train")
dev_filename = self.data_dir.joinpath("sectLabel.dev")
test_filename = self.data_dir.joinpath("sectLabel.test")
data_manager = TextClassificationDatasetManager(
train_filename=train_filename,
dev_filename=dev_filename,
test_filename=test_filename,
)
return data_manager
def build_sectlabel_elmobilstm_model(dirname: str):
exp_dirpath = pathlib.Path(dirname)
DATA_PATH = pathlib.Path(DATA_DIR)
train_file = DATA_PATH.joinpath("sectLabel.train")
dev_file = DATA_PATH.joinpath("sectLabel.dev")
test_file = DATA_PATH.joinpath("sectLabel.test")
data_manager = TextClassificationDatasetManager(
train_filename=str(train_file),
dev_filename=str(dev_file),
test_filename=str(test_file),
)
DEVICE = "cpu"
EMBEDDING_TYPE = "glove_6B_50"
HIDDEN_DIM = 512
BIDIRECTIONAL = True
COMBINE_STRATEGY = "concat"
elmo_embedder = BowElmoEmbedder(cuda_device_id=-1 if DEVICE ==
"cpu" else int(DEVICE.split("cuda:")[1]))
vanilla_embedder = WordEmbedder(embedding_type=EMBEDDING_TYPE)
embedders = ConcatEmbedders([vanilla_embedder, elmo_embedder])
encoder = LSTM2VecEncoder(
embedder=embedders,
hidden_dim=HIDDEN_DIM,
bidirectional=BIDIRECTIONAL,
combine_strategy=COMBINE_STRATEGY,
device=torch.device(DEVICE),
)
encoding_dim = (2 * HIDDEN_DIM if BIDIRECTIONAL
and COMBINE_STRATEGY == "concat" else HIDDEN_DIM)
model = SimpleClassifier(
encoder=encoder,
encoding_dim=encoding_dim,
num_classes=23,
classification_layer_bias=True,
datasets_manager=data_manager,
)
inference = ClassificationInference(
model=model,
model_filepath=str(exp_dirpath.joinpath("checkpoints",
"best_model.pt")),
datasets_manager=data_manager,
)
return inference
def setup_sectlabel_dataset_manager():
data_dir = pathlib.Path(DATA_DIR)
sect_label_train_file = data_dir.joinpath("sectLabel.train")
sect_label_dev_file = data_dir.joinpath("sectLabel.dev")
sect_label_test_file = data_dir.joinpath("sectLabel.test")
dataset_manager = TextClassificationDatasetManager(
train_filename=sect_label_train_file,
dev_filename=sect_label_dev_file,
test_filename=sect_label_test_file,
)
return dataset_manager
def build_dataset(self):
# train_file = DATA_PATH.joinpath("coda-19.train")
# dev_file = DATA_PATH.joinpath("coda-19.dev")
# test_file = DATA_PATH.joinpath("coda-19.test")
train_filename = "./backend/abstract_tagging/sciwing/coda19_classification_elmo_slower/data/coda-19.train"
dev_filename = "./backend/abstract_tagging/sciwing/coda19_classification_elmo_slower/data/coda-19.dev"
test_filename = "./backend/abstract_tagging/sciwing/coda19_classification_elmo_slower/data/coda-19.test"
data_manager = TextClassificationDatasetManager(
train_filename=train_filename,
dev_filename=dev_filename,
test_filename=test_filename,
)
return data_manager
def clf_dataset_manager(tmpdir_factory, request):
train_file = tmpdir_factory.mktemp("train_data").join("train_file.txt")
train_file.write("train_line1###label1\ntrain_line2###label2")
dev_file = tmpdir_factory.mktemp("dev_data").join("dev_file.txt")
dev_file.write("dev_line1###label1\ndev_line2###label2")
test_file = tmpdir_factory.mktemp("test_data").join("test_file.txt")
test_file.write("dev_line1###label1\ndev_line2###label2")
clf_dataset_manager = TextClassificationDatasetManager(
train_filename=str(train_file),
dev_filename=str(dev_file),
test_filename=str(test_file),
)
return clf_dataset_manager
def build_sectlabel_bilstm_model(dirname: str):
exp_dirpath = pathlib.Path(dirname)
DATA_PATH = pathlib.Path(DATA_DIR)
train_file = DATA_PATH.joinpath("sectLabel.train")
dev_file = DATA_PATH.joinpath("sectLabel.dev")
test_file = DATA_PATH.joinpath("sectLabel.test")
data_manager = TextClassificationDatasetManager(
train_filename=str(train_file),
dev_filename=str(dev_file),
test_filename=str(test_file),
)
HIDDEN_DIM = 512
BIDIRECTIONAL = True
COMBINE_STRATEGY = "concat"
classifier_encoding_dim = 2 * HIDDEN_DIM if BIDIRECTIONAL else HIDDEN_DIM
embedder = WordEmbedder(embedding_type="glove_6B_50")
encoder = LSTM2VecEncoder(
embedder=embedder,
hidden_dim=HIDDEN_DIM,
combine_strategy=COMBINE_STRATEGY,
bidirectional=BIDIRECTIONAL,
)
model = SimpleClassifier(
encoder=encoder,
encoding_dim=classifier_encoding_dim,
num_classes=23,
classification_layer_bias=True,
datasets_manager=data_manager,
)
inference = ClassificationInference(
model=model,
model_filepath=str(exp_dirpath.joinpath("checkpoints", "best_model.pt")),
datasets_manager=data_manager,
)
return inference
def build_sectlabel_bow_model(dirname: str):
"""
Parameters
----------
dirname : The directory where sciwing stores your outputs for the model
Returns
-------
"""
exp_dirpath = pathlib.Path(dirname)
DATA_PATH = pathlib.Path(DATA_DIR)
train_file = DATA_PATH.joinpath("sectLabel.train")
dev_file = DATA_PATH.joinpath("sectLabel.dev")
test_file = DATA_PATH.joinpath("sectLabel.test")
data_manager = TextClassificationDatasetManager(
train_filename=str(train_file),
dev_filename=str(dev_file),
test_filename=str(test_file),
)
embedder = WordEmbedder(embedding_type="glove_6B_50")
encoder = BOW_Encoder(embedder=embedder)
model = SimpleClassifier(
encoder=encoder,
encoding_dim=embedder.get_embedding_dimension(),
num_classes=data_manager.num_labels["label"],
classification_layer_bias=True,
datasets_manager=data_manager,
)
infer = ClassificationInference(
model=model,
model_filepath=str(exp_dirpath.joinpath("checkpoints",
"best_model.pt")),
datasets_manager=data_manager,
)
return infer
def _get_data(self):
train_file = cached_path(
path=self.data_dir.joinpath("scicite.train"),
url=self.train_data_url,
unzip=False,
)
dev_file = cached_path(
path=self.data_dir.joinpath("scicite.dev"),
url=self.dev_data_url,
unzip=False,
)
test_file = cached_path(
path=self.data_dir.joinpath("scicite.test"),
url=self.test_data_url,
unzip=False,
)
data_manager = TextClassificationDatasetManager(
train_filename=train_file,
dev_filename=dev_file,
test_filename=test_file)
return data_manager
def build_sectlabel_bow_bert(dirname: str):
exp_dirpath = pathlib.Path(dirname)
DATA_PATH = pathlib.Path(DATA_DIR)
train_file = DATA_PATH.joinpath("sectLabel.train")
dev_file = DATA_PATH.joinpath("sectLabel.dev")
test_file = DATA_PATH.joinpath("sectLabel.test")
data_manager = TextClassificationDatasetManager(
train_filename=str(train_file),
dev_filename=str(dev_file),
test_filename=str(test_file),
)
embedder = BertEmbedder(
dropout_value=0.0,
aggregation_type="average",
bert_type="bert-base-uncased",
device=torch.device("cpu"),
)
encoder = BOW_Encoder(embedder=embedder, aggregation_type="average")
model = SimpleClassifier(
encoder=encoder,
encoding_dim=768,
num_classes=23,
classification_layer_bias=True,
datasets_manager=data_manager,
)
parsect_inference = ClassificationInference(
model=model,
model_filepath=str(exp_dirpath.joinpath("checkpoints",
"best_model.pt")),
datasets_manager=data_manager,
)
return parsect_inference
def _get_data(self):
train_filename = self.data_dir.joinpath("sectLabel.train")
dev_filename = self.data_dir.joinpath("sectLabel.dev")
test_filename = self.data_dir.joinpath("sectLabel.test")
train_filename = cached_path(path=train_filename,
url=self.train_data_url,
unzip=False)
dev_filename = cached_path(path=dev_filename,
url=self.dev_data_url,
unzip=False)
test_filename = cached_path(path=test_filename,
url=self.test_data_url,
unzip=False)
data_manager = TextClassificationDatasetManager(
train_filename=train_filename,
dev_filename=dev_filename,
test_filename=test_filename,
)
return data_manager
parser.add_argument("--vocab_store_location",
help="File in which the vocab is stored")
parser.add_argument("--sample_proportion",
help="Sample proportion for debugging",
type=float)
args = parser.parse_args()
DATA_PATH = pathlib.Path(DATA_DIR)
train_file = DATA_PATH.joinpath("sectLabel.train")
dev_file = DATA_PATH.joinpath("sectLabel.dev")
test_file = DATA_PATH.joinpath("sectLabel.test")
data_manager = TextClassificationDatasetManager(
train_filename=str(train_file),
dev_filename=str(dev_file),
test_filename=str(test_file),
)
# BowElmoEmbedder embeds sentences using ELMO
embedder = BowElmoEmbedder(layer_aggregation=args.layer_aggregation,
device=args.device)
encoder = BOW_Encoder(embedder=embedder,
aggregation_type=args.word_aggregation,
device=args.device)
model = SimpleClassifier(
encoder=encoder,
encoding_dim=1024,
num_classes=data_manager.num_labels["label"],
"Directory where the checkpoints during model training are stored.",
)
parser.add_argument("--sample_proportion",
help="Sample data size",
type=float)
args = parser.parse_args()
DATA_DIR = pathlib.Path(DATA_DIR)
train_filename = DATA_DIR.joinpath("genericSect.train")
dev_filename = DATA_DIR.joinpath("genericSect.dev")
test_filename = DATA_DIR.joinpath("genericSect.test")
data_manager = TextClassificationDatasetManager(
train_filename=train_filename,
dev_filename=dev_filename,
test_filename=test_filename,
)
embedder = BowElmoEmbedder(layer_aggregation=args.layer_aggregation,
device=args.device)
encoder = BOW_Encoder(aggregation_type=args.word_aggregation,
embedder=embedder,
device=args.device)
model = SimpleClassifier(
encoder=encoder,
encoding_dim=1024,
num_classes=12,
classification_layer_bias=True,