test_encodings = self.tokenizer(train_texts,
truncation=True,
padding=True)
# Create IMDB dataset
self.data_test = IMDBTransformerDataset(test_encodings, test_labels)
train_ds = IMDBTransformerDataset(train_encodings, train_labels)
self.data_train, self.data_val = random_split(train_ds, [20000, 5000])
def __repr__(self) -> str:
"""Print infor about the dataset"""
basic = f"IMDB Dataset\nNum classes: {len(self.mapping)}\nMapping: {self.mapping}\n"
data = (
f"Train/val/test sizes: {len(self.data_train), len(self.data_val), len(self.data_test)}"
)
return basic + data
def read_imdb_split(split_dir):
split_dir = Path(split_dir)
texts = []
labels = []
for label_dir in ["pos", "neg"]:
for text_file in (split_dir / label_dir).iterdir():
texts.append(text_file.read_text())
labels.append(0 if label_dir is "neg" else 1)
return texts, labels
if __name__ == "__main__":
load_and_print_info(IMDBTransformer)
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
train_encodings = tokenizer(train_texts, truncation=True, padding=True)
test_encodings = tokenizer(train_texts, truncation=True, padding=True)
# Create IMDB dataset
self.data_test = IMDBDataset(test_encodings, test_labels)
train_ds = IMDBDataset(train_encodings, train_labels)
self.data_train, self.data_val = random_split(train_ds, [20000, 5000])
def __repr__(self) -> str:
"""Print infor about the dataset"""
basic = (
f"IMDB Dataset\nNum classes: {len(self.mapping)}\nMapping: {self.mapping}\n"
)
data = f"Train/val/test sizes: {len(self.data_train), len(self.data_val), len(self.data_test)}"
return basic + data
def read_imdb_split(split_dir):
split_dir = Path(split_dir)
texts = []
labels = []
for label_dir in ["pos", "neg"]:
for text_file in (split_dir / label_dir).iterdir():
texts.append(text_file.read_text())
labels.append(0 if label_dir is "neg" else 1)
return texts, labels
if __name__ == "__main__":
load_and_print_info(IMDB)
val_df["labels"])
test_texts, test_labels = list(test_df["sentences"]), list(
test_df["labels"])
train_encodings = self.tokenizer(train_texts,
truncation=True,
padding=True)
val_encodings = self.tokenizer(val_texts,
truncation=True,
padding=True)
test_encodings = self.tokenizer(train_texts,
truncation=True,
padding=True)
# Create IMDB dataset
self.data_train = UIT_VSFCTranformerDataset(train_encodings,
train_labels)
self.data_val = UIT_VSFCTranformerDataset(val_encodings, val_labels)
self.data_test = UIT_VSFCTranformerDataset(test_encodings, test_labels)
def __repr__(self) -> str:
"""Print infor about the dataset"""
basic = f"UIT-VSFC Dataset\nNum classes: {len(self.mapping)}\nMapping: {self.mapping}\n"
data = (
f"Train/val/test sizes: {len(self.data_train), len(self.data_val), len(self.data_test)}"
)
return basic + data
if __name__ == "__main__":
load_and_print_info(UIT_VSFCTransformer)
