def test_iterative_train_test_split(tags, df):
# Process
df, tags_above_freq, tags_below_freq = data.prepare(df=df,
include=tags,
min_tag_freq=1)
df.text = df.text.apply(data.preprocess)
# Encode labels
labels = df.tags
label_encoder = data.MultiLabelLabelEncoder()
label_encoder.fit(labels)
y = label_encoder.encode(labels)
# Split data
X = df.text.to_numpy()
X_train, X_, y_train, y_ = data.iterative_train_test_split(X=X,
y=y,
train_size=0.7)
X_val, X_test, y_val, y_test = data.iterative_train_test_split(
X=X_, y=y_, train_size=0.5)
assert len(X_train) == len(y_train)
assert len(X_val) == len(y_val)
assert len(X_test) == len(y_test)
assert len(X_train) / float(len(X)) == pytest.approx(
0.7, abs=0.05) # 0.7 ± 0.05
assert len(X_val) / float(len(X)) == pytest.approx(0.15,
abs=0.05) # 0.15 ± 0.05
assert len(X_test) / float(len(X)) == pytest.approx(
0.15, abs=0.05) # 0.15 ± 0.05
class TestLabelEncoder:
@classmethod
def setup_class(cls):
"""Called before every class initialization."""
pass
@classmethod
def teardown_class(cls):
"""Called after every class initialization."""
pass
def setup_method(self):
"""Called before every method."""
self.label_encoder = data.LabelEncoder()
def teardown_method(self):
"""Called after every method."""
del self.label_encoder
def test_empty_init(self):
label_encoder = data.LabelEncoder()
assert label_encoder.index_to_class == {}
assert len(label_encoder.classes) == 0
def test_dict_init(self):
class_to_index = {"apple": 0, "banana": 1}
label_encoder = data.LabelEncoder(class_to_index=class_to_index)
assert label_encoder.index_to_class == {0: "apple", 1: "banana"}
assert len(label_encoder.classes) == 2
def test_len(self):
assert len(self.label_encoder) == 0
def test_save_and_load(self):
with tempfile.TemporaryDirectory() as dp:
fp = Path(dp, "label_encoder.json")
self.label_encoder.save(fp=fp)
label_encoder = data.LabelEncoder.load(fp=fp)
assert len(label_encoder.classes) == 0
@pytest.mark.parametrize(
"label_encoder, output",
[
(data.MultiClassLabelEncoder(), "<MultiClassLabelEncoder(num_classes=0)>"),
(data.MultiLabelLabelEncoder(), "<MultiLabelLabelEncoder(num_classes=0)>"),
],
)
def test_str(self, label_encoder, output):
assert str(label_encoder) == output
@pytest.mark.parametrize(
"label_encoder, y",
[
(data.MultiClassLabelEncoder(), ["apple", "apple", "banana"]),
(data.MultiLabelLabelEncoder(), [["apple"], ["apple", "banana"]]),
],
)
def test_fit(self, label_encoder, y):
label_encoder.fit(y)
assert "apple" in label_encoder.class_to_index
assert "banana" in label_encoder.class_to_index
assert len(label_encoder.classes) == 2
@pytest.mark.parametrize(
"label_encoder, y, y_encoded",
[
(
data.MultiClassLabelEncoder(class_to_index={"apple": 0, "banana": 1}),
["apple", "apple", "banana"],
[0, 0, 1],
),
(
data.MultiLabelLabelEncoder(class_to_index={"apple": 0, "banana": 1}),
[["apple"], ["apple", "banana"]],
[[1, 0], [1, 1]],
),
],
)
def test_encode_decode(self, label_encoder, y, y_encoded):
label_encoder.fit(y)
assert np.array_equal(label_encoder.encode(y), np.array(y_encoded))
assert label_encoder.decode(y_encoded) == y
def run(params: Namespace, trial: optuna.trial._trial.Trial = None) -> Dict:
"""Operations for training.
Args:
params (Namespace): Input parameters for operations.
trial (optuna.trial._trial.Trail, optional): Optuna optimization trial. Defaults to None.
Returns:
Artifacts to save and load for later.
"""
# 1. Set seed
utils.set_seed(seed=params.seed)
# 2. Set device
device = utils.set_device(cuda=params.cuda)
# 3. Load data
projects_fp = Path(config.DATA_DIR, "projects.json")
tags_fp = Path(config.DATA_DIR, "tags.json")
projects = utils.load_dict(filepath=projects_fp)
tags_dict = utils.list_to_dict(utils.load_dict(filepath=tags_fp),
key="tag")
df = pd.DataFrame(projects)
if params.shuffle:
df = df.sample(frac=1).reset_index(drop=True)
df = df[:params.subset] # None = all samples
# 4. Prepare data (feature engineering, filter, clean)
df, tags_above_freq, tags_below_freq = data.prepare(
df=df,
include=list(tags_dict.keys()),
exclude=config.EXCLUDED_TAGS,
min_tag_freq=params.min_tag_freq,
)
params.num_samples = len(df)
# 5. Preprocess data
df.text = df.text.apply(data.preprocess,
lower=params.lower,
stem=params.stem)
# 6. Encode labels
labels = df.tags
label_encoder = data.MultiLabelLabelEncoder()
label_encoder.fit(labels)
y = label_encoder.encode(labels)
# Class weights
all_tags = list(itertools.chain.from_iterable(labels.values))
counts = np.bincount(
[label_encoder.class_to_index[class_] for class_ in all_tags])
class_weights = {i: 1.0 / count for i, count in enumerate(counts)}
# 7. Split data
utils.set_seed(seed=params.seed) # needed for skmultilearn
X = df.text.to_numpy()
X_train, X_, y_train, y_ = data.iterative_train_test_split(
X=X, y=y, train_size=params.train_size)
X_val, X_test, y_val, y_test = data.iterative_train_test_split(
X=X_, y=y_, train_size=0.5)
test_df = pd.DataFrame({
"text": X_test,
"tags": label_encoder.decode(y_test)
})
# 8. Tokenize inputs
tokenizer = data.Tokenizer(char_level=params.char_level)
tokenizer.fit_on_texts(texts=X_train)
X_train = np.array(tokenizer.texts_to_sequences(X_train), dtype=object)
X_val = np.array(tokenizer.texts_to_sequences(X_val), dtype=object)
X_test = np.array(tokenizer.texts_to_sequences(X_test), dtype=object)
# 9. Create dataloaders
train_dataset = data.CNNTextDataset(X=X_train,
y=y_train,
max_filter_size=params.max_filter_size)
val_dataset = data.CNNTextDataset(X=X_val,
y=y_val,
max_filter_size=params.max_filter_size)
train_dataloader = train_dataset.create_dataloader(
batch_size=params.batch_size)
val_dataloader = val_dataset.create_dataloader(
batch_size=params.batch_size)
# 10. Initialize model
model = models.initialize_model(
params=params,
vocab_size=len(tokenizer),
num_classes=len(label_encoder),
device=device,
)
# 11. Train model
logger.info(
f"Parameters: {json.dumps(params.__dict__, indent=2, cls=NumpyEncoder)}"
)
params, model, loss = train.train(
params=params,
train_dataloader=train_dataloader,
val_dataloader=val_dataloader,
model=model,
device=device,
class_weights=class_weights,
trial=trial,
)
# 12. Evaluate model
artifacts = {
"params": params,
"label_encoder": label_encoder,
"tokenizer": tokenizer,
"model": model,
"loss": loss,
}
device = torch.device("cpu")
y_true, y_pred, performance = eval.evaluate(df=test_df,
artifacts=artifacts)
artifacts["performance"] = performance
return artifacts
df = df[:args.num_samples] # None = all samples
# 4. Clean data
df, tags_above_frequency = data.clean(
df=df,
include=list(tags_dict.keys()),
exclude=config.EXCLUDE,
min_tag_freq=args.min_tag_freq,
)
# 5. Preprocess data
df.text = df.text.apply(data.preprocess, lower=args.lower, stem=args.stem)
# 6. Encode labels
labels = df.tags
label_encoder = data.MultiLabelLabelEncoder()
label_encoder.fit(labels)
y = label_encoder.encode(labels)
# Class weights
all_tags = list(itertools.chain.from_iterable(labels.values))
counts = np.bincount(
[label_encoder.class_to_index[class_] for class_ in all_tags])
class_weights = {i: 1.0 / count for i, count in enumerate(counts)}
# 7. Split data
utils.set_seed(seed=args.seed) # needed for skmultilearn
X = df.text.to_numpy()
X_train, X_, y_train, y_ = data.iterative_train_test_split(
X=X, y=y, train_size=args.train_size)
X_val, X_test, y_val, y_test = data.iterative_train_test_split(
def train(params: Namespace, trial: optuna.trial._trial.Trial = None) -> Dict:
"""Operations for training.
Args:
params (Namespace): Input parameters for operations.
trial (optuna.trial._trial.Trail, optional): Optuna optimization trial. Defaults to None.
Returns:
Artifacts to save and load for later.
"""
# Set up
utils.set_seed(seed=params.seed)
device = utils.set_device(cuda=params.cuda)
# Load features
features_fp = Path(config.DATA_DIR, "features.json")
tags_fp = Path(config.DATA_DIR, "tags.json")
features = utils.load_dict(filepath=features_fp)
tags_dict = utils.list_to_dict(utils.load_dict(filepath=tags_fp),
key="tag")
df = pd.DataFrame(features)
if params.shuffle:
df = df.sample(frac=1).reset_index(drop=True)
df = df[:params.subset] # None = all samples
# Prepare data (filter, clean, etc.)
df, tags_above_freq, tags_below_freq = data.prepare(
df=df,
include=list(tags_dict.keys()),
exclude=config.EXCLUDED_TAGS,
min_tag_freq=params.min_tag_freq,
)
params.num_samples = len(df)
# Preprocess data
df.text = df.text.apply(data.preprocess,
lower=params.lower,
stem=params.stem)
# Encode labels
labels = df.tags
label_encoder = data.MultiLabelLabelEncoder()
label_encoder.fit(labels)
y = label_encoder.encode(labels)
# Class weights
all_tags = list(itertools.chain.from_iterable(labels.values))
counts = np.bincount(
[label_encoder.class_to_index[class_] for class_ in all_tags])
class_weights = {i: 1.0 / count for i, count in enumerate(counts)}
# Split data
utils.set_seed(seed=params.seed) # needed for skmultilearn
X = df.text.to_numpy()
X_train, X_, y_train, y_ = data.iterative_train_test_split(
X=X, y=y, train_size=params.train_size)
X_val, X_test, y_val, y_test = data.iterative_train_test_split(
X=X_, y=y_, train_size=0.5)
test_df = pd.DataFrame({
"text": X_test,
"tags": label_encoder.decode(y_test)
})
# Tokenize inputs
tokenizer = data.Tokenizer(char_level=params.char_level)
tokenizer.fit_on_texts(texts=X_train)
X_train = np.array(tokenizer.texts_to_sequences(X_train), dtype=object)
X_val = np.array(tokenizer.texts_to_sequences(X_val), dtype=object)
X_test = np.array(tokenizer.texts_to_sequences(X_test), dtype=object)
# Create dataloaders
train_dataset = data.CNNTextDataset(X=X_train,
y=y_train,
max_filter_size=params.max_filter_size)
val_dataset = data.CNNTextDataset(X=X_val,
y=y_val,
max_filter_size=params.max_filter_size)
train_dataloader = train_dataset.create_dataloader(
batch_size=params.batch_size)
val_dataloader = val_dataset.create_dataloader(
batch_size=params.batch_size)
# Initialize model
model = models.initialize_model(
params=params,
vocab_size=len(tokenizer),
num_classes=len(label_encoder),
device=device,
)
# Train model
logger.info(
f"Parameters: {json.dumps(params.__dict__, indent=2, cls=NumpyEncoder)}"
)
class_weights_tensor = torch.Tensor(np.array(list(class_weights.values())))
loss_fn = nn.BCEWithLogitsLoss(weight=class_weights_tensor)
optimizer = torch.optim.Adam(model.parameters(), lr=params.lr)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode="min",
factor=0.05,
patience=5)
# Trainer module
trainer = Trainer(
model=model,
device=device,
loss_fn=loss_fn,
optimizer=optimizer,
scheduler=scheduler,
trial=trial,
)
# Train
best_val_loss, best_model = trainer.train(params.num_epochs,
params.patience,
train_dataloader, val_dataloader)
# Find best threshold
_, y_true, y_prob = trainer.eval_step(dataloader=train_dataloader)
params.threshold = find_best_threshold(y_true=y_true, y_prob=y_prob)
# Evaluate model
artifacts = {
"params": params,
"label_encoder": label_encoder,
"tokenizer": tokenizer,
"model": best_model,
"loss": best_val_loss,
}
device = torch.device("cpu")
y_true, y_pred, performance = eval.evaluate(df=test_df,
artifacts=artifacts)
artifacts["performance"] = performance
return artifacts