def score(
self,
data,
metric="accuracy",
break_ties="random",
verbose=True,
print_confusion_matrix=True,
**kwargs,
):
"""Scores the predictive performance of the Classifier on all tasks
Args:
data: a Pytorch DataLoader, Dataset, or tuple with Tensors (X,Y):
X: The input for the predict method
Y: An [n] or [n, 1] torch.Tensor or np.ndarray of target labels
in {1,...,k}
metric: A metric (string) with which to score performance or a
list of such metrics
break_ties: A tie-breaking policy (see Classifier._break_ties())
verbose: The verbosity for just this score method; it will not
update the class config.
print_confusion_matrix: Print confusion matrix (overwritten to False if
verbose=False)
Returns:
scores: A (float) score or a list of such scores if kwarg metric
is a list
"""
Y_p, Y, Y_s = self._get_predictions(data,
break_ties=break_ties,
return_probs=True,
**kwargs)
# Evaluate on the specified metrics
return_list = isinstance(metric, list)
metric_list = metric if isinstance(metric, list) else [metric]
scores = []
for metric in metric_list:
score = metric_score(Y, Y_p, metric, probs=Y_s, ignore_in_gold=[0])
scores.append(score)
if verbose:
if type(score) != list:
print(f"{metric.capitalize()}: {score:.7f}")
else:
print(f"{metric.capitalize()}: {score}")
# Optionally print confusion matrix
if print_confusion_matrix and verbose:
confusion_matrix(Y, Y_p, pretty_print=True)
# If a single metric was given as a string (not list), return a float
if len(scores) == 1 and not return_list:
return scores[0]
else:
return scores
plot_title="Disease Associates Gene Dev PRC",
metric='PR',
font_size=16)
# In[21]:
label_model = LabelModel(k=2, seed=100)
label_model.train_model(validation_data[1][0],
n_epochs=1000,
verbose=False,
lr=0.01,
l2=2.067)
dev_predictions = convert_labels(label_model.predict(validation_data[1][1]),
'categorical', 'onezero')
dev_marginals = label_model.predict_proba(validation_data[1][1])[:, 0]
# In[22]:
plt.rcParams.update({'font.size': 16})
plt.figure(figsize=(10, 6))
plot_predictions_histogram(dev_predictions,
candidate_dfs['dev'].curated_dsh.astype(int).values,
title="Prediction Histogram for Dev Set")
# In[23]:
confusion_matrix(
convert_labels(candidate_dfs['dev'].curated_dsh.values, 'onezero',
'categorical'),
convert_labels(dev_predictions, 'onezero', 'categorical'))
def _train(self, train_loader, loss_fn, X_dev=None, Y_dev=None):
"""The internal training routine called by train() after initial setup
Args:
train_loader: a torch DataLoader of X (data) and Y (labels) for
the train split
loss_fn: the loss function to minimize (maps *data -> loss)
X_dev: the dev set model input
Y_dev: the dev set target labels
If either of X_dev or Y_dev is not provided, then no checkpointing or
evaluation on the dev set will occur.
"""
train_config = self.config["train_config"]
evaluate_dev = X_dev is not None and Y_dev is not None
# Set the optimizer
optimizer_config = train_config["optimizer_config"]
optimizer = self._set_optimizer(optimizer_config)
# Set the lr scheduler
scheduler_config = train_config["scheduler_config"]
lr_scheduler = self._set_scheduler(scheduler_config, optimizer)
# Create the checkpointer if applicable
if evaluate_dev and train_config["checkpoint"]:
checkpoint_config = train_config["checkpoint_config"]
model_class = type(self).__name__
checkpointer = Checkpointer(
model_class, **checkpoint_config, verbose=self.config["verbose"]
)
# Train the model
for epoch in range(train_config["n_epochs"]):
epoch_loss = 0.0
for data in train_loader:
# Zero the parameter gradients
optimizer.zero_grad()
# Forward pass to calculate outputs
loss = loss_fn(*data)
if torch.isnan(loss):
msg = "Loss is NaN. Consider reducing learning rate."
raise Exception(msg)
# Backward pass to calculate gradients
loss.backward()
# TODO: restore this once it has unit tests
# Clip gradients
# if grad_clip:
# torch.nn.utils.clip_grad_norm(
# self.net.parameters(), grad_clip)
# Perform optimizer step
optimizer.step()
# Keep running sum of losses
epoch_loss += loss.detach()
# Calculate average loss per training example
# Saving division until this stage protects against the potential
# mistake of averaging batch losses when the last batch is an orphan
train_loss = epoch_loss / len(train_loader.dataset)
# Checkpoint performance on dev
if evaluate_dev and (epoch % train_config["validation_freq"] == 0):
val_metric = train_config["validation_metric"]
dev_score = self.score(
X_dev, Y_dev, metric=val_metric, verbose=False
)
if train_config["checkpoint"]:
checkpointer.checkpoint(self, epoch, dev_score)
# Apply learning rate scheduler
if (
lr_scheduler is not None
and epoch + 1 >= scheduler_config["lr_freeze"]
):
if scheduler_config["scheduler"] == "reduce_on_plateau":
if evaluate_dev:
lr_scheduler.step(dev_score)
else:
lr_scheduler.step()
# Report progress
if self.config["verbose"] and (
epoch % train_config["print_every"] == 0
or epoch == train_config["n_epochs"] - 1
):
msg = f"[E:{epoch}]\tTrain Loss: {train_loss:.3f}"
if evaluate_dev:
msg += f"\tDev score: {dev_score:.3f}"
print(msg)
# Restore best model if applicable
if evaluate_dev and train_config["checkpoint"]:
checkpointer.restore(model=self)
if self.config["verbose"]:
print("Finished Training")
if evaluate_dev:
Y_p_dev = self.predict(X_dev)
if not self.multitask:
print("Confusion Matrix (Dev)")
confusion_matrix(Y_p_dev, Y_dev, pretty_print=True)
# baseline majority:
"""
print(L_test)
Y_baseline = []
for row in L_test:
print(row)
Y_baseline.append(Counter(row).most_common()[0][0])
print(np.asarray(Y_baseline))
"""
Y_tes = label_model.predict(Ls[2])
pickling_on = open("data_encompassing/ar/ar_{}{}".format(flag0, flag), "wb")
print(Y_tes, type(Y_tes))
pickle.dump(Y_tes, pickling_on)
cm = confusion_matrix(Ys[1], Y_dev_p)
try:
from metal.contrib.visualization.analysis import (
plot_predictions_histogram,
plot_probabilities_histogram,
)
plot_predictions_histogram(Y_tes, Ys[2], title="Label Distribution")
Y_dev_ps = label_model.predict_proba(Ls[2])
plot_probabilities_histogram(Y_dev_ps[:, 0],
title="Probablistic Label Distribution")
except ModuleNotFoundError:
print(
def train(self, X_train, Y_train, X_dev=None, Y_dev=None, **kwargs):
self.config = recursive_merge_dicts(self.config, kwargs)
train_config = self.config['train_config']
Y_train = self._to_torch(Y_train)
Y_dev = self._to_torch(Y_dev)
if train_config['use_cuda']:
raise NotImplementedError
# TODO: fix this
# X = X.cuda(self.gpu_id)
# Y = Y.cuda(self.gpu_id)
# TODO: put model on gpu
# Make data loaders
loader_config = train_config['data_loader_config']
train_loader = self._make_data_loader(X_train, Y_train, loader_config)
evaluate_dev = (X_dev is not None and Y_dev is not None)
# Set the optimizer
optimizer_config = train_config['optimizer_config']
optimizer = self._set_optimizer(optimizer_config)
# Set the lr scheduler
scheduler_config = train_config['scheduler_config']
lr_scheduler = self._set_scheduler(scheduler_config, optimizer)
# Initialize the model
self.reset()
# Train the model
for epoch in range(train_config['n_epochs']):
epoch_loss = 0.0
for i, data in enumerate(train_loader):
X, Y = data
# Zero the parameter gradients
optimizer.zero_grad()
# Forward pass to calculate outputs
output = self.forward(X)
loss = self._get_loss(output, Y)
# Backward pass to calculate gradients
loss.backward()
# Clip gradients
# if grad_clip:
# torch.nn.utils.clip_grad_norm(
# self.net.parameters(), grad_clip)
# Perform optimizer step
optimizer.step()
# Keep running sum of losses
epoch_loss += loss.detach() * X.shape[0]
# Calculate average loss per training example
# Saving division until this stage protects against the potential
# mistake of averaging batch losses when the last batch is an orphan
train_loss = epoch_loss / len(train_loader.dataset)
if evaluate_dev:
val_metric = train_config['validation_metric']
dev_score = self.score(X_dev, Y_dev, metric=val_metric,
verbose=False)
# Apply learning rate scheduler
if (lr_scheduler is not None
and epoch + 1 >= scheduler_config['lr_freeze']):
if scheduler_config['scheduler'] == 'reduce_on_plateau':
if evaluate_dev:
lr_scheduler.step(dev_score)
else:
lr_scheduler.step()
# Report progress
if (self.config['verbose'] and
(epoch % train_config['print_every'] == 0
or epoch == train_config['n_epochs'] - 1)):
msg = f'[E:{epoch+1}]\tTrain Loss: {train_loss:.3f}'
if evaluate_dev:
msg += f'\tDev score: {dev_score:.3f}'
print(msg)
if self.config['verbose']:
print('Finished Training')
if self.config['show_plots']:
if self.k == 2:
Y_p_train = self.predict_proba(X_train)
plot_probabilities_histogram(Y_p_train[:, 0],
title="Training Set Predictions")
if X_dev is not None and Y_dev is not None:
Y_ph_dev = self.predict(X_dev)
print("Confusion Matrix (Dev)")
mat = confusion_matrix(Y_ph_dev, Y_dev, pretty_print=True)