def get_callbacks(self, stage: str):
callbacks = {
"criterion":
dl.CriterionCallback(metric_key="loss",
input_key="logits",
target_key="targets"),
"optimizer":
dl.OptimizerCallback(
metric_key="loss",
grad_clip_fn=nn.utils.clip_grad_norm_,
grad_clip_params={"max_norm": 1.0},
),
# "scheduler": dl.SchedulerCallback(loader_key="valid", metric_key="loss"),
"accuracy":
dl.AccuracyCallback(input_key="logits",
target_key="targets",
topk_args=(1, 3, 5)),
"classification":
dl.PrecisionRecallF1SupportCallback(input_key="logits",
target_key="targets",
num_classes=10),
"checkpoint":
dl.CheckpointCallback(self._logdir,
loader_key="valid",
metric_key="loss",
minimize=True,
save_n_best=3),
}
if SETTINGS.ml_required:
callbacks["confusion_matrix"] = dl.ConfusionMatrixCallback(
input_key="logits", target_key="targets", num_classes=10)
return callbacks
def get_callbacks(self, stage: str):
return {
"criterion":
dl.CriterionCallback(metric_key="loss",
input_key="logits",
target_key="targets"),
"optimizer":
dl.OptimizerCallback(metric_key="loss"),
# "scheduler": dl.SchedulerCallback(loader_key="valid", metric_key="loss"),
"accuracy":
dl.AccuracyCallback(input_key="logits",
target_key="targets",
topk_args=(1, 3, 5)),
"classification":
dl.PrecisionRecallF1SupportCallback(input_key="logits",
target_key="targets",
num_classes=10),
"confusion_matrix":
dl.ConfusionMatrixCallback(input_key="logits",
target_key="targets",
num_classes=10),
"checkpoint":
dl.CheckpointCallback(self._logdir,
loader_key="valid",
metric_key="loss",
minimize=True,
save_n_best=3),
}
def train_experiment(device, engine=None):
with TemporaryDirectory() as logdir:
# sample data
num_samples, num_features, num_classes = int(1e4), int(1e1), 4
X = torch.rand(num_samples, num_features)
y = (torch.rand(num_samples, ) * num_classes).to(torch.int64)
# pytorch loaders
dataset = TensorDataset(X, y)
loader = DataLoader(dataset, batch_size=32, num_workers=1)
loaders = {"train": loader, "valid": loader}
# model, criterion, optimizer, scheduler
model = torch.nn.Linear(num_features, num_classes)
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters())
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, [2])
# model training
runner = dl.SupervisedRunner(input_key="features",
output_key="logits",
target_key="targets",
loss_key="loss")
callbacks = [
dl.AccuracyCallback(input_key="logits",
target_key="targets",
num_classes=num_classes),
dl.PrecisionRecallF1SupportCallback(input_key="logits",
target_key="targets",
num_classes=4),
]
if SETTINGS.ml_required:
callbacks.append(
dl.ConfusionMatrixCallback(input_key="logits",
target_key="targets",
num_classes=4))
if SETTINGS.amp_required and (engine is None or not isinstance(
engine,
(dl.AMPEngine, dl.DataParallelAMPEngine,
dl.DistributedDataParallelAMPEngine),
)):
callbacks.append(
dl.AUCCallback(input_key="logits", target_key="targets"))
runner.train(
engine=engine or dl.DeviceEngine(device),
model=model,
criterion=criterion,
optimizer=optimizer,
scheduler=scheduler,
loaders=loaders,
logdir=logdir,
num_epochs=1,
valid_loader="valid",
valid_metric="accuracy03",
minimize_valid_metric=False,
verbose=False,
callbacks=callbacks,
)
def get_callbacks(self, stage: str):
callbacks = {
"scores":
dl.BatchTransformCallback(
input_key="logits",
output_key="scores",
transform=partial(torch.softmax, dim=1),
scope="on_batch_end",
),
"labels":
dl.BatchTransformCallback(
input_key="scores",
output_key="labels",
transform=partial(torch.argmax, dim=1),
scope="on_batch_end",
),
"criterion":
dl.CriterionCallback(metric_key="loss",
input_key="logits",
target_key="targets"),
"optimizer":
dl.OptimizerCallback(
metric_key="loss",
grad_clip_fn=nn.utils.clip_grad_norm_,
grad_clip_params={"max_norm": 1.0},
),
# "scheduler": dl.SchedulerCallback(loader_key="valid", metric_key="loss"),
"accuracy":
dl.AccuracyCallback(input_key="logits",
target_key="targets",
topk_args=(1, 3, 5)),
"classification":
dl.PrecisionRecallF1SupportCallback(input_key="logits",
target_key="targets",
num_classes=10),
"checkpoint":
dl.CheckpointCallback(self._logdir,
loader_key="valid",
metric_key="loss",
minimize=True,
save_n_best=3),
}
if SETTINGS.ml_required:
callbacks["confusion_matrix"] = dl.ConfusionMatrixCallback(
input_key="logits", target_key="targets", num_classes=10)
callbacks["f1_score"] = dl.SklearnBatchCallback(
keys={
"y_pred": "labels",
"y_true": "targets"
},
metric_fn="f1_score",
metric_key="sk_f1",
average="macro",
zero_division=1,
)
return callbacks
def train_experiment(device, engine=None):
with TemporaryDirectory() as logdir:
model = nn.Sequential(nn.Flatten(), nn.Linear(28 * 28, 10))
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.02)
loaders = {
"train":
DataLoader(MNIST(os.getcwd(),
train=False,
download=True,
transform=ToTensor()),
batch_size=32),
"valid":
DataLoader(MNIST(os.getcwd(),
train=False,
download=True,
transform=ToTensor()),
batch_size=32),
}
runner = dl.SupervisedRunner(input_key="features",
output_key="logits",
target_key="targets",
loss_key="loss")
callbacks = [
dl.AccuracyCallback(input_key="logits",
target_key="targets",
topk_args=(1, 3, 5)),
dl.PrecisionRecallF1SupportCallback(input_key="logits",
target_key="targets",
num_classes=10),
]
if SETTINGS.ml_required:
callbacks.append(
dl.ConfusionMatrixCallback(input_key="logits",
target_key="targets",
num_classes=10))
if SETTINGS.amp_required and (engine is None or not isinstance(
engine,
(dl.AMPEngine, dl.DataParallelAMPEngine,
dl.DistributedDataParallelAMPEngine),
)):
callbacks.append(
dl.AUCCallback(input_key="logits", target_key="targets"))
if SETTINGS.onnx_required:
callbacks.append(
dl.OnnxCallback(logdir=logdir, input_key="features"))
if SETTINGS.pruning_required:
callbacks.append(
dl.PruningCallback(pruning_fn="l1_unstructured", amount=0.5))
if SETTINGS.quantization_required:
callbacks.append(dl.QuantizationCallback(logdir=logdir))
if engine is None or not isinstance(engine,
dl.DistributedDataParallelEngine):
callbacks.append(
dl.TracingCallback(logdir=logdir, input_key="features"))
# model training
runner.train(
engine=engine or dl.DeviceEngine(device),
model=model,
criterion=criterion,
optimizer=optimizer,
loaders=loaders,
num_epochs=1,
callbacks=callbacks,
logdir=logdir,
valid_loader="valid",
valid_metric="loss",
minimize_valid_metric=True,
verbose=False,
load_best_on_end=True,
timeit=False,
check=False,
overfit=False,
fp16=False,
ddp=False,
)
# model inference
for prediction in runner.predict_loader(loader=loaders["valid"]):
assert prediction["logits"].detach().cpu().numpy().shape[-1] == 10
# model post-processing
features_batch = next(iter(loaders["valid"]))[0]
# model stochastic weight averaging
model.load_state_dict(
utils.get_averaged_weights_by_path_mask(logdir=logdir,
path_mask="*.pth"))
# model onnx export
if SETTINGS.onnx_required:
utils.onnx_export(
model=runner.model,
batch=runner.engine.sync_device(features_batch),
file="./mnist.onnx",
verbose=False,
)
# model quantization
if SETTINGS.quantization_required:
utils.quantize_model(model=runner.model)
# model pruning
if SETTINGS.pruning_required:
utils.prune_model(model=runner.model,
pruning_fn="l1_unstructured",
amount=0.8)
# model tracing
utils.trace_model(model=runner.model, batch=features_batch)
criterion=criterion,
optimizer=optimizer,
scheduler=scheduler,
logdir="./catalyst_logs",
num_epochs=EPOCHS,
# valid_loader="valid",
# valid_metric="accuracy03",
# minimize_valid_metric=False,
verbose=True,
# uncomment for extra metrics:
callbacks=[
dl.AccuracyCallback(input_key="logits",
target_key="mask_class",
num_classes=NUM_CLASES),
# dl.PrecisionRecallF1SupportCallback(input_key="logits", target_key="mask_class", num_classes=NUM_CLASES),
# dl.AUCCallback(input_key="logits", target_key="mask_class"),
dl.ConfusionMatrixCallback(input_key="logits",
target_key="mask_class",
num_classes=NUM_CLASES),
],
)
"""
torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda, last_epoch=-1, verbose=False)
torch.optim.lr_scheduler.MultiplicativeLR(optimizer, lr_lambda, last_epoch=-1, verbose=False)
torch.optim.lr_scheduler.StepLR(optimizer, step_size, gamma=0.1, last_epoch=-1, verbose=False)
torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones, gamma=0.1, last_epoch=-1, verbose=False)
torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma, last_epoch=-1, verbose=False)
torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max, eta_min=0, last_epoch=-1, verbose=False)
torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode="min", factor=0.1, patience=10, threshold=0.0001, threshold_mode="rel", cooldown=0, min_lr=0, eps=1e-08, verbose=False)
torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr, max_lr, step_size_up=2000, step_size_down=None, mode="triangular", gamma=1.0, scale_fn=None, scale_mode="cycle", cycle_momentum=True, base_momentum=0.8, max_momentum=0.9, last_epoch=-1, verbose=False)
"""
def train_experiment(device):
with TemporaryDirectory() as logdir:
# sample data
num_samples, num_features, num_classes1, num_classes2 = int(1e4), int(
1e1), 4, 10
X = torch.rand(num_samples, num_features)
y1 = (torch.rand(num_samples, ) * num_classes1).to(torch.int64)
y2 = (torch.rand(num_samples, ) * num_classes2).to(torch.int64)
# pytorch loaders
dataset = TensorDataset(X, y1, y2)
loader = DataLoader(dataset, batch_size=32, num_workers=1)
loaders = {"train": loader, "valid": loader}
class CustomModule(nn.Module):
def __init__(self, in_features: int, out_features1: int,
out_features2: int):
super().__init__()
self.shared = nn.Linear(in_features, 128)
self.head1 = nn.Linear(128, out_features1)
self.head2 = nn.Linear(128, out_features2)
def forward(self, x):
x = self.shared(x)
y1 = self.head1(x)
y2 = self.head2(x)
return y1, y2
# model, criterion, optimizer, scheduler
model = CustomModule(num_features, num_classes1, num_classes2)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters())
scheduler = optim.lr_scheduler.MultiStepLR(optimizer, [2])
class CustomRunner(dl.Runner):
def handle_batch(self, batch):
x, y1, y2 = batch
y1_hat, y2_hat = self.model(x)
self.batch = {
"features": x,
"logits1": y1_hat,
"logits2": y2_hat,
"targets1": y1,
"targets2": y2,
}
# model training
runner = CustomRunner()
runner.train(
model=model,
criterion=criterion,
optimizer=optimizer,
scheduler=scheduler,
loaders=loaders,
num_epochs=1,
verbose=False,
callbacks=[
dl.CriterionCallback(metric_key="loss1",
input_key="logits1",
target_key="targets1"),
dl.CriterionCallback(metric_key="loss2",
input_key="logits2",
target_key="targets2"),
dl.MetricAggregationCallback(prefix="loss",
metrics=["loss1", "loss2"],
mode="mean"),
dl.OptimizerCallback(metric_key="loss"),
dl.SchedulerCallback(),
dl.AccuracyCallback(
input_key="logits1",
target_key="targets1",
num_classes=num_classes1,
prefix="one_",
),
dl.AccuracyCallback(
input_key="logits2",
target_key="targets2",
num_classes=num_classes2,
prefix="two_",
),
dl.ConfusionMatrixCallback(
input_key="logits1",
target_key="targets1",
num_classes=num_classes1,
prefix="one_cm",
),
# catalyst[ml] required
dl.ConfusionMatrixCallback(
input_key="logits2",
target_key="targets2",
num_classes=num_classes2,
prefix="two_cm",
),
# catalyst[ml] required
dl.CheckpointCallback(
"./logs/one",
loader_key="valid",
metric_key="one_accuracy",
minimize=False,
save_n_best=1,
),
dl.CheckpointCallback(
"./logs/two",
loader_key="valid",
metric_key="two_accuracy03",
minimize=False,
save_n_best=3,
),
],
loggers={
"console": dl.ConsoleLogger(),
"tb": dl.TensorboardLogger("./logs/tb")
},
)
def train_experiment(engine=None):
with TemporaryDirectory() as logdir:
# sample data
num_samples, num_features, num_classes1, num_classes2 = int(1e4), int(
1e1), 4, 10
X = torch.rand(num_samples, num_features)
y1 = (torch.rand(num_samples) * num_classes1).to(torch.int64)
y2 = (torch.rand(num_samples) * num_classes2).to(torch.int64)
# pytorch loaders
dataset = TensorDataset(X, y1, y2)
loader = DataLoader(dataset, batch_size=32, num_workers=1)
loaders = {"train": loader, "valid": loader}
# model, criterion, optimizer, scheduler
model = CustomModule(num_features, num_classes1, num_classes2)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters())
scheduler = optim.lr_scheduler.MultiStepLR(optimizer, [2])
callbacks = [
dl.CriterionCallback(metric_key="loss1",
input_key="logits1",
target_key="targets1"),
dl.CriterionCallback(metric_key="loss2",
input_key="logits2",
target_key="targets2"),
dl.MetricAggregationCallback(metric_key="loss",
metrics=["loss1", "loss2"],
mode="mean"),
dl.BackwardCallback(metric_key="loss"),
dl.OptimizerCallback(metric_key="loss"),
dl.SchedulerCallback(),
dl.AccuracyCallback(
input_key="logits1",
target_key="targets1",
num_classes=num_classes1,
prefix="one_",
),
dl.AccuracyCallback(
input_key="logits2",
target_key="targets2",
num_classes=num_classes2,
prefix="two_",
),
dl.CheckpointCallback(
"./logs/one",
loader_key="valid",
metric_key="one_accuracy01",
minimize=False,
topk=1,
),
dl.CheckpointCallback(
"./logs/two",
loader_key="valid",
metric_key="two_accuracy03",
minimize=False,
topk=3,
),
]
if SETTINGS.ml_required:
# catalyst[ml] required
callbacks.append(
dl.ConfusionMatrixCallback(
input_key="logits1",
target_key="targets1",
num_classes=num_classes1,
prefix="one_cm",
))
# catalyst[ml] required
callbacks.append(
dl.ConfusionMatrixCallback(
input_key="logits2",
target_key="targets2",
num_classes=num_classes2,
prefix="two_cm",
))
# model training
runner = CustomRunner()
runner.train(
engine=engine,
model=model,
criterion=criterion,
optimizer=optimizer,
scheduler=scheduler,
loaders=loaders,
num_epochs=1,
verbose=False,
callbacks=callbacks,
loggers={
"console": dl.ConsoleLogger(),
"tb": dl.TensorboardLogger("./logs/tb"),
},
)
