def learn(
self,
n_epoch: int, callbacks: Iterable[Callback] = None,
metrics: Dict[str, Metric] = None, final_metric: str = 'accuracy', load_path=None
) -> float:
if load_path is not None:
load_model(self._model, load_path)
self._cb_handler = CallbackHandler(self, n_epoch, callbacks, metrics, final_metric)
self._cb_handler.on_train_begin()
for e in range(n_epoch):
print("Epoch " + str(e))
self._model.train()
self._cb_handler.on_epoch_begin()
for inputs, lengths, labels in self._train_data:
self.learn_one_iter(inputs, lengths, labels)
stop_training = self.evaluate()
if stop_training:
print("Patience exceeded. Training finished.")
break
return self._cb_handler.on_train_end()
def learn(self,
n_epoch: int,
upscale_factor: int,
downsampling_mode: str = 'nearest',
callbacks: Iterable[Callback] = None,
metrics: Dict[str, Metric] = None,
final_metric: str = 'accuracy',
load_path=None) -> float:
assert upscale_factor > 1
if load_path is not None:
load_model(self._model, load_path)
self.scale_factor = 1.0 / upscale_factor
self.downsampling_mode = downsampling_mode
self._cb_handler = CallbackHandler(self, n_epoch, callbacks, metrics,
final_metric)
self._cb_handler.on_train_begin()
for e in range(n_epoch):
self._model.train()
self._cb_handler.on_epoch_begin()
for images in self._train_data:
self.learn_one_iter(images)
stop_training = self.evaluate()
if stop_training:
print("Patience exceeded. Training finished.")
break
return self._cb_handler.on_train_end()
def run_classifier_test():
print("Starting classifier test")
# progress_bar_test()
torch.backends.cudnn.benchmark = True
# data = CIFAR10('data/', train=True, download=True, transform=ToTensor())
# train_size = int(0.8 * len(data))
# val_size = len(data) - train_size
# train_dataset, val_dataset = torch.utils.data.random_split(data, [train_size, val_size])
# train_dataset.dataset.transform = Compose(
# [
# RandomHorizontalFlip(),
# RandomResizedCrop(size=32, scale=(0.95, 1.0)),
# # Cutout(length=16, n_holes=1),
# ToTensor()
# ]
# )
#
# test_dataset = torchvision.datasets.CIFAR10('data/', train=False, download=True, transform=ToTensor())
# kernel = partial(PolynomialKernel, dp=3, cp=2.0)
train_val_dataset = ImageFolder(
'data/imagenette-160/train',
transform=Compose([
Resize((128, 128)),
ToTensor()
])
)
test_dataset = ImageFolder(
'data/imagenette-160/val',
transform=Compose([
Resize((128, 128)),
ToTensor()
])
)
train_size = int(0.8 * len(train_val_dataset))
val_size = len(train_val_dataset) - train_size
train_dataset, val_dataset = random_split(train_val_dataset, [train_size, val_size])
train_dataset.dataset.transform = Compose(
[
RandomHorizontalFlip(),
RandomResizedCrop(size=(128, 128), scale=(0.95, 1.0)),
# Cutout(length=16, n_holes=1),
ToTensor()
]
)
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=128, shuffle=True)
val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=128, shuffle=False)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=128, shuffle=False)
class SEResNeXtShakeShake(ResNeXtBlock):
def __init__(self, in_channels, reduction_ratio=16, cardinality=2, activation=nn.ReLU,
normalization=nn.BatchNorm2d):
super(SEResNeXtShakeShake, self).__init__(
branches=nn.ModuleList(
[
nn.Sequential(
ConvolutionalLayer(
in_channels, in_channels // 4, kernel_size=1, padding=0,
activation=activation, normalization=normalization
),
ConvolutionalLayer(
in_channels // 4, in_channels, kernel_size=3, padding=1,
activation=activation, normalization=normalization
),
# ConvolutionalLayer(
# in_channels // 4, in_channels, kernel_size=1, padding=0,
# activation=activation, normalization=normalization
# ),
SEBlock(in_channels, reduction_ratio)
) for _ in range(cardinality)
]
),
use_shake_shake=True
)
class StandAloneMultiheadAttentionLayer(nn.Sequential):
def __init__(
self, num_heads, in_channels, out_channels, kernel_size, stride=1, padding=3,
activation=nn.ReLU, normalization=nn.BatchNorm2d
):
layers = [
StandAloneMultiheadAttention(
num_heads=num_heads,
in_channels=in_channels,
out_channels=out_channels,
kernel_size=kernel_size,
stride=stride,
padding=padding,
bias=False
),
activation(),
normalization(num_features=out_channels),
]
super(StandAloneMultiheadAttentionLayer, self).__init__(*layers)
class SEResNeXtShakeShakeAttention(ResNeXtBlock):
def __init__(self, num_heads, in_channels, reduction_ratio=16, cardinality=2, activation=nn.ReLU,
normalization=nn.BatchNorm2d):
super(SEResNeXtShakeShakeAttention, self).__init__(
branches=nn.ModuleList(
[
nn.Sequential(
ConvolutionalLayer(
in_channels=in_channels,
out_channels=in_channels // 2,
kernel_size=1,
activation=activation,
normalization=normalization
),
StandAloneMultiheadAttentionLayer(
num_heads=num_heads,
in_channels=in_channels // 2,
out_channels=in_channels // 2,
kernel_size=3,
activation=activation,
normalization=normalization
),
ConvolutionalLayer(
in_channels=in_channels // 2,
out_channels=in_channels,
kernel_size=1,
activation=activation,
normalization=normalization
),
SEBlock(in_channels, reduction_ratio)
) for _ in range(cardinality)
]
),
use_shake_shake=True
)
# layer_1 = ManifoldMixupModule(ConvolutionalLayer(in_channels=3, out_channels=16, kernel_size=3, activation=nn.ReLU))
# block_1 = ManifoldMixupModule(SEResNeXtShakeShake(in_channels=16, activation=nn.ReLU))
model = Sequential(
ConvolutionalLayer(in_channels=3, out_channels=16, kernel_size=3, activation=nn.ReLU),
SEResNeXtShakeShake(in_channels=16, activation=nn.ReLU),
# layer_1,
# block_1,
ConvolutionalLayer(
in_channels=16, out_channels=32,
activation=nn.ReLU,
kernel_size=2, stride=2
),
SEResNeXtShakeShake(in_channels=32),
ConvolutionalLayer(
in_channels=32, out_channels=64,
kernel_size=2, stride=2
),
SEResNeXtShakeShake(in_channels=64),
ConvolutionalLayer(
in_channels=64, out_channels=128,
kernel_size=2, stride=2
),
SEResNeXtShakeShake(in_channels=128),
ConvolutionalLayer(
in_channels=128, out_channels=256,
kernel_size=2, stride=2
),
SEResNeXtShakeShake(in_channels=256),
ConvolutionalLayer(
in_channels=256, out_channels=512,
kernel_size=2, stride=2
),
SEResNeXtShakeShake(in_channels=512),
# SEResNeXtShakeShakeAttention(num_heads=8, in_channels=512),
FeedforwardBlock(
in_channels=512,
out_features=10,
pool_output_size=2,
hidden_layer_sizes=(256, 128)
)
).to(get_device())
# lsuv_init(module=model, input=get_first_batch(train_loader, callbacks = [ToDeviceCallback()])[0])
# print(count_trainable_parameters(model)) # 14437816 3075928
optimizer = SGD(model.parameters(), weight_decay=0.0001, lr=0.30, momentum=0.9)
learner = SupervisedImageLearner(
train_data=train_loader,
val_data=val_loader,
model=model,
criterion=SmoothedCrossEntropy().to(get_device()),
optimizer=optimizer,
mixup=True
)
# lr_finder = LRFinder(
# model=model,
# train_data=train_loader,
# criterion=SmoothedCrossEntropy(),
# optimizer=partial(SGD, lr=0.074, weight_decay=0.0001, momentum=0.9),
# device=get_device()
# )
# lr_finder.find_lr(warmup=100, callbacks=[ToDeviceCallback()])
swa = StochasticWeightAveraging(learner, average_after=5025, update_every=670)
callbacks = [
# ManifoldMixupCallback(learner=learner, modules=[layer_1, block_1]),
ToDeviceCallback(),
InputProgressiveResizing(initial_size=80, max_size=160, upscale_every=10, upscale_factor=math.sqrt(2)),
# MixedPrecisionV2(),
Tensorboard(),
NaNWarner(),
# ReduceLROnPlateauCB(optimizer, monitor='accuracy', mode='max', patience=10),
LRSchedulerCB(CosineAnnealingLR(optimizer, eta_min=0.10, T_max=335)),
swa,
LossLogger(),
ModelCheckpoint(learner=learner, filepath="weights/model.pt", monitor='accuracy', mode='max'),
ProgressBarCB()
]
metrics = {
"accuracy": Accuracy(),
"loss": Loss()
}
final = learner.learn(
n_epoch=500,
callbacks=callbacks,
metrics=metrics,
final_metric='accuracy'
)
print(final)
load_model(model=model, path="weights/model.pt")
classifier = ImageClassifier(model, tta_transform=Compose([
ToPILImage(),
RandomHorizontalFlip(),
RandomResizedCrop(size=(128, 128), scale=(0.95, 1.0)),
ToTensor()
]))
print(classifier.evaluate(test_loader))
print("Test SWA:")
model = swa.get_averaged_model()
classifier = ImageClassifier(model, tta_transform=Compose([
ToPILImage(),
RandomHorizontalFlip(),
RandomResizedCrop(size=(128, 128), scale=(0.95, 1.0)),
ToTensor()
]))
print(classifier.evaluate(test_loader))
LossLogger(),
ModelCheckpoint(learner=learner,
filepath="weights/model.pt",
monitor='accuracy',
mode='max'),
]
metrics = {"accuracy": Accuracy(), "loss": Loss()}
final = learner.learn(n_epoch=500,
callbacks=callbacks,
metrics=metrics,
final_metric='accuracy')
print(final)
load_model(model=model, path="weights/model.pt")
classifier = ImageClassifier(model,
tta_transform=Compose([
ToPILImage(),
RandomHorizontalFlip(),
RandomResizedCrop(size=(128, 128),
scale=(0.95, 1.0)),
ToTensor()
]))
print(classifier.evaluate(test_loader))
print("Test SWA:")
model = swa.get_averaged_model()
classifier = ImageClassifier(model,
tta_transform=Compose([
ToPILImage(),