def build_model(self):
print("=> building model")
self.model = MNISTModel().to(self.device)
self.criterion = nn.CrossEntropyLoss().to(self.device)
self.optimizer = torch.optim.SGD(self.model.parameters(),
lr=self.lr,
momentum=0.9,
weight_decay=1e-4)
class MNISTAgent(Agent):
"""MNISTAgent for MNIST and Fashion-MNIST."""
def __init__(self, global_args, subset=tuple(range(10))):
super().__init__(global_args, subset, fine='MNIST')
self.distr_type = global_args.distr_type
if self.distr_type == 'uniform':
self.distribution = np.array([0.1] * 10)
elif self.distr_type == 'dirichlet':
self.distribution = np.random.dirichlet([global_args.alpha] * 10)
else:
raise ValueError(f'Invalid distribution type: {self.distr_type}.')
self.subset = subset
def load_data(self, data_alloc, center=False):
print("=> loading data")
if center:
self.train_loader, self.test_loader, self.num_train = data_alloc.create_dataset_for_center(
self.batch_size, self.num_workers)
else:
self.train_loader, self.test_loader, self.num_train = data_alloc.create_dataset_for_client(
self.distribution, self.batch_size, self.num_workers,
self.subset)
def build_model(self):
print("=> building model")
self.model = MNISTModel().to(self.device)
self.criterion = nn.CrossEntropyLoss().to(self.device)
self.optimizer = torch.optim.SGD(self.model.parameters(),
lr=self.lr,
momentum=0.9,
weight_decay=1e-4)
def create():
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
weights_path = fb.zoo.fetch_weights(
'https://github.com/maurapintor/mnist-pretrained/releases/download/v1.2/mnist_cnn.pt',
unzip=False)
model = MNISTModel()
state_dict = torch.load(weights_path, map_location=device)
model.load_state_dict(state_dict)
model.eval()
preprocessing = {'mean': 0.5, 'std': 0.5}
fmodel = fb.models.PyTorchModel(model,
bounds=(0, 1),
preprocessing=preprocessing,
device=device)
return fmodel
def train(args):
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=(0.1307, ), std=(0.3081, ))
])
train_dataset = datasets.MNIST(root='data',
train=True,
transform=transform,
download=True)
test_dataset = datasets.MNIST(root='data',
train=False,
transform=transform,
download=True)
train_loader = DataLoader(dataset=train_dataset,
batch_size=256,
sampler=sampler.SubsetRandomSampler(
list(range(0, 55000))))
valid_loader = DataLoader(dataset=train_dataset,
batch_size=256,
sampler=sampler.SubsetRandomSampler(
list(range(55000, 60000))))
test_loader = DataLoader(dataset=test_dataset, batch_size=256)
model = MNISTModel(margin=args.margin)
if args.gpu > -1:
model.cuda(args.gpu)
criterion = nn.CrossEntropyLoss()
if args.optimizer == 'sgd':
optimizer = optim.SGD(params=model.parameters(),
lr=0.1,
momentum=0.9,
weight_decay=0.0005)
min_lr = 0.001
elif args.optimizer == 'adam':
optimizer = optim.Adam(model.parameters(), weight_decay=0.0005)
min_lr = 0.00001
else:
raise ValueError('Unknown optimizer')
lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer,
mode='max',
factor=0.1,
patience=5,
verbose=True,
min_lr=min_lr)
summary_writer = SummaryWriter(os.path.join(args.save_dir, 'log'))
def var(tensor, volatile=False):
if args.gpu > -1:
tensor = tensor.cuda(args.gpu)
return Variable(tensor, volatile=volatile)
global_step = 0
def train_epoch():
nonlocal global_step
model.train()
for train_batch in train_loader:
train_x, train_y = var(train_batch[0]), var(train_batch[1])
logit = model(input=train_x, target=train_y)
loss = criterion(input=logit, target=train_y)
optimizer.zero_grad()
loss.backward()
clip_grad_norm(model.parameters(), max_norm=10)
optimizer.step()
global_step += 1
summary_writer.add_scalar(tag='train_loss',
scalar_value=loss.data[0],
global_step=global_step)
def validate():
model.eval()
loss_sum = num_correct = denom = 0
for valid_batch in valid_loader:
valid_x, valid_y = (var(valid_batch[0], volatile=True),
var(valid_batch[1], volatile=True))
logit = model(valid_x)
y_pred = logit.max(1)[1]
loss = criterion(input=logit, target=valid_y)
loss_sum += loss.data[0] * valid_x.size(0)
num_correct += y_pred.eq(valid_y).long().sum().data[0]
denom += valid_x.size(0)
loss = loss_sum / denom
accuracy = num_correct / denom
summary_writer.add_scalar(tag='valid_loss',
scalar_value=loss,
global_step=global_step)
summary_writer.add_scalar(tag='valid_accuracy',
scalar_value=accuracy,
global_step=global_step)
lr_scheduler.step(accuracy)
return loss, accuracy
def test():
model.eval()
num_correct = denom = 0
for test_batch in test_loader:
test_x, test_y = (var(test_batch[0], volatile=True),
var(test_batch[1], volatile=True))
logit = model(test_x)
y_pred = logit.max(1)[1]
num_correct += y_pred.eq(test_y).long().sum().data[0]
denom += test_x.size(0)
accuracy = num_correct / denom
summary_writer.add_scalar(tag='test_accuracy',
scalar_value=accuracy,
global_step=global_step)
return accuracy
best_valid_accuracy = 0
for epoch in range(1, args.max_epoch + 1):
train_epoch()
valid_loss, valid_accuracy = validate()
print('Epoch {epoch}: Valid loss = {valid_loss:.5f}')
print('Epoch {epoch}: Valid accuracy = {valid_accuracy:.5f}')
test_accuracy = test()
print('Epoch {epoch}: Test accuracy = {test_accuracy:.5f}')
if valid_accuracy > best_valid_accuracy:
model_filename = ('{epoch:02d}'
'-{valid_loss:.5f}'
'-{valid_accuracy:.5f}'
'-{test_accuracy:.5f}.pt')
model_path = os.path.join(args.save_dir, model_filename)
torch.save(model.state_dict(), model_path)
print('Epoch {epoch}: Saved the new best model to: {model_path}')
best_valid_accuracy = valid_accuracy
import torch
from model import MNISTModel
model = MNISTModel()
model.load_state_dict(torch.load('mnist_classification.pth'))
ip = torch.randn(1, 28, 28)
op = model.predict(torch.unsqueeze(ip, 0))
print(op)
import os
from argparse import ArgumentParser
from model import MNISTModel
from pytorch_lightning import Trainer
parser = ArgumentParser()
parser.add_argument("--data_dir",
type=str,
default=os.environ.get("MNIST_DIR", "."))
parser = Trainer.add_argparse_args(parser)
parser = MNISTModel.add_model_specific_args(parser)
hparams = parser.parse_args()
# Init Model
model = MNISTModel(hparams)
# Init Trainer
trainer = Trainer.from_argparse_args(hparams)
# Fit the Model
trainer.fit(model)
optimizer.step()
optimizer.zero_grad()
train_result = model.epoch_end(train_history)
valid_result = model.evaluate(valid_loader)
epoch_result = {
'train_loss': train_result['loss'],
'train_acc': train_result['acc'],
'val_loss': valid_result['loss'],
'val_acc': valid_result['acc']
}
model.epoch_end_log(epoch, epoch_result)
history.append(epoch_result)
return history
model = MNISTModel()
history = []
mnist_ds = MNIST('.', download=True, train=True, transform=t.ToTensor())
train_ds, valid_ds = random_split(mnist_ds, [55000, 5000])
train_dl = DataLoader(train_ds, batch_size=64, shuffle=True)
valid_dl = DataLoader(valid_ds, batch_size=64)
device = get_default_device()
train_dl = DeviceDataLoader(train_dl, device)
valid_dl = DeviceDataLoader(valid_dl, device)
model = to_device(model, device)
history += fit(model, 5, 1e-3, train_dl, valid_dl, opt=optim.Adam)
def main():
# Training settings
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
parser.add_argument('--batch-size',
type=int,
default=128,
metavar='N',
help='input batch size for training (default: 128)')
parser.add_argument('--test-batch-size',
type=int,
default=128,
metavar='N',
help='input batch size for testing (default: 128)')
parser.add_argument('--epochs',
type=int,
default=50,
metavar='N',
help='number of epochs to train (default: 50)')
parser.add_argument('--lr',
type=float,
default=0.01,
metavar='LR',
help='learning rate (default: 0.01)')
parser.add_argument('--wd',
type=float,
default=1e-6,
metavar='WD',
help='Weight decay (default: 1e-6)')
parser.add_argument('--momentum',
type=float,
default=0.9,
metavar='M',
help='momentum (default: 0.9)')
parser.add_argument('--no-cuda',
action='store_true',
default=False,
help='disables CUDA training')
parser.add_argument(
'--log-interval',
type=int,
default=10,
metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument('--save-model',
action='store_true',
default=False,
help='For Saving the current Model')
args = parser.parse_args()
use_cuda = not args.no_cuda and torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
train_loader = DataLoader(datasets.MNIST('../data',
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, ),
(0.5, ))
])),
batch_size=args.batch_size,
shuffle=True,
**kwargs)
test_loader = DataLoader(datasets.MNIST('../data',
train=False,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, ),
(0.5, ))
])),
batch_size=args.test_batch_size,
shuffle=True,
**kwargs)
model = MNISTModel().to(device)
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
weight_decay=args.wd,
momentum=args.momentum,
nesterov=True)
for epoch in range(1, args.epochs + 1):
train(args, model, device, train_loader, optimizer, epoch)
test(model, device, test_loader)
if args.save_model:
torch.save(model.state_dict(), "mnist_cnn.pt")
batch_size=batch_size,
shuffle=True,
num_workers=8)
test_dataset = datasets.MNIST(root='dataset',
train=False,
transform=img_transform,
download=True)
test_dataloader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=8)
# load model
model = MNISTModel()
# setup optimizer
optimizer = optim.Adam(model.parameters(), lr=lr)
loss_func = torch.nn.CrossEntropyLoss()
model = model.cuda()
loss_func = loss_func.cuda()
def train_one_epoch(model, dataloader, epoch):
model.train()
for i, (img, label) in enumerate(dataloader):
# training model using source data
import tensorflow as tf
import numpy as np
from model import MNIST
from model import MNISTModel
save_path = "models\mnist"
data, model = MNIST(), MNISTModel().get()
model.compile(loss=tf.keras.losses.categorical_crossentropy,
optimizer='adam',
metrics=['accuracy'])
# Train
model.fit(data.train_data,
data.train_labels,
batch_size=256,
epochs=1,
verbose=1,
validation_data=(data.validation_data, data.validation_labels))
model.save(save_path)
score = model.evaluate(data.test_data, data.test_labels, verbose=0)
print("Test Loss:", score[0])
print("Test Accuracy:", score[1])
return torch.utils.data.DataLoader(dataset,
num_workers=cpu_count(),
shuffle=True,
batch_size=200)
def savePlot():
plt.xlabel("Epoch")
plt.ylabel("Aerage Loss")
plt.plot(train_losses, label="Training Set")
plt.plot(test_losses, label="Test Set")
plt.legend()
plt.savefig("model_losses.svg")
plt.close()
if __name__ == "__main__":
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
train_loader = getLoader(True)
test_loader = getLoader(False)
model = MNISTModel(28, 28).to(device)
optimizer = torch.optim.Adadelta(model.parameters())
lossFunction = torch.nn.NLLLoss()
train_losses = []
test_losses = []
for epoch in range(1, 11):
trainModel()
testModel()
torch.save(model.state_dict(), "trainedModel.pt")
savePlot()