def build_model(self): print("=> building model") self.model = MNISTModel().to(self.device) self.criterion = nn.CrossEntropyLoss().to(self.device) self.optimizer = torch.optim.SGD(self.model.parameters(), lr=self.lr, momentum=0.9, weight_decay=1e-4)
class MNISTAgent(Agent): """MNISTAgent for MNIST and Fashion-MNIST.""" def __init__(self, global_args, subset=tuple(range(10))): super().__init__(global_args, subset, fine='MNIST') self.distr_type = global_args.distr_type if self.distr_type == 'uniform': self.distribution = np.array([0.1] * 10) elif self.distr_type == 'dirichlet': self.distribution = np.random.dirichlet([global_args.alpha] * 10) else: raise ValueError(f'Invalid distribution type: {self.distr_type}.') self.subset = subset def load_data(self, data_alloc, center=False): print("=> loading data") if center: self.train_loader, self.test_loader, self.num_train = data_alloc.create_dataset_for_center( self.batch_size, self.num_workers) else: self.train_loader, self.test_loader, self.num_train = data_alloc.create_dataset_for_client( self.distribution, self.batch_size, self.num_workers, self.subset) def build_model(self): print("=> building model") self.model = MNISTModel().to(self.device) self.criterion = nn.CrossEntropyLoss().to(self.device) self.optimizer = torch.optim.SGD(self.model.parameters(), lr=self.lr, momentum=0.9, weight_decay=1e-4)
def create(): device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') weights_path = fb.zoo.fetch_weights( 'https://github.com/maurapintor/mnist-pretrained/releases/download/v1.2/mnist_cnn.pt', unzip=False) model = MNISTModel() state_dict = torch.load(weights_path, map_location=device) model.load_state_dict(state_dict) model.eval() preprocessing = {'mean': 0.5, 'std': 0.5} fmodel = fb.models.PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing, device=device) return fmodel
def train(args): transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=(0.1307, ), std=(0.3081, )) ]) train_dataset = datasets.MNIST(root='data', train=True, transform=transform, download=True) test_dataset = datasets.MNIST(root='data', train=False, transform=transform, download=True) train_loader = DataLoader(dataset=train_dataset, batch_size=256, sampler=sampler.SubsetRandomSampler( list(range(0, 55000)))) valid_loader = DataLoader(dataset=train_dataset, batch_size=256, sampler=sampler.SubsetRandomSampler( list(range(55000, 60000)))) test_loader = DataLoader(dataset=test_dataset, batch_size=256) model = MNISTModel(margin=args.margin) if args.gpu > -1: model.cuda(args.gpu) criterion = nn.CrossEntropyLoss() if args.optimizer == 'sgd': optimizer = optim.SGD(params=model.parameters(), lr=0.1, momentum=0.9, weight_decay=0.0005) min_lr = 0.001 elif args.optimizer == 'adam': optimizer = optim.Adam(model.parameters(), weight_decay=0.0005) min_lr = 0.00001 else: raise ValueError('Unknown optimizer') lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer, mode='max', factor=0.1, patience=5, verbose=True, min_lr=min_lr) summary_writer = SummaryWriter(os.path.join(args.save_dir, 'log')) def var(tensor, volatile=False): if args.gpu > -1: tensor = tensor.cuda(args.gpu) return Variable(tensor, volatile=volatile) global_step = 0 def train_epoch(): nonlocal global_step model.train() for train_batch in train_loader: train_x, train_y = var(train_batch[0]), var(train_batch[1]) logit = model(input=train_x, target=train_y) loss = criterion(input=logit, target=train_y) optimizer.zero_grad() loss.backward() clip_grad_norm(model.parameters(), max_norm=10) optimizer.step() global_step += 1 summary_writer.add_scalar(tag='train_loss', scalar_value=loss.data[0], global_step=global_step) def validate(): model.eval() loss_sum = num_correct = denom = 0 for valid_batch in valid_loader: valid_x, valid_y = (var(valid_batch[0], volatile=True), var(valid_batch[1], volatile=True)) logit = model(valid_x) y_pred = logit.max(1)[1] loss = criterion(input=logit, target=valid_y) loss_sum += loss.data[0] * valid_x.size(0) num_correct += y_pred.eq(valid_y).long().sum().data[0] denom += valid_x.size(0) loss = loss_sum / denom accuracy = num_correct / denom summary_writer.add_scalar(tag='valid_loss', scalar_value=loss, global_step=global_step) summary_writer.add_scalar(tag='valid_accuracy', scalar_value=accuracy, global_step=global_step) lr_scheduler.step(accuracy) return loss, accuracy def test(): model.eval() num_correct = denom = 0 for test_batch in test_loader: test_x, test_y = (var(test_batch[0], volatile=True), var(test_batch[1], volatile=True)) logit = model(test_x) y_pred = logit.max(1)[1] num_correct += y_pred.eq(test_y).long().sum().data[0] denom += test_x.size(0) accuracy = num_correct / denom summary_writer.add_scalar(tag='test_accuracy', scalar_value=accuracy, global_step=global_step) return accuracy best_valid_accuracy = 0 for epoch in range(1, args.max_epoch + 1): train_epoch() valid_loss, valid_accuracy = validate() print('Epoch {epoch}: Valid loss = {valid_loss:.5f}') print('Epoch {epoch}: Valid accuracy = {valid_accuracy:.5f}') test_accuracy = test() print('Epoch {epoch}: Test accuracy = {test_accuracy:.5f}') if valid_accuracy > best_valid_accuracy: model_filename = ('{epoch:02d}' '-{valid_loss:.5f}' '-{valid_accuracy:.5f}' '-{test_accuracy:.5f}.pt') model_path = os.path.join(args.save_dir, model_filename) torch.save(model.state_dict(), model_path) print('Epoch {epoch}: Saved the new best model to: {model_path}') best_valid_accuracy = valid_accuracy
import torch from model import MNISTModel model = MNISTModel() model.load_state_dict(torch.load('mnist_classification.pth')) ip = torch.randn(1, 28, 28) op = model.predict(torch.unsqueeze(ip, 0)) print(op)
import os from argparse import ArgumentParser from model import MNISTModel from pytorch_lightning import Trainer parser = ArgumentParser() parser.add_argument("--data_dir", type=str, default=os.environ.get("MNIST_DIR", ".")) parser = Trainer.add_argparse_args(parser) parser = MNISTModel.add_model_specific_args(parser) hparams = parser.parse_args() # Init Model model = MNISTModel(hparams) # Init Trainer trainer = Trainer.from_argparse_args(hparams) # Fit the Model trainer.fit(model)
optimizer.step() optimizer.zero_grad() train_result = model.epoch_end(train_history) valid_result = model.evaluate(valid_loader) epoch_result = { 'train_loss': train_result['loss'], 'train_acc': train_result['acc'], 'val_loss': valid_result['loss'], 'val_acc': valid_result['acc'] } model.epoch_end_log(epoch, epoch_result) history.append(epoch_result) return history model = MNISTModel() history = [] mnist_ds = MNIST('.', download=True, train=True, transform=t.ToTensor()) train_ds, valid_ds = random_split(mnist_ds, [55000, 5000]) train_dl = DataLoader(train_ds, batch_size=64, shuffle=True) valid_dl = DataLoader(valid_ds, batch_size=64) device = get_default_device() train_dl = DeviceDataLoader(train_dl, device) valid_dl = DeviceDataLoader(valid_dl, device) model = to_device(model, device) history += fit(model, 5, 1e-3, train_dl, valid_dl, opt=optim.Adam)
def main(): # Training settings parser = argparse.ArgumentParser(description='PyTorch MNIST Example') parser.add_argument('--batch-size', type=int, default=128, metavar='N', help='input batch size for training (default: 128)') parser.add_argument('--test-batch-size', type=int, default=128, metavar='N', help='input batch size for testing (default: 128)') parser.add_argument('--epochs', type=int, default=50, metavar='N', help='number of epochs to train (default: 50)') parser.add_argument('--lr', type=float, default=0.01, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--wd', type=float, default=1e-6, metavar='WD', help='Weight decay (default: 1e-6)') parser.add_argument('--momentum', type=float, default=0.9, metavar='M', help='momentum (default: 0.9)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument( '--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', default=False, help='For Saving the current Model') args = parser.parse_args() use_cuda = not args.no_cuda and torch.cuda.is_available() device = torch.device("cuda" if use_cuda else "cpu") kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} train_loader = DataLoader(datasets.MNIST('../data', train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.5, ), (0.5, )) ])), batch_size=args.batch_size, shuffle=True, **kwargs) test_loader = DataLoader(datasets.MNIST('../data', train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.5, ), (0.5, )) ])), batch_size=args.test_batch_size, shuffle=True, **kwargs) model = MNISTModel().to(device) optimizer = optim.SGD(model.parameters(), lr=args.lr, weight_decay=args.wd, momentum=args.momentum, nesterov=True) for epoch in range(1, args.epochs + 1): train(args, model, device, train_loader, optimizer, epoch) test(model, device, test_loader) if args.save_model: torch.save(model.state_dict(), "mnist_cnn.pt")
batch_size=batch_size, shuffle=True, num_workers=8) test_dataset = datasets.MNIST(root='dataset', train=False, transform=img_transform, download=True) test_dataloader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False, num_workers=8) # load model model = MNISTModel() # setup optimizer optimizer = optim.Adam(model.parameters(), lr=lr) loss_func = torch.nn.CrossEntropyLoss() model = model.cuda() loss_func = loss_func.cuda() def train_one_epoch(model, dataloader, epoch): model.train() for i, (img, label) in enumerate(dataloader): # training model using source data
import tensorflow as tf import numpy as np from model import MNIST from model import MNISTModel save_path = "models\mnist" data, model = MNIST(), MNISTModel().get() model.compile(loss=tf.keras.losses.categorical_crossentropy, optimizer='adam', metrics=['accuracy']) # Train model.fit(data.train_data, data.train_labels, batch_size=256, epochs=1, verbose=1, validation_data=(data.validation_data, data.validation_labels)) model.save(save_path) score = model.evaluate(data.test_data, data.test_labels, verbose=0) print("Test Loss:", score[0]) print("Test Accuracy:", score[1])
return torch.utils.data.DataLoader(dataset, num_workers=cpu_count(), shuffle=True, batch_size=200) def savePlot(): plt.xlabel("Epoch") plt.ylabel("Aerage Loss") plt.plot(train_losses, label="Training Set") plt.plot(test_losses, label="Test Set") plt.legend() plt.savefig("model_losses.svg") plt.close() if __name__ == "__main__": device = torch.device("cuda" if torch.cuda.is_available() else "cpu") train_loader = getLoader(True) test_loader = getLoader(False) model = MNISTModel(28, 28).to(device) optimizer = torch.optim.Adadelta(model.parameters()) lossFunction = torch.nn.NLLLoss() train_losses = [] test_losses = [] for epoch in range(1, 11): trainModel() testModel() torch.save(model.state_dict(), "trainedModel.pt") savePlot()