def main(args):
logs = []
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
trainloader, testloader = get_dataset(args, transform)
net = AlexNet()
if args.no_distributed:
optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.0)
else:
optimizer = DownpourSGD(net.parameters(), lr=args.lr, n_push=args.num_push, n_pull=args.num_pull, model=net)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=1, verbose=True, min_lr=1e-3)
# train
net.train()
if args.cuda:
net = net.cuda()
for epoch in range(args.epochs): # loop over the dataset multiple times
print("Training for epoch {}".format(epoch))
for i, data in enumerate(trainloader, 0):
# get the inputs
inputs, labels = data
if args.cuda:
inputs, labels = inputs.cuda(), labels.cuda()
# zero the parameter gradients
optimizer.zero_grad()
# forward + backward + optimize
outputs = net(inputs)
loss = F.cross_entropy(outputs, labels)
loss.backward()
optimizer.step()
_, predicted = torch.max(outputs, 1)
accuracy = accuracy_score(predicted, labels)
log_obj = {
'timestamp': datetime.now(),
'iteration': i,
'training_loss': loss.item(),
'training_accuracy': accuracy,
}
if i % args.log_interval == 0 and i > 0: # print every n mini-batches
log_obj['test_loss'], log_obj['test_accuracy']= evaluate( net, testloader, args)
print("Timestamp: {timestamp} | "
"Iteration: {iteration:6} | "
"Loss: {training_loss:6.4f} | "
"Accuracy : {training_accuracy:6.4f} | "
"Test Loss: {test_loss:6.4f} | "
"Test Accuracy: {test_accuracy:6.4f}".format(**log_obj))
logs.append(log_obj)
val_loss, val_accuracy = evaluate(net, testloader, args, verbose=True)
scheduler.step(val_loss)
df = pd.DataFrame(logs)
print(df)
if args.no_distributed:
if args.cuda:
df.to_csv('log/gpu.csv', index_label='index')
else:
df.to_csv('log/single.csv', index_label='index')
else:
df.to_csv('log/node{}.csv'.format(dist.get_rank()), index_label='index')
print('Finished Training')
loader_test = torch.utils.data.DataLoader(
datasets.ImageFolder(valdir, transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.batch_size, shuffle=False,
num_workers=args.workers, pin_memory=True)
# Load the pretrained model
net = AlexNet()
net.load_state_dict(torch.load('/home/choong/.torch/models/alexnet-owt-4df8aa71.pth'), strict=False)
if torch.cuda.is_available():
print('CUDA enabled.')
net.cuda()
print("--- Pretrained network loaded ---")
# test(net, loader_test)
# prune the weights
masks = weight_prune(net, param['pruning_perc'])
net.set_masks(masks)
net = nn.DataParallel(net)
print("--- {}% parameters pruned ---".format(param['pruning_perc']))
test(net, loader_test)
# Retraining
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.RMSprop(net.parameters(), lr=param['learning_rate'],
weight_decay=param['weight_decay'])
help='whether or not to use all subjects')
parser.add_argument('--l2',
default=0,
type=float,
help='L2 regularization weight')
args = parser.parse_args()
if args.feature_extractor == 'alexnet':
feat_extractor = AlexNet(args.feature_name)
elif args.feature_extractor == 'vgg16':
feat_extractor = VGG16(args.feature_name)
else:
raise ValueError('unimplemented feature extractor: {}'.format(
args.feature_extractor))
if torch.cuda.is_available():
feat_extractor.cuda()
subj_file = 'subjall.npy' if args.allsubj else 'subj1.npy'
voxels, stimuli = voxel_data(os.path.join(args.bold5000_folder, subj_file),
args.roi)
voxel_pcs = PCA(n_components=voxels.shape[1]).fit_transform(voxels)
stimuli = [
os.path.join(args.bold5000_folder, 'stimuli', s) for s in stimuli
]
features = condition_features(stimuli, feat_extractor)
cv_r = []
cv = KFold(n_splits=5, shuffle=True, random_state=27)
for train_idx, val_idx in cv.split(features):
features_train, features_val = features[train_idx], features[val_idx]
def main():
global args, best_prec1
args = parser.parse_args()
print(args)
# Set # classes
if args.data == 'UCF101':
num_classes = 101
else:
num_classes = 0
print('Specify the dataset to use ')
# Create model
if args.pretrained:
print("=> using pre-trained model '{}'".format(args.arch))
model = models.__dict__[args.arch](pretrained=True)
else:
print("=> creating model '{}'".format(args.arch))
model = models.__dict__[args.arch]()
if args.arch.startswith('alexnet'):
model = AlexNet(num_classes=num_classes)
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
# Modify last layer of the model
model_ft = models.resnet18(pretrained=True)
num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, 101)
model = model_ft.cuda()
model = torch.nn.DataParallel(model).cuda() # Using one GPU (device_ids = 1)
# print(model)
# Define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(), args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# Optionally resume from a checkpoint
if args.resume:
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})"
.format(args.resume, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
cudnn.benchmark = True
# Data loading code
traindir = os.path.join(args.data, 'train')
testdir = os.path.join(args.data, 'test')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_loader = torch.utils.data.DataLoader(
datasets.ImageFolder(traindir, transforms.Compose([
transforms.RandomCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])),
batch_size = args.batch_size, shuffle=True,
num_workers=args.workers, pin_memory=True
)
val_loader = torch.utils.data.DataLoader(
datasets.ImageFolder(testdir, transforms.Compose([
transforms.Scale(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size = args.batch_size, shuffle=False,
num_workers=args.workers, pin_memory=True
)
if args.evaluate:
validate(val_loader, model, criterion)
return
for epoch in range(args.start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
# Train for one epoch
train(train_loader, model, criterion, optimizer, epoch)
# # Evaluate on validation set
prec1 = validate(val_loader, model, criterion)
# Remember best prec@1 and save checkpoint
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
save_checkpoint({
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
}, is_best)