def generate_inverted_image_specific_layer(self, input_image, img_size, target_layer=3):
# Generate a random image which we will optimize
opt_img = Variable(1e-1 * torch.randn(1, 3, img_size, img_size), requires_grad=True)
# Define optimizer for previously created image
optimizer = SGD([opt_img], lr=1e4, momentum=0.9)
# Get the output from the model after a forward pass until target_layer
# with the input image (real image, NOT the randomly generated one)
input_image_layer_output = \
self.get_output_from_specific_layer(input_image, target_layer)
# Alpha regularization parametrs
# Parameter alpha, which is actually sixth norm
alpha_reg_alpha = 6
# The multiplier, lambda alpha
alpha_reg_lambda = 1e-7
# Total variation regularization parameters
# Parameter beta, which is actually second norm
tv_reg_beta = 2
# The multiplier, lambda beta
tv_reg_lambda = 1e-8
for i in range(201):
optimizer.zero_grad()
# Get the output from the model after a forward pass until target_layer
# with the generated image (randomly generated one, NOT the real image)
output = self.get_output_from_specific_layer(opt_img, target_layer)
# Calculate euclidian loss
euc_loss = 1e-1 * self.euclidian_loss(input_image_layer_output.detach(), output)
# Calculate alpha regularization
reg_alpha = alpha_reg_lambda * self.alpha_norm(opt_img, alpha_reg_alpha)
# Calculate total variation regularization
reg_total_variation = tv_reg_lambda * self.total_variation_norm(opt_img,
tv_reg_beta)
# Sum all to optimize
loss = euc_loss + reg_alpha + reg_total_variation
# Step
loss.backward()
optimizer.step()
# Generate image every 5 iterations
if i % 5 == 0:
print('Iteration:', str(i), 'Loss:', loss.data.numpy()[0])
x = recreate_image(opt_img)
cv2.imwrite('../generated/Inv_Image_Layer_' + str(target_layer) +
'_Iteration_' + str(i) + '.jpg', x)
# Reduce learning rate every 40 iterations
if i % 40 == 0:
for param_group in optimizer.param_groups:
param_group['lr'] *= 1/10
def test_mask_same_after_update(generate_batch):
from torch.optim import SGD
unary, tags, lengths = generate_batch
h = unary.size(2)
constraint = torch.rand(h, h) < 0.5
crf = CRF(h, constraint=constraint)
opt = SGD(crf.parameters(), lr=10)
m1 = crf.constraint.numpy()
t1 = crf.transitions_p.detach().clone().numpy()
l = crf.neg_log_loss(unary, tags, lengths)
l = torch.mean(l)
l.backward()
opt.step()
m2 = crf.constraint.numpy()
t2 = crf.transitions_p.detach().numpy()
np.testing.assert_allclose(m1, m2)
with pytest.raises(AssertionError):
np.testing.assert_allclose(t1, t2)
def generate(self):
initial_learning_rate = 6
for i in range(1, 150):
# Process image and return variable
self.processed_image = preprocess_image(self.created_image)
# Define optimizer for the image
optimizer = SGD([self.processed_image], lr=initial_learning_rate)
# Forward
output = self.model(self.processed_image)
# Target specific class
class_loss = -output[0, self.target_class]
print('Iteration:', str(i), 'Loss', "{0:.2f}".format(class_loss.data.numpy()[0]))
# Zero grads
self.model.zero_grad()
# Backward
class_loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(self.processed_image)
# Save image
cv2.imwrite('../generated/c_specific_iteration_'+str(i)+'.jpg', self.created_image)
return self.processed_image
def test_scheduler_reduce_on_plateau_plugin_with_val_stream(self):
# Regression test for issue #858 (part 2)
n_epochs = 20
criterion = CrossEntropyLoss()
def _prepare_rng_critical_parts(seed=1234):
torch.random.manual_seed(seed)
initial_benchmark = PluginTests.create_benchmark(seed=seed)
val_benchmark = benchmark_with_validation_stream(initial_benchmark,
0.3,
shuffle=True)
return (val_benchmark, _PlainMLP(input_size=6, hidden_size=10))
self._verify_rop_tests_reproducibility(_prepare_rng_critical_parts,
n_epochs, criterion)
# Everything is in order, now we can test the plugin support for the
# ReduceLROnPlateau scheduler!
for reset_lr, reset_scheduler in itertools.product((True, False),
(True, False)):
with self.subTest(reset_lr=reset_lr,
reset_scheduler=reset_scheduler):
# First, obtain the reference (expected) lr timeline by running
# a plain PyTorch training loop with ReduceLROnPlateau.
benchmark, model = _prepare_rng_critical_parts()
expected_lrs = []
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
for exp_idx, exp in enumerate(benchmark.train_stream):
expected_lrs.append([])
model.train()
if reset_lr:
for group in optimizer.param_groups:
group["lr"] = 0.001
if reset_scheduler:
scheduler = ReduceLROnPlateau(optimizer)
for epoch in range(n_epochs):
for x, y, t in TaskBalancedDataLoader(
exp.dataset,
oversample_small_groups=True,
num_workers=0,
batch_size=32,
shuffle=False,
pin_memory=False,
):
optimizer.zero_grad()
outputs = model(x)
loss = criterion(outputs, y)
loss.backward()
optimizer.step()
for group in optimizer.param_groups:
expected_lrs[-1].append(group["lr"])
break
val_loss = Mean()
val_exp = benchmark.valid_stream[exp_idx]
model.eval()
with torch.no_grad():
for x, y, t in DataLoader(
val_exp.dataset,
num_workers=0,
batch_size=100,
pin_memory=False,
):
outputs = model(x)
loss = criterion(outputs, y)
val_loss.update(loss, weight=len(x))
scheduler.step(val_loss.result())
# Now we have the correct timeline stored in expected_lrs
# Let's test the plugin!
benchmark, model = _prepare_rng_critical_parts()
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
PluginTests._test_scheduler_plugin(
benchmark,
model,
optimizer,
scheduler,
n_epochs,
reset_lr,
reset_scheduler,
expected_lrs,
criterion=criterion,
metric="val_loss",
eval_on_valid_stream=True,
)
def main(args: argparse.Namespace):
logger = CompleteLogger(args.log, args.phase)
print(args)
if args.seed is not None:
random.seed(args.seed)
torch.manual_seed(args.seed)
cudnn.deterministic = True
warnings.warn('You have chosen to seed training. '
'This will turn on the CUDNN deterministic setting, '
'which can slow down your training considerably! '
'You may see unexpected behavior when restarting '
'from checkpoints.')
cudnn.benchmark = True
# Data loading code
normalize = T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
if args.center_crop:
train_transform = T.Compose([
ResizeImage(256),
T.CenterCrop(224),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalize
])
else:
train_transform = T.Compose([
ResizeImage(256),
T.RandomResizedCrop(224),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalize
])
val_transform = T.Compose([
ResizeImage(256),
T.CenterCrop(224),
T.ToTensor(),
normalize
])
dataset = datasets.__dict__[args.data]
train_source_dataset = dataset(root=args.root, task=args.source, download=True, transform=train_transform)
train_source_loader = DataLoader(train_source_dataset, batch_size=args.batch_size,
shuffle=True, num_workers=args.workers, drop_last=True)
train_target_dataset = dataset(root=args.root, task=args.target, download=True, transform=train_transform)
train_target_loader = DataLoader(train_target_dataset, batch_size=args.batch_size,
shuffle=True, num_workers=args.workers, drop_last=True)
val_dataset = dataset(root=args.root, task=args.target, download=True, transform=val_transform)
val_loader = DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers)
if args.data == 'DomainNet':
test_dataset = dataset(root=args.root, task=args.target, split='test', download=True, transform=val_transform)
test_loader = DataLoader(test_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers)
else:
test_loader = val_loader
train_source_iter = ForeverDataIterator(train_source_loader)
train_target_iter = ForeverDataIterator(train_target_loader)
# create model
print("=> using pre-trained model '{}'".format(args.arch))
G = models.__dict__[args.arch](pretrained=True).to(device) # feature extractor
num_classes = train_source_dataset.num_classes
# two image classifier heads
F1 = ImageClassifierHead(G.out_features, num_classes, args.bottleneck_dim).to(device)
F2 = ImageClassifierHead(G.out_features, num_classes, args.bottleneck_dim).to(device)
# define optimizer
# the learning rate is fixed according to origin paper
optimizer_g = SGD(G.parameters(), lr=args.lr, weight_decay=0.0005)
optimizer_f = SGD([
{"params": F1.parameters()},
{"params": F2.parameters()},
], momentum=0.9, lr=args.lr, weight_decay=0.0005)
# resume from the best checkpoint
if args.phase != 'train':
checkpoint = torch.load(logger.get_checkpoint_path('best'), map_location='cpu')
G.load_state_dict(checkpoint['G'])
F1.load_state_dict(checkpoint['F1'])
F2.load_state_dict(checkpoint['F2'])
# analysis the model
if args.phase == 'analysis':
# extract features from both domains
feature_extractor = G.to(device)
source_feature = collect_feature(train_source_loader, feature_extractor, device)
target_feature = collect_feature(train_target_loader, feature_extractor, device)
# plot t-SNE
tSNE_filename = osp.join(logger.visualize_directory, 'TSNE.png')
tsne.visualize(source_feature, target_feature, tSNE_filename)
print("Saving t-SNE to", tSNE_filename)
# calculate A-distance, which is a measure for distribution discrepancy
A_distance = a_distance.calculate(source_feature, target_feature, device)
print("A-distance =", A_distance)
return
if args.phase == 'test':
acc1 = validate(test_loader, G, F1, F2, args)
print(acc1)
return
# start training
best_acc1 = 0.
best_results = None
for epoch in range(args.epochs):
# train for one epoch
train(train_source_iter, train_target_iter, G, F1, F2, optimizer_g, optimizer_f, epoch, args)
# evaluate on validation set
results = validate(val_loader, G, F1, F2, args)
# remember best acc@1 and save checkpoint
torch.save({
'G': G.state_dict(),
'F1': F1.state_dict(),
'F2': F2.state_dict()
}, logger.get_checkpoint_path('latest'))
if max(results) > best_acc1:
shutil.copy(logger.get_checkpoint_path('latest'), logger.get_checkpoint_path('best'))
best_acc1 = max(results)
best_results = results
print("best_acc1 = {:3.1f}, results = {}".format(best_acc1, best_results))
# evaluate on test set
checkpoint = torch.load(logger.get_checkpoint_path('best'), map_location='cpu')
G.load_state_dict(checkpoint['G'])
F1.load_state_dict(checkpoint['F1'])
F2.load_state_dict(checkpoint['F2'])
results = validate(test_loader, G, F1, F2, args)
print("test_acc1 = {:3.1f}".format(max(results)))
logger.close()
model = load_checkpoint(model, args.scratch)
# model = torch.nn.DataParallel(model, device_ids=args.gpu_ids)
# Build SWALP model
if args.swa:
swa_model = models.__dict__[args.arch](dataset=args.dataset,
depth=args.depth,
cfg=checkpoint['cfg'])
swa_n = 0
swa_model.cuda()
if args.cuda:
model.cuda()
optimizer = SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# insert quantizations into the optimization loops
optimizer = OptimLP(optimizer,
weight_quant=quant_dict["weight"],
grad_quant=quant_dict["grad"],
momentum_quant=quant_dict["momentum"])
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
def main():
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
if not os.path.exists(args.outdir):
os.mkdir(args.outdir)
if (args.scale_down == 1 or args.dataset == "imagenet"):
train_dataset = get_dataset(args.dataset, 'train')
test_dataset = get_dataset(args.dataset, 'test')
else:
train_dataset = datasets.CIFAR10(
"./dataset_cache",
train=True,
download=True,
transform=transforms.Compose([
transforms.Resize(int(32 / args.scale_down)),
transforms.RandomCrop(int(32 / args.scale_down),
padding=int(4 / args.scale_down)),
transforms.RandomHorizontalFlip(),
transforms.ToTensor()
]))
test_dataset = datasets.CIFAR10("./dataset_cache",
train=False,
download=True,
transform=transforms.Compose([
transforms.Resize(
int(32 / args.scale_down)),
transforms.ToTensor()
]))
pin_memory = (args.dataset == "imagenet")
train_loader = DataLoader(train_dataset,
shuffle=True,
batch_size=args.batch,
num_workers=args.workers,
pin_memory=pin_memory)
test_loader = DataLoader(test_dataset,
shuffle=False,
batch_size=args.batch,
num_workers=args.workers,
pin_memory=pin_memory)
model = get_architecture(args.arch, args.dataset)
#model = torch.nn.DataParallel(model)
logfilename = os.path.join(args.outdir, 'log.txt')
init_logfile(logfilename,
"epoch\ttime\tlr\ttrain loss\ttrain acc\ttestloss\ttest acc")
criterion = CrossEntropyLoss()
optimizer = SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = StepLR(optimizer,
step_size=args.lr_step_size,
gamma=args.gamma)
for epoch in range(args.epochs):
before = time.time()
train_loss, train_acc = train(train_loader, model, criterion,
optimizer, epoch, args.noise_sd)
test_loss, test_acc = test(test_loader, model, criterion,
args.noise_sd)
after = time.time()
log(
logfilename, "{}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}".format(
epoch, str(datetime.timedelta(seconds=(after - before))),
scheduler.get_lr()[0], train_loss, train_acc, test_loss,
test_acc))
scheduler.step(epoch)
torch.save(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}, os.path.join(args.outdir, 'checkpoint.pth.tar'))
LAYERS = (3, 4, 23, 3)
if __name__ == "__main__":
# loss and network
from tests.python.common import data_loader
criterion = nn.CrossEntropyLoss()
net = ResNet14(in_channels=3, out_channels=5, D=2)
print(net)
# a data loader must return a tuple of coords, features, and labels.
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
net = net.to(device)
optimizer = SGD(net.parameters(), lr=1e-2)
for i in range(10):
optimizer.zero_grad()
# Get new data
coords, feat, label = data_loader()
input = ME.SparseTensor(feat, coords, device=device)
label = label.to(device)
# Forward
output = net(input)
# Loss
loss = criterion(output.F, label)
print("Iteration: ", i, ", Loss: ", loss.item())
def _optimizer(self, name, model):
m = {
'adam': Adam(model.parameters()),
'sgd': SGD(model.parameters(), lr=0.5)
}
return m[name]
eval_root = args.e
eval_indices = args.ei
eval_files = args.ef
logfile = args.l
save = args.w
best = args.b
force_cuda = args.nc
# Construct model, optimizer, and criterion
vgg16_base = vgg16(pretrained=True)
vgg16_base.classifier[6] = Linear(4096, 120)
if force_cuda and cuda.is_available():
vgg16_base = vgg16_base.cuda()
sgd = SGD(vgg16_base.parameters(),
lr=.001,
momentum=.9,
weight_decay=.0005)
crit = CrossEntropyLoss()
# Construct dataset variables
image_resolution = (300, 300)
# Construct training dataset and loader
train_transform = Compose([
Lambda(maybe_blur),
Lambda(maybe_darken_a_lot),
Lambda(maybe_rotate),
Lambda(maybe_random_perspective),
Lambda(maybe_random_crop),
Lambda(maybe_random_erase),
ColorJitter(brightness=(.1, .8),
def main(self):
log.info("Starting {}, {}".format(type(self).__name__, self.cli_args))
self.use_cuda = torch.cuda.is_available()
self.device = torch.device("cuda" if self.use_cuda else "cpu")
self.model = LunaModel()
if self.use_cuda:
if torch.cuda.device_count() > 1:
self.model = nn.DataParallel(self.model)
self.model = self.model.to(self.device)
self.optimizer = SGD(self.model.parameters(), lr=0.01, momentum=0.9)
train_dl = DataLoader(
LunaDataset(
test_stride=10,
isTestSet_bool=False,
),
batch_size=self.cli_args.batch_size * (torch.cuda.device_count() if self.use_cuda else 1),
num_workers=self.cli_args.num_workers,
pin_memory=self.use_cuda,
)
test_dl = DataLoader(
LunaDataset(
test_stride=10,
isTestSet_bool=True,
),
batch_size=self.cli_args.batch_size * (torch.cuda.device_count() if self.use_cuda else 1),
num_workers=self.cli_args.num_workers,
pin_memory=self.use_cuda,
)
for epoch_ndx in range(1, self.cli_args.epochs + 1):
log.info("Epoch {} of {}, {}/{} batches of size {}*{}".format(
epoch_ndx,
self.cli_args.epochs,
len(train_dl),
len(test_dl),
self.cli_args.batch_size,
(torch.cuda.device_count() if self.use_cuda else 1),
))
# Training loop, very similar to below
self.model.train()
trainingMetrics_tensor = torch.zeros(3, len(train_dl.dataset), 1)
batch_iter = enumerateWithEstimate(
train_dl,
"E{} Training".format(epoch_ndx),
start_ndx=train_dl.num_workers,
)
for batch_ndx, batch_tup in batch_iter:
self.optimizer.zero_grad()
loss_var = self.computeBatchLoss(batch_ndx, batch_tup, train_dl.batch_size, trainingMetrics_tensor)
loss_var.backward()
self.optimizer.step()
del loss_var
# Testing loop, very similar to above, but simplified
with torch.no_grad():
self.model.eval()
testingMetrics_tensor = torch.zeros(3, len(test_dl.dataset), 1)
batch_iter = enumerateWithEstimate(
test_dl,
"E{} Testing ".format(epoch_ndx),
start_ndx=test_dl.num_workers,
)
for batch_ndx, batch_tup in batch_iter:
self.computeBatchLoss(batch_ndx, batch_tup, test_dl.batch_size, testingMetrics_tensor)
self.logMetrics(epoch_ndx, trainingMetrics_tensor, testingMetrics_tensor)
class LunaTrainingApp(object):
def __init__(self, sys_argv=None):
if sys_argv is None:
sys_argv = sys.argv[1:]
parser = argparse.ArgumentParser()
parser.add_argument('--batch-size',
help='Batch size to use for training',
default=32,
type=int,
)
parser.add_argument('--num-workers',
help='Number of worker processes for background data loading',
default=8,
type=int,
)
parser.add_argument('--epochs',
help='Number of epochs to train for',
default=1,
type=int,
)
self.cli_args = parser.parse_args(sys_argv)
self.time_str = datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
def main(self):
log.info("Starting {}, {}".format(type(self).__name__, self.cli_args))
self.use_cuda = torch.cuda.is_available()
self.device = torch.device("cuda" if self.use_cuda else "cpu")
self.model = LunaModel()
if self.use_cuda:
if torch.cuda.device_count() > 1:
self.model = nn.DataParallel(self.model)
self.model = self.model.to(self.device)
self.optimizer = SGD(self.model.parameters(), lr=0.01, momentum=0.9)
train_dl = DataLoader(
LunaDataset(
test_stride=10,
isTestSet_bool=False,
),
batch_size=self.cli_args.batch_size * (torch.cuda.device_count() if self.use_cuda else 1),
num_workers=self.cli_args.num_workers,
pin_memory=self.use_cuda,
)
test_dl = DataLoader(
LunaDataset(
test_stride=10,
isTestSet_bool=True,
),
batch_size=self.cli_args.batch_size * (torch.cuda.device_count() if self.use_cuda else 1),
num_workers=self.cli_args.num_workers,
pin_memory=self.use_cuda,
)
for epoch_ndx in range(1, self.cli_args.epochs + 1):
log.info("Epoch {} of {}, {}/{} batches of size {}*{}".format(
epoch_ndx,
self.cli_args.epochs,
len(train_dl),
len(test_dl),
self.cli_args.batch_size,
(torch.cuda.device_count() if self.use_cuda else 1),
))
# Training loop, very similar to below
self.model.train()
trainingMetrics_tensor = torch.zeros(3, len(train_dl.dataset), 1)
batch_iter = enumerateWithEstimate(
train_dl,
"E{} Training".format(epoch_ndx),
start_ndx=train_dl.num_workers,
)
for batch_ndx, batch_tup in batch_iter:
self.optimizer.zero_grad()
loss_var = self.computeBatchLoss(batch_ndx, batch_tup, train_dl.batch_size, trainingMetrics_tensor)
loss_var.backward()
self.optimizer.step()
del loss_var
# Testing loop, very similar to above, but simplified
with torch.no_grad():
self.model.eval()
testingMetrics_tensor = torch.zeros(3, len(test_dl.dataset), 1)
batch_iter = enumerateWithEstimate(
test_dl,
"E{} Testing ".format(epoch_ndx),
start_ndx=test_dl.num_workers,
)
for batch_ndx, batch_tup in batch_iter:
self.computeBatchLoss(batch_ndx, batch_tup, test_dl.batch_size, testingMetrics_tensor)
self.logMetrics(epoch_ndx, trainingMetrics_tensor, testingMetrics_tensor)
def computeBatchLoss(self, batch_ndx, batch_tup, batch_size, metrics_tensor):
input_tensor, label_tensor, _series_list, _center_list = batch_tup
input_devtensor = input_tensor.to(self.device)
label_devtensor = label_tensor.to(self.device)
prediction_devtensor = self.model(input_devtensor)
loss_devtensor = nn.MSELoss(reduction='none')(prediction_devtensor, label_devtensor)
start_ndx = batch_ndx * batch_size
end_ndx = start_ndx + label_tensor.size(0)
metrics_tensor[METRICS_LABEL_NDX, start_ndx:end_ndx] = label_tensor
metrics_tensor[METRICS_PRED_NDX, start_ndx:end_ndx] = prediction_devtensor.to('cpu')
metrics_tensor[METRICS_LOSS_NDX, start_ndx:end_ndx] = loss_devtensor.to('cpu')
# TODO: replace with torch.autograd.detect_anomaly
# assert np.isfinite(metrics_tensor).all()
return loss_devtensor.mean()
def logMetrics(self,
epoch_ndx,
trainingMetrics_tensor,
testingMetrics_tensor,
classificationThreshold_float=0.5,
):
log.info("E{} {}".format(
epoch_ndx,
type(self).__name__,
))
for mode_str, metrics_tensor in [('trn', trainingMetrics_tensor), ('tst', testingMetrics_tensor)]:
metrics_ary = metrics_tensor.detach().numpy()[:,:,0]
assert np.isfinite(metrics_ary).all()
benLabel_mask = metrics_ary[METRICS_LABEL_NDX] <= classificationThreshold_float
benPred_mask = metrics_ary[METRICS_PRED_NDX] <= classificationThreshold_float
malLabel_mask = ~benLabel_mask
malPred_mask = ~benPred_mask
benLabel_count = benLabel_mask.sum()
malLabel_count = malLabel_mask.sum()
benCorrect_count = (benLabel_mask & benPred_mask).sum()
malCorrect_count = (malLabel_mask & malPred_mask).sum()
metrics_dict = {}
metrics_dict['loss/all'] = metrics_ary[METRICS_LOSS_NDX].mean()
metrics_dict['loss/ben'] = metrics_ary[METRICS_LOSS_NDX, benLabel_mask].mean()
metrics_dict['loss/mal'] = metrics_ary[METRICS_LOSS_NDX, malLabel_mask].mean()
metrics_dict['correct/all'] = (malCorrect_count + benCorrect_count) / metrics_ary.shape[1] * 100
metrics_dict['correct/ben'] = (benCorrect_count) / benLabel_count * 100
metrics_dict['correct/mal'] = (malCorrect_count) / malLabel_count * 100
log.info(("E{} {:8} "
+ "{loss/all:.4f} loss, "
+ "{correct/all:-5.1f}% correct"
).format(
epoch_ndx,
mode_str,
**metrics_dict,
))
log.info(("E{} {:8} "
+ "{loss/ben:.4f} loss, "
+ "{correct/ben:-5.1f}% correct").format(
epoch_ndx,
mode_str + '_ben',
**metrics_dict,
))
log.info(("E{} {:8} "
+ "{loss/mal:.4f} loss, "
+ "{correct/mal:-5.1f}% correct").format(
epoch_ndx,
mode_str + '_mal',
**metrics_dict,
))
def init_optimizer(self, classifier_module):
return SGD(classifier_module().parameters(), lr=0.05)
train_dataloader = DataLoader(
dataset = VGGFace2Dataset(
training_triplets_path = triplets_path,
data_dir = data_directory,
transform = data_transforms
),
batch_size = 1,
shuffle = False
)
net = Recog_Net()
net.cuda()
margin = 0.05
l2_distance = PairwiseDistance(2).cuda()
optimizer_model = SGD(net.parameters(), lr=0.1)
for epoch in range(0,5):
triplet_loss_sum = 0
num_valid_training_triplets = 0
progress_bar = enumerate(tqdm(train_dataloader))
for batch_idx, (batch_sample) in progress_bar:
anc_image = batch_sample['anc_img'].view(-1, 3, 220, 220).cuda()
pos_image = batch_sample['pos_img'].view(-1, 3, 220, 220).cuda()
neg_image = batch_sample['neg_img'].view(-1, 3, 220, 220).cuda()
anc_embedding = net(anc_image)
pos_embedding = net(pos_image)
neg_embedding = net(neg_image)
json.dump(dataset, handle, indent=1)
print('Written dataset to disk.')
print('Reading dataset into memory.')
with open(dataset_file_path, 'r') as handle:
dataset_meta = json.load(handle)
dataset = ConstructionSequenceDataset(dataset_meta['construction_sequences'],
device=device)
print('Dataset has size {len}.'.format(len=len(dataset)))
"""
Instantiating new model
"""
model = DeepGG(v_max=param_deepgg_v_max,
node_hidden_size=param_deepgg_node_hidden_size,
num_prop_rounds=param_deepgg_prop_rounds)
optimizer = SGD(model.parameters(), lr=param_learning_rate)
model.to(device)
model.train()
"""
Training loop
"""
print('Starting training.', flush=True)
time.sleep(0.1)
ts_losses = []
with tqdm(total=len(dataset) * param_train_epochs) as pbar:
for cur_epoch in range(param_train_epochs):
for seq_idx, seq in enumerate(dataset):
#train_seq = torch.tensor(seq, device=device)
train_seq = seq
train_img, train_label = DatasetFor5layer.getTrain()
train_label = torch.from_numpy(train_label)
val_img, val_label = DatasetFor5layer.getVal()
val_label = torch.from_numpy(val_label)
test_img = DatasetFor5layer.getTest()
model = my_model.Net()
optimizer = Adam(model.parameters(), lr=1e-3)
if key == '2':
train_img, train_label = Dataset.getTrain()
train_label = torch.from_numpy(train_label)
val_img, val_label = Dataset.getVal()
val_label = torch.from_numpy(val_label)
test_img = Dataset.getTest()
model = models.resnet18(pretrained=False, num_classes=2)
optimizer = SGD(model.parameters(), lr=1e-2)
if key == '3':
train_img, train_label = Dataset.getTrain()
train_label = torch.from_numpy(train_label)
val_img, val_label = Dataset.getVal()
val_label = torch.from_numpy(val_label)
test_img = Dataset.getTest()
model = models.resnet18(pretrained=True)
model.fc = Linear(in_features=512, out_features=2)
for para in list(model.parameters())[:-2]:
para.requires_grad = False
optimizer = Adam(params=[model.fc.weight, model.fc.bias], lr=1e-3)
print('the training layer is:')
for name, param in model.named_parameters(): # 查看可优化的参数有哪些
if param.requires_grad:
def main():
# Create output directory
path_output = './checkpoints/'
if not os.path.exists(path_output):
os.makedirs(path_output)
# Hyperparameters, to change
epochs = 50
batch_size = 24
alpha = 1 # it's the trade-off parameter of loss function, what values should it take?
gamma = 1
# Source domains name
save_interval = 10 # save every 10 epochs
root = 'data/'
source1 = 'sketch'
source2 = 'sketch'
source3 = 'sketch'
target = 'quickdraw'
# Dataloader
dataset_s1 = dataset.DA(dir=root,
name=source1,
img_size=(224, 224),
train=True)
dataset_s2 = dataset.DA(dir=root,
name=source2,
img_size=(224, 224),
train=True)
dataset_s3 = dataset.DA(dir=root,
name=source3,
img_size=(224, 224),
train=True)
dataset_t = dataset.DA(dir=root,
name=target,
img_size=(224, 224),
train=True)
dataset_val = dataset.DA(dir=root,
name=target,
img_size=(224, 224),
train=False,
real_val=False)
dataloader_s1 = DataLoader(dataset_s1,
batch_size=batch_size,
shuffle=True,
num_workers=2)
dataloader_s2 = DataLoader(dataset_s2,
batch_size=batch_size,
shuffle=True,
num_workers=2)
dataloader_s3 = DataLoader(dataset_s3,
batch_size=batch_size,
shuffle=True,
num_workers=2)
dataloader_t = DataLoader(dataset_t,
batch_size=batch_size,
shuffle=True,
num_workers=2)
dataloader_val = DataLoader(dataset_val,
batch_size=batch_size,
shuffle=False,
num_workers=2)
len_data = min(len(dataset_s1), len(dataset_s2), len(dataset_s3),
len(dataset_t)) # length of "shorter" domain
len_dataloader = min(len(dataloader_s1), len(dataloader_s2),
len(dataloader_s3), len(dataloader_t))
# Define networks
feature_extractor = models.feature_extractor()
classifier_1 = models.class_classifier()
classifier_2 = models.class_classifier()
classifier_3 = models.class_classifier()
classifier_1.apply(weight_init)
classifier_2.apply(weight_init)
classifier_3.apply(weight_init)
discriminator_1 = models.discriminator()
discriminator_1.apply(weight_init)
if torch.cuda.is_available():
feature_extractor = feature_extractor.cuda()
classifier_1 = classifier_1.cuda()
classifier_2 = classifier_2.cuda()
classifier_3 = classifier_3.cuda()
discriminator_1 = discriminator_1.cuda()
# discriminator_2 = discriminator_2.cuda()
# discriminator_3 = discriminator_3.cuda()
# Define loss
# mom_loss = momentumLoss.Loss()
cl_loss = nn.CrossEntropyLoss()
disc_loss = nn.NLLLoss()
# Optimizers
# Change the LR
optimizer_features = SGD(feature_extractor.parameters(),
lr=0.0001,
momentum=0.9,
weight_decay=5e-4)
optimizer_classifier = SGD(([{
'params': classifier_1.parameters()
}, {
'params': classifier_2.parameters()
}, {
'params': classifier_3.parameters()
}]),
lr=0.002,
momentum=0.9,
weight_decay=5e-4)
optimizer_discriminator = SGD(([
{
'params': discriminator_1.parameters()
},
]),
lr=0.002,
momentum=0.9,
weight_decay=5e-4)
# Lists
train_loss = []
acc_on_target = []
best_acc = 0.0
w1_mean = 0.0
w2_mean = 0.0
w3_mean = 0.0
for epoch in range(epochs):
epochTic = timeit.default_timer()
tot_loss = 0.0
feature_extractor.train()
classifier_1.train(), classifier_2.train(), classifier_3.train()
if epoch + 1 == 5:
optimizer_classifier = SGD(([{
'params': classifier_1.parameters()
}, {
'params': classifier_2.parameters()
}, {
'params': classifier_3.parameters()
}]),
lr=0.001,
momentum=0.9,
weight_decay=5e-4)
optimizer_discriminator = SGD(
([{
'params': discriminator_1.parameters()
}]),
lr=0.001,
momentum=0.9,
weight_decay=5e-4)
if epoch + 1 == 10:
optimizer_classifier = SGD(([{
'params': classifier_1.parameters()
}, {
'params': classifier_2.parameters()
}, {
'params': classifier_3.parameters()
}]),
lr=0.0001,
momentum=0.9,
weight_decay=5e-4)
optimizer_discriminator = SGD(
([{
'params': discriminator_1.parameters()
}]),
lr=0.0001,
momentum=0.9,
weight_decay=5e-4)
print('*************************************************')
for i, (data_1, data_2, data_3, data_t) in enumerate(
zip(dataloader_s1, dataloader_s2, dataloader_s3,
dataloader_t)):
p = float(i + epoch * len_data) / epochs / len_data
alpha = 2. / (1. + np.exp(-10 * p)) - 1
img1, lb1 = data_1
img2, lb2 = data_2
img3, lb3 = data_3
imgt, _ = data_t
# Prepare data
cur_batch = min(img1.shape[0], img2.shape[0], img3.shape[0],
imgt.shape[0])
img1, lb1 = Variable(img1[0:cur_batch, :, :, :]).cuda(), Variable(
lb1[0:cur_batch]).cuda()
img2, lb2 = Variable(img2[0:cur_batch, :, :, :]).cuda(), Variable(
lb2[0:cur_batch]).cuda()
img3, lb3 = Variable(img3[0:cur_batch, :, :, :]).cuda(), Variable(
lb3[0:cur_batch]).cuda()
imgt = Variable(imgt[0:cur_batch, :, :, :]).cuda()
# Forward
optimizer_features.zero_grad()
optimizer_classifier.zero_grad()
optimizer_discriminator.zero_grad()
# Extract Features
ft1 = feature_extractor(img1)
ft2 = feature_extractor(img2)
ft3 = feature_extractor(img3)
ft_t = feature_extractor(imgt)
# Train the discriminator
ds_s1 = discriminator_1(torch.cat((ft1, ft2, ft3)), alpha)
ds_t = discriminator_1(ft_t, alpha)
# Class Prediction
cl1 = classifier_1(ft1)
cl2 = classifier_2(ft2)
cl3 = classifier_3(ft3)
# Compute the "discriminator loss"
ds_label = torch.zeros(cur_batch * 3).long()
dt_label = torch.ones(cur_batch).long()
d_s = disc_loss(ds_s1, ds_label.cuda())
d_t = disc_loss(ds_t, dt_label.cuda())
# Compute "momentum loss"
# loss_mom = mom_loss(ft1, ft2, ft3, ft_t)
# Cross entropy loss
l1 = cl_loss(cl1, lb1)
l2 = cl_loss(cl2, lb2)
l3 = cl_loss(cl3, lb3)
# Classifier Weight
total_class_loss = 1 / l1 + 1 / l2 + 1 / l3
w1 = (1 / l1) / total_class_loss
w2 = (1 / l2) / total_class_loss
w3 = (1 / l3) / total_class_loss
w1_mean += w1
w2_mean += w2
w3_mean += w3
# total loss
# loss = l1 + l2 + l3 + alpha * loss_mom + gamma * (d_l1 + d_l2 + d_l3)
loss = l1 + l2 + l3 + gamma * (d_s + d_t)
loss.backward()
optimizer_features.step()
optimizer_classifier.step()
optimizer_discriminator.step()
tot_loss += loss.item() * cur_batch
# Progress indicator
print('\rTraining... Progress: %.1f %%' %
(100 * (i + 1) / len_dataloader),
end='')
tot_t_loss = tot_loss / (len_data)
w1_mean /= len_dataloader
w2_mean /= len_dataloader
w3_mean /= len_dataloader
print(w1_mean, w2_mean, w3_mean)
# Print
train_loss.append(tot_t_loss)
print('\rEpoch [%d/%d], Training loss: %.4f' %
(epoch + 1, epochs, tot_t_loss),
end='\n')
####################################################################################################################
# Compute the accuracy at the end of each epoch
feature_extractor.eval()
classifier_1.eval(), classifier_2.eval(), classifier_3.eval()
discriminator_1.eval()
tot_acc = 0
with torch.no_grad():
for i, (imgt, lbt) in enumerate(dataloader_val):
cur_batch = imgt.shape[0]
imgt = imgt.cuda()
lbt = lbt.cuda()
# Forward the test images
ft_t = feature_extractor(imgt)
pred1 = classifier_1(ft_t)
pred2 = classifier_2(ft_t)
pred3 = classifier_3(ft_t)
# e1 = discriminator_1(ft_t, alpha)[:,0].data.cpu().numpy()
# e2 = discriminator_1(ft_t, alpha)[:,0].data.cpu().numpy()
# e3 = discriminator_1(ft_t, alpha)[:,0].data.cpu().numpy()
# a1 = np.exp(e1) / (np.exp(e1)+np.exp(e2)+np.exp(e3))
# a2 = np.exp(e2) / (np.exp(e1) + np.exp(e2) + np.exp(e3))
# a3 = np.exp(e3) / (np.exp(e1) + np.exp(e2) + np.exp(e3))
# a1 = torch.Tensor(a1).unsqueeze(1).repeat(1, 345).cuda()
# a2 = torch.Tensor(a2).unsqueeze(1).repeat(1, 345).cuda()
# a3 = torch.Tensor(a3).unsqueeze(1).repeat(1, 345).cuda()
# Compute accuracy
# output = pred1*a1 + pred2*a2 + pred3*a3
output = pred1 * w1_mean + pred2 * w2_mean + pred3 * w3_mean
_, pred = torch.max(output, dim=1)
correct = pred.eq(lbt.data.view_as(pred))
accuracy = torch.mean(correct.type(torch.FloatTensor))
tot_acc += accuracy.item() * cur_batch
# Progress indicator
print('\rValidation... Progress: %.1f %%' %
(100 * (i + 1) / len(dataloader_val)),
end='')
tot_t_acc = tot_acc / (len(dataset_val))
# Print
acc_on_target.append(tot_t_acc)
print('\rEpoch [%d/%d], Accuracy on target: %.4f' %
(epoch + 1, epochs, tot_t_acc),
end='\n')
# Save every save_interval
if best_acc < tot_t_acc:
torch.save(
{
'epoch':
epoch,
'feature_extractor':
feature_extractor.state_dict(),
'{}_classifier'.format(source1):
classifier_1.state_dict(),
'{}_classifier'.format(source2):
classifier_2.state_dict(),
'{}_classifier'.format(source3):
classifier_3.state_dict(),
'{}_discriminator'.format(source1):
discriminator_1.state_dict(),
# '{}_discriminator'.format(source2): discriminator_2.state_dict(),
# '{}_discriminator'.format(source3): discriminator_3.state_dict(),
'features_optimizer':
optimizer_features.state_dict(),
'classifier_optimizer':
optimizer_classifier.state_dict(),
'loss':
tot_loss,
'{}_weight'.format(source1):
w1_mean,
'{}_weight'.format(source2):
w2_mean,
'{}_weight'.format(source3):
w3_mean,
},
os.path.join(path_output, target + '-{}.pth'.format(epoch)))
print('Saved best model!')
best_acc = tot_t_acc
# Pirnt elapsed time per epoch
epochToc = timeit.default_timer()
(t_min, t_sec) = divmod((epochToc - epochTic), 60)
print('Elapsed time is: %d min: %d sec' % (t_min, t_sec))
# Save training loss and accuracy on target (if not 'real')
pkl.dump(train_loss, open('{}train_loss.p'.format(path_output), 'wb'))
pkl.dump(acc_on_target,
open('{}target_accuracy.p'.format(path_output), 'wb'))
# Send notification
subjcet = 'Epoch {} in train_weights.py'.format(epoch + 1)
content = (
'Accuracy on {} : %.4f \nTraining Loss: %.4f \n{}_weight: %.4f , {}_weight: %.4f , {}_weight: %.4f'
.format(target, source1, source2, source3) %
(tot_t_acc, tot_t_loss, w1_mean, w2_mean, w3_mean))
sendNotification(subject=subjcet, content=content)
])
train_dataset = dataset(train_x, train_y, True, transform, True)
test_dataset = dataset(test_x, None, False, transform, False)
train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=64, shuffle=True)
acc_loader = DataLoader(train_dataset, batch_size=100, shuffle=False)
num_epoch = 80
generator = MCD.generator().cuda()
classifier_1 = MCD.classifier().cuda()
classifier_2 = MCD.classifier().cuda()
criterion = nn.CrossEntropyLoss().cuda()
optimizer_generator = SGD(generator.parameters(),
lr=0.0025,
weight_decay=0.0005,
momentum=0.9)
optimizer_classifier_1 = SGD(classifier_1.parameters(),
lr=0.0025,
weight_decay=0.0005,
momentum=0.9)
optimizer_classifier_2 = SGD(classifier_2.parameters(),
lr=0.0025,
weight_decay=0.0005,
momentum=0.9)
for epoch in range(num_epoch):
train_acc = 0
start = time()
generator.train()
classifier_1.train()
def main():
ENOT_HOME_DIR.mkdir(exist_ok=True)
ENOT_DATASETS_DIR.mkdir(exist_ok=True)
PROJECT_DIR.mkdir(exist_ok=True)
prepare_log(PROJECT_DIR / 'experiments' / 'multigpu_example_2x2_config_v1')
init_torch(cuda_optimize_for_speed=True)
distributed = is_dist()
n_workers = get_world_size()
dataloaders = create_imagenette_dataloaders(
ENOT_DATASETS_DIR,
PROJECT_DIR,
input_size=(224, 224),
batch_size=64,
num_workers=4, # Each spawned process in single node will use this number of workers.
dist=distributed, # Flag to use DistributedSampler to sample different images in different worker processes.
)
# Building search space to pretrain.
model = build_mobilenet(
search_ops=SEARCH_OPS,
num_classes=10,
blocks_out_channels=[24, 32, 64, 96, 160, 320],
blocks_count=[2, 3, 4, 3, 3, 1],
blocks_stride=[2, 2, 2, 1, 2, 1],
)
search_space = SearchSpaceModel(model).cuda() # We do not wrap model with DistributedDataParallel.
# Synchronize search space across workers.
sync_model(search_space, reduce_parameters=False, reduce_buffers=False)
# Log the total size of gradients to transfer.
if is_local_master():
total_gradient_bytes = sum([x.element_size() * x.nelement() for x in search_space.model_parameters()])
total_gradient_megabytes = total_gradient_bytes / (1024 * 1024)
logging.info(f'Gradients to transfer (in megabytes): {total_gradient_megabytes:.3f}Mb')
train_loader = dataloaders['pretrain_train_dataloader']
len_train_loader = len(train_loader)
# Dataloader for master-only validation.
if is_master():
dataloaders = create_imagenette_dataloaders(
ENOT_DATASETS_DIR,
PROJECT_DIR,
input_size=(224, 224),
batch_size=64,
num_workers=8, # More workers for faster validation.
dist=False, # We only validate in master process, so this dataloader is master-only.
)
validation_loader = dataloaders['pretrain_validation_dataloader']
validation_len = len(validation_loader)
# We should use ``search_space.model_parameters()`` parameters in pre-train phase.
optimizer = SGD(params=search_space.model_parameters(), lr=LR * n_workers, momentum=0.9, weight_decay=1e-4)
# Wrap regular optimizer with ``EnotPretrainOptimizer``, and use it later.
enot_optimizer = EnotPretrainOptimizer(search_space=search_space, optimizer=optimizer)
scheduler = CosineAnnealingLR(optimizer, T_max=len_train_loader * N_EPOCHS)
scheduler = WarmupScheduler(scheduler, warmup_steps=len_train_loader * N_WARMUP_EPOCHS)
metric_function = accuracy # Change if you want.
loss_function = nn.CrossEntropyLoss().cuda()
if is_local_master(): # Fancy logging output.
logging.info('')
for epoch in range(N_EPOCHS):
# Setting current epoch in DistributedSampler, see it's documentation:
# https://pytorch.org/docs/stable/data.html, torch.utils.data.distributed.DistributedSampler class.
if distributed:
train_loader.sampler.set_epoch(epoch)
if is_local_master():
logging.info(f'EPOCH #{epoch}')
search_space.train()
train_metrics_acc = {
'loss': 0.0,
'accuracy': 0.0,
'n': 0,
}
train_queue = extract_data_from_queue(
train_loader,
data_parallel=True,
use_tqdm=is_local_master(), # Show tqdm bar only in local master process.
)
for num_sample, inputs, labels in train_queue:
with torch.no_grad(): # Initialize output distribution optimization.
if not search_space.output_distribution_optimization_enabled:
search_space.initialize_output_distribution_optimization(inputs)
enot_optimizer.zero_grad()
# Executable closure with forward-backward passes.
def closure():
pred_labels = search_space(inputs)
batch_loss = loss_function(pred_labels, labels)
batch_loss.backward()
batch_metric = metric_function(pred_labels, labels)
train_metrics_acc['loss'] += batch_loss.item()
train_metrics_acc['accuracy'] += batch_metric.item()
train_metrics_acc['n'] += 1
enot_optimizer.step(closure) # Performing enot optimizer step, which internally calls closure.
if scheduler is not None:
scheduler.step()
if is_local_master(): # Log training stats in each local master.
train_loss = train_metrics_acc['loss'] / train_metrics_acc['n']
train_accuracy = train_metrics_acc['accuracy'] / train_metrics_acc['n']
if scheduler is not None:
logging.info(f'lr: {scheduler.get_lr()[0]:.4f}')
logging.info('Train metrics:')
logging.info(f' loss: {train_loss:.4f}')
logging.info(f' accuracy: {train_accuracy:.2f}')
if is_master(): # Validate only in master process.
search_space.eval()
validation_loss = 0
validation_accuracy = 0
validation_queue = extract_data_from_queue(
validation_loader,
data_parallel=True,
use_tqdm=True,
)
for num_sample, inputs, labels in validation_queue:
search_space.sample_random_arch()
with torch.no_grad():
val_predictions = search_space(inputs)
val_batch_loss = loss_function(val_predictions, labels)
val_batch_metric = metric_function(val_predictions, labels)
validation_loss += val_batch_loss.item()
validation_accuracy += val_batch_metric.item()
validation_loss /= validation_len
validation_accuracy /= validation_len
logging.info('Validation metrics:')
logging.info(f' loss: {validation_loss:.4f}')
logging.info(f' accuracy: {validation_accuracy:.2f}')
if is_local_master(): # Fancy logging output.
logging.info('')
def sacred_main(_run: Run, seed, showoff, batch_size, model_desc, deterministic, train_datasets,
lr_min, lr_max, max_iters, ema_beta, weight_decay, momentum):
seed_all(seed)
init_algorithms(deterministic=deterministic)
model = create_model(model_desc).to(global_opts['device'])
data_loader = create_train_dataloader(train_datasets, model.data_specs, batch_size,
examples_per_epoch=(max_iters * batch_size))
data_iter = iter(data_loader)
print(json.dumps(model_desc, sort_keys=True, indent=2))
def do_training_iteration(optimiser):
batch = next(data_iter)
in_var = batch['input'].to(global_opts['device'], torch.float32)
target_var = batch['target'].to(global_opts['device'], torch.float32)
mask_var = batch['joint_mask'].to(global_opts['device'], torch.float32)
# Calculate predictions and loss
out_var = model(in_var)
loss = forward_loss(model, out_var, target_var, mask_var, batch['valid_depth'])
# Calculate gradients
optimiser.zero_grad()
loss.backward()
# Update parameters
optimiser.step()
return loss.item()
optimiser = SGD(model.parameters(), lr=1, weight_decay=weight_decay, momentum=momentum)
tel = tele.Telemetry({
'config': ValueMeter(skip_reset=True),
'host_info': ValueMeter(skip_reset=True),
'loss_lr_fig': ValueMeter(),
})
notebook = None
if showoff:
title = 'Hyperparameter search ({}@{})'.format(model_desc['type'], model_desc['version'])
notebook = create_showoff_notebook(title, ['lrfinder'])
from tele.showoff import views
tel.sink(tele.showoff.Conf(notebook), [
views.Inspect(['config'], 'Experiment configuration', flatten=True),
views.Inspect(['host_info'], 'Host information', flatten=True),
views.FrameContent(['loss_lr_fig'], 'Loss vs learning rate graph', 'plotly'),
])
def set_progress(value):
if notebook is not None:
notebook.set_progress(value)
tel['config'].set_value(_run.config)
tel['host_info'].set_value(get_host_info())
lrs = np.geomspace(lr_min, lr_max, max_iters)
losses = []
avg_loss = 0
min_loss = np.inf
for i, lr in enumerate(tqdm(lrs, ascii=True)):
set_progress(i / len(lrs))
for param_group in optimiser.param_groups:
param_group['lr'] = lr
loss = do_training_iteration(optimiser)
avg_loss = ema_beta * avg_loss + (1 - ema_beta) * loss
smoothed_loss = avg_loss / (1 - ema_beta ** (i + 1))
if min_loss > 0 and smoothed_loss > 4 * min_loss:
break
min_loss = min(smoothed_loss, min_loss)
losses.append(smoothed_loss)
if i % 10 == 0:
fig = go.Figure(
data=[go.Scatter(x=lrs[:len(losses)].tolist(), y=losses, mode='lines')],
layout=go.Layout(
margin=go.Margin(l=60, r=40, b=80, t=20, pad=4),
xaxis=go.XAxis(title='Learning rate', type='log', exponentformat='power'),
yaxis=go.YAxis(title='Training loss'),
)
)
tel['loss_lr_fig'].set_value(fig)
tel.step()
set_progress(1)
ax1.axis('off')
ax1.set_xticklabels([])
ax1.set_yticklabels([])
ax1.set_title(str(allFilters - scores[f_num]))
plt.subplots_adjust(wspace=1.0, hspace=0.1)
plt.savefig(folderName + "_filters_l2.png")
plt.close()
#This code applies the maximal activation for each filter and then plots the resulting image.
fig = plt.figure(figsize=(num_cols, num_rows))
REGULARIZATION = 0.0001
for importance, f_num in enumerate(np.argsort(scores)):
print(importance)
im_as_var = Variable(torch_image.cuda(), requires_grad=True)
optimizer = SGD([im_as_var], lr=12, weight_decay=1e-4)
for i in range(1, 501):
optimizer.zero_grad()
x = im_as_var
first = [
model.module.first_layer(x[:, i, :, :].unsqueeze(1))
for i in range(x.shape[1])
]
x1 = torch.cat(first, 1)
second = [
model.module.second_layer[1](model.module.second_layer[0](
x1[:, i, :, :].unsqueeze(1))) for i in range(x1.shape[1])
]
x = torch.cat(second, 1)
#x = model.module.first_two_layers[0](x)
class Session:
def __init__(self, dt_split):
torch.manual_seed(66)
torch.cuda.manual_seed_all(66)
# torch.cuda.set_device(settings.DEVICE)
self.log_dir = settings.LOG_DIR
self.model_dir = settings.MODEL_DIR
ensure_dir(self.log_dir)
ensure_dir(self.model_dir)
logger.info('set log dir as %s' % self.log_dir)
logger.info('set model dir as %s' % self.model_dir)
self.step = 1
self.writer = SummaryWriter(osp.join(self.log_dir, 'train.events'))
dataset = TrainDataset(split=dt_split)
self.dataloader = DataLoader(
dataset, batch_size=settings.BATCH_SIZE, pin_memory=True,
num_workers=settings.NUM_WORKERS, shuffle=True, drop_last=True)
self.net = EMANet(settings.N_CLASSES, settings.N_LAYERS).cuda()
self.opt = SGD(
params=[
{
'params': get_params(self.net, key='1x'),
'lr': 1 * settings.LR,
'weight_decay': settings.WEIGHT_DECAY,
},
{
'params': get_params(self.net, key='1y'),
'lr': 1 * settings.LR,
'weight_decay': 0,
},
{
'params': get_params(self.net, key='2x'),
'lr': 2 * settings.LR,
'weight_decay': 0.0,
}],
momentum=settings.LR_MOM)
# self.net = DataParallel(self.net, device_ids=settings.DEVICES)
self.net = DataParallel(self.net)
patch_replication_callback(self.net)
def write(self, out):
for k, v in out.items():
self.writer.add_scalar(k, v, self.step)
out['lr'] = self.opt.param_groups[0]['lr']
out['step'] = self.step
outputs = [
'{}: {:.4g}'.format(k, v)
for k, v in out.items()]
logger.info(' '.join(outputs))
def save_checkpoints(self, name):
ckp_path = osp.join(self.model_dir, name)
obj = {
'net': self.net.module.state_dict(),
'step': self.step,
}
torch.save(obj, ckp_path)
def load_checkpoints(self, name):
ckp_path = osp.join(self.model_dir, name)
try:
obj = torch.load(ckp_path,
map_location=lambda storage, loc: storage.cuda())
logger.info('Load checkpoint %s' % ckp_path)
except FileNotFoundError:
logger.error('No checkpoint %s!' % ckp_path)
return
self.net.module.load_state_dict(obj['net'])
self.step = obj['step']
def train_batch(self, image, label):
loss, mu = self.net(image, label)
with torch.no_grad():
mu = mu.mean(dim=0, keepdim=True)
momentum = settings.EM_MOM
self.net.module.emau.mu *= momentum
self.net.module.emau.mu += mu * (1 - momentum)
loss = loss.mean()
self.opt.zero_grad()
loss.backward()
self.opt.step()
return loss.item()
class Word2Vec:
def __init__(self,
data_path,
vocabulary_size,
embedding_size,
learning_rate=1.0):
self.corpus = read_own_data(data_path)
self.data, self.word_count, self.word2index, self.index2word = build_dataset(
self.corpus, vocabulary_size)
self.vocabs = list(set(self.data))
self.model: SkipGramNeg = SkipGramNeg(vocabulary_size,
embedding_size).cuda()
self.model_optim = SGD(self.model.parameters(), lr=learning_rate)
def train(self,
train_steps,
skip_window=1,
num_skips=2,
num_neg=20,
batch_size=128,
data_offest=0,
vali_size=3,
output_dir='out'):
self.outputdir = os.mkdir(output_dir)
avg_loss = 0
pipeline = DataPipeline(self.data, self.vocabs, self.word_count,
data_offest)
vali_examples = random.sample(self.vocabs, vali_size)
for step in range(train_steps):
batch_inputs, batch_labels = pipeline.generate_batch(
batch_size, num_skips, skip_window)
batch_neg = pipeline.get_neg_data(batch_size, num_neg,
batch_inputs)
batch_inputs = torch.tensor(batch_inputs, dtype=torch.long).cuda()
batch_labels = torch.tensor(batch_labels, dtype=torch.long).cuda()
batch_neg = torch.tensor(batch_neg, dtype=torch.long).cuda()
loss = self.model(batch_inputs, batch_labels, batch_neg)
self.model_optim.zero_grad()
loss.backward()
self.model_optim.step()
avg_loss += loss.item()
if step % 2000 == 0 and step > 0:
avg_loss /= 2000
print('Average loss at step ', step, ': ', avg_loss)
avg_loss = 0
if step % 10000 == 0 and vali_size > 0:
nearest(self.model,
vali_examples,
vali_size,
self.index2word,
top_k=8)
# checkpoint
if step % 100000 == 0 and step > 0:
torch.save(self.model.state_dict(),
self.outputdir + '/model_step%d.pt' % step)
# save model at last
torch.save(self.model.state_dict(),
self.outputdir + '/model_step%d.pt' % train_steps)
def save_model(self, out_path):
torch.save(self.model.state_dict(), out_path + '/model.pt')
def get_list_vector(self):
sd = self.model.state_dict()
return sd['input_emb.weight'].tolist()
def save_vector_txt(self, path_dir):
embeddings = self.get_list_vector()
fo = open(path_dir + '/vector.txt', 'w')
for idx in range(len(embeddings)):
word = self.index2word[idx]
embed = embeddings[idx]
embed_list = [str(i) for i in embed]
line_str = ' '.join(embed_list)
fo.write(word + ' ' + line_str + '\n')
fo.close()
def load_model(self, model_path):
self.model.load_state_dict(torch.load(model_path))
def vector(self, index):
self.model.predict(index)
def most_similar(self, word, top_k=8):
index = self.word2index[word]
index = torch.tensor(index, dtype=torch.long).cuda().unsqueeze(0)
emb = self.model.predict(index)
sim = torch.mm(emb, self.model.input_emb.weight.transpose(0, 1))
nearest = (-sim[0]).sort()[1][1:top_k + 1]
top_list = []
for k in range(top_k):
close_word = self.index2word[nearest[k].item()]
top_list.append(close_word)
return top_list
def main(args):
# Config
device = torch.device(
f"cuda:{args.cuda}"
if torch.cuda.is_available() and args.cuda >= 0
else "cpu"
)
# model
class SimpleMLP(nn.Module):
def __init__(self, num_classes=10, input_size=28 * 28):
super(SimpleMLP, self).__init__()
self.features = nn.Sequential(
nn.Linear(input_size, 512),
nn.ReLU(inplace=True),
nn.Dropout(),
)
self.classifier = nn.Linear(512, num_classes)
self._input_size = input_size
def forward(self, x):
x = x.contiguous()
x = x.view(x.size(0), self._input_size)
x = self.features(x)
x = self.classifier(x)
return x
model = SimpleMLP(num_classes=10)
# CL Benchmark Creation
print("Creating the benchmark...")
list_train_dataset = []
list_test_dataset = []
rng_permute = np.random.RandomState(0)
train_transform = transforms.Compose(
[ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]
)
test_transform = transforms.Compose(
[ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]
)
# for every incremental experience
idx_permutations = []
for i in range(2):
idx_permutations.append(
torch.from_numpy(rng_permute.permutation(784)).type(torch.int64)
)
# add the permutation to the default dataset transformation
train_transform_list = train_transform.transforms.copy()
train_transform_list.append(
transforms.Lambda(
lambda x, i=i: x.view(-1)[idx_permutations[i]].view(1, 28, 28)
)
)
new_train_transform = transforms.Compose(train_transform_list)
test_transform_list = test_transform.transforms.copy()
test_transform_list.append(
transforms.Lambda(
lambda x, i=i: x.view(-1)[idx_permutations[i]].view(1, 28, 28)
)
)
new_test_transform = transforms.Compose(test_transform_list)
# get the datasets with the constructed transformation
permuted_train = MNIST(
root=expanduser("~") + "/.avalanche/data/mnist/",
train=True,
download=True,
transform=new_train_transform,
)
permuted_test = MNIST(
root=expanduser("~") + "/.avalanche/data/mnist/",
train=False,
download=True,
transform=new_test_transform,
)
list_train_dataset.append(permuted_train)
list_test_dataset.append(permuted_test)
# Train
optimizer = SGD(model.parameters(), lr=0.001, momentum=0.9)
criterion = CrossEntropyLoss()
model.to(device)
print("Starting training...")
for task_id, train_dataset in enumerate(list_train_dataset):
print("Starting task:", task_id)
train_data_loader = DataLoader(
train_dataset, batch_size=32, shuffle=False
)
for ep in range(1):
print("Epoch: ", ep)
for iteration, (train_mb_x, train_mb_y) in enumerate(
train_data_loader
):
optimizer.zero_grad()
train_mb_x = train_mb_x.to(device)
train_mb_y = train_mb_y.to(device)
# Forward
logits = model(train_mb_x)
# Loss
loss = criterion(logits, train_mb_y)
if iteration % 100 == 0:
print("Iter: {}, Loss: {}".format(iteration, loss.item()))
# Backward
loss.backward()
# Update
optimizer.step()
# Test
acc_results = []
print("Starting testing...")
for task_id, test_dataset in enumerate(list_test_dataset):
test_data_loader = DataLoader(test_dataset, batch_size=32)
correct = 0
for iteration, (test_mb_x, test_mb_y) in enumerate(
test_data_loader
):
# Move mini-batch data to device
test_mb_x = test_mb_x.to(device)
test_mb_y = test_mb_y.to(device)
# Forward
test_logits = model(test_mb_x)
preds = torch.argmax(test_logits.long(), dim=1)
# compute acc
correct += (test_mb_y.eq(preds)).sum().item()
print("Task:", task_id)
acc = (correct / len(test_dataset)) * 100
print("Accuracy results: ", acc)
acc_results.append(acc)
def train(train_source_iter: ForeverDataIterator, train_target_iter: ForeverDataIterator,
G: nn.Module, F1: ImageClassifierHead, F2: ImageClassifierHead,
optimizer_g: SGD, optimizer_f: SGD, epoch: int, args: argparse.Namespace):
batch_time = AverageMeter('Time', ':3.1f')
data_time = AverageMeter('Data', ':3.1f')
losses = AverageMeter('Loss', ':3.2f')
trans_losses = AverageMeter('Trans Loss', ':3.2f')
cls_accs = AverageMeter('Cls Acc', ':3.1f')
tgt_accs = AverageMeter('Tgt Acc', ':3.1f')
progress = ProgressMeter(
args.iters_per_epoch,
[batch_time, data_time, losses, trans_losses, cls_accs, tgt_accs],
prefix="Epoch: [{}]".format(epoch))
# switch to train mode
G.train()
F1.train()
F2.train()
end = time.time()
for i in range(args.iters_per_epoch):
x_s, labels_s = next(train_source_iter)
x_t, labels_t = next(train_target_iter)
x_s = x_s.to(device)
x_t = x_t.to(device)
labels_s = labels_s.to(device)
labels_t = labels_t.to(device)
x = torch.cat((x_s, x_t), dim=0)
assert x.requires_grad is False
# measure data loading time
data_time.update(time.time() - end)
# Step A train all networks to minimize loss on source domain
optimizer_g.zero_grad()
optimizer_f.zero_grad()
g = G(x)
y_1 = F1(g)
y_2 = F2(g)
y1_s, y1_t = y_1.chunk(2, dim=0)
y2_s, y2_t = y_2.chunk(2, dim=0)
y1_t, y2_t = F.softmax(y1_t, dim=1), F.softmax(y2_t, dim=1)
loss = F.cross_entropy(y1_s, labels_s) + F.cross_entropy(y2_s, labels_s) + \
0.01 * (entropy(y1_t) + entropy(y2_t))
loss.backward()
optimizer_g.step()
optimizer_f.step()
# Step B train classifier to maximize discrepancy
optimizer_g.zero_grad()
optimizer_f.zero_grad()
g = G(x)
y_1 = F1(g)
y_2 = F2(g)
y1_s, y1_t = y_1.chunk(2, dim=0)
y2_s, y2_t = y_2.chunk(2, dim=0)
y1_t, y2_t = F.softmax(y1_t, dim=1), F.softmax(y2_t, dim=1)
loss = F.cross_entropy(y1_s, labels_s) + F.cross_entropy(y2_s, labels_s) + \
0.01 * (entropy(y1_t) + entropy(y2_t)) - classifier_discrepancy(y1_t, y2_t) * args.trade_off
loss.backward()
optimizer_f.step()
# Step C train genrator to minimize discrepancy
for k in range(args.num_k):
optimizer_g.zero_grad()
g = G(x)
y_1 = F1(g)
y_2 = F2(g)
y1_s, y1_t = y_1.chunk(2, dim=0)
y2_s, y2_t = y_2.chunk(2, dim=0)
y1_t, y2_t = F.softmax(y1_t, dim=1), F.softmax(y2_t, dim=1)
mcd_loss = classifier_discrepancy(y1_t, y2_t) * args.trade_off
mcd_loss.backward()
optimizer_g.step()
cls_acc = accuracy(y1_s, labels_s)[0]
tgt_acc = accuracy(y1_t, labels_t)[0]
losses.update(loss.item(), x_s.size(0))
cls_accs.update(cls_acc.item(), x_s.size(0))
tgt_accs.update(tgt_acc.item(), x_t.size(0))
trans_losses.update(mcd_loss.item(), x_s.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
progress.display(i)
)['val_data'] # data.Subset(CIFAR10(args.data, train=True, transform=test_transform, download=True),
test_loader = load_data_for_defense(args.data + '/test')['dev_data']
print(len(train_loader), len(val_loader), len(test_loader))
m = wide_resnet(num_classes=10, depth=28, widen_factor=10, dropRate=args.drop)
model = NormalizedModel(model=m, mean=image_mean, std=image_std).to(
DEVICE) # keep images in the [0, 1] range
cifar10_best = './weights/best/2AT_cifar10_ep_13_val_acc0.8770.pth'
model_dict = torch.load(cifar10_best)
model.load_state_dict(model_dict)
if torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
optimizer = SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
if args.adv == 0:
scheduler = lr_scheduler.StepLR(optimizer,
step_size=args.lr_step,
gamma=args.lr_decay)
else:
scheduler = lr_scheduler.MultiStepLR(optimizer,
milestones=[60, 120, 160],
gamma=0.2)
attacker = DDN(steps=args.steps, device=DEVICE)
max_loss = torch.log(torch.tensor(10.)).item() # for callback
best_acc = 0
best_epoch = 0
def test_scheduler_reduce_on_plateau_plugin(self):
# Regression test for issue #858
n_epochs = 20
criterion = CrossEntropyLoss()
def _prepare_rng_critical_parts(seed=1234):
torch.random.manual_seed(seed)
return (
PluginTests.create_benchmark(seed=seed),
_PlainMLP(input_size=6, hidden_size=10),
)
self._verify_rop_tests_reproducibility(_prepare_rng_critical_parts,
n_epochs, criterion)
# Everything is in order, now we can test the plugin support for the
# ReduceLROnPlateau scheduler!
for reset_lr, reset_scheduler in itertools.product((True, False),
(True, False)):
with self.subTest(reset_lr=reset_lr,
reset_scheduler=reset_scheduler):
# First, obtain the reference (expected) lr timeline by running
# a plain PyTorch training loop with ReduceLROnPlateau.
benchmark, model = _prepare_rng_critical_parts()
model.train()
expected_lrs = []
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
for exp in benchmark.train_stream:
if reset_lr:
for group in optimizer.param_groups:
group["lr"] = 0.001
if reset_scheduler:
scheduler = ReduceLROnPlateau(optimizer)
expected_lrs.append([])
train_loss = Mean()
for epoch in range(n_epochs):
train_loss.reset()
for x, y, t in TaskBalancedDataLoader(
exp.dataset,
oversample_small_groups=True,
num_workers=0,
batch_size=32,
shuffle=False,
pin_memory=False,
):
optimizer.zero_grad()
outputs = model(x)
loss = criterion(outputs, y)
train_loss.update(loss, weight=len(x))
loss.backward()
optimizer.step()
scheduler.step(train_loss.result())
for group in optimizer.param_groups:
expected_lrs[-1].append(group["lr"])
break
# Now we have the correct timeline stored in expected_lrs.
# Let's test the plugin!
benchmark, model = _prepare_rng_critical_parts()
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
PluginTests._test_scheduler_plugin(
benchmark,
model,
optimizer,
scheduler,
n_epochs,
reset_lr,
reset_scheduler,
expected_lrs,
criterion=criterion,
metric="train_loss",
)
# Other tests
benchmark, model = _prepare_rng_critical_parts()
optimizer = SGD(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer)
scheduler2 = MultiStepLR(optimizer, [1, 2, 3])
# The metric must be set
with self.assertRaises(Exception):
LRSchedulerPlugin(scheduler, metric=None)
# Doesn't make sense to set the metric when using a non-metric
# based scheduler (should warn)
with self.assertWarns(Warning):
LRSchedulerPlugin(scheduler2, metric="train_loss")
# Must raise an error on unsupported metric
with self.assertRaises(Exception):
LRSchedulerPlugin(scheduler, metric="cuteness")
def train(hyp, # path/to/hyp.yaml or hyp dictionary
opt,
device,
callbacks
):
save_dir, epochs, batch_size, weights, single_cls, evolve, data, cfg, resume, noval, nosave, workers, freeze, = \
Path(opt.save_dir), opt.epochs, opt.batch_size, opt.weights, opt.single_cls, opt.evolve, opt.data, opt.cfg, \
opt.resume, opt.noval, opt.nosave, opt.workers, opt.freeze
# Directories
w = save_dir / 'weights' # weights dir
(w.parent if evolve else w).mkdir(parents=True, exist_ok=True) # make dir
last, best = w / 'last.pt', w / 'best.pt'
# Hyperparameters
if isinstance(hyp, str):
with open(hyp) as f:
hyp = yaml.safe_load(f) # load hyps dict
LOGGER.info(colorstr('hyperparameters: ') + ', '.join(f'{k}={v}' for k, v in hyp.items()))
# Save run settings
with open(save_dir / 'hyp.yaml', 'w') as f:
yaml.safe_dump(hyp, f, sort_keys=False)
with open(save_dir / 'opt.yaml', 'w') as f:
yaml.safe_dump(vars(opt), f, sort_keys=False)
data_dict = None
# Loggers
if RANK in [-1, 0]:
loggers = Loggers(save_dir, weights, opt, hyp, LOGGER) # loggers instance
if loggers.wandb:
data_dict = loggers.wandb.data_dict
if resume:
weights, epochs, hyp = opt.weights, opt.epochs, opt.hyp
# Register actions
for k in methods(loggers):
callbacks.register_action(k, callback=getattr(loggers, k))
# Config
plots = not evolve # create plots
cuda = device.type != 'cpu'
init_seeds(1 + RANK)
with torch_distributed_zero_first(RANK):
data_dict = data_dict or check_dataset(data) # check if None
train_path, val_path = data_dict['train'], data_dict['val']
nc = 1 if single_cls else int(data_dict['nc']) # number of classes
names = ['item'] if single_cls and len(data_dict['names']) != 1 else data_dict['names'] # class names
assert len(names) == nc, f'{len(names)} names found for nc={nc} dataset in {data}' # check
is_coco = data.endswith('coco.yaml') and nc == 80 # COCO dataset
# Model
check_suffix(weights, '.pt') # check weights
pretrained = weights.endswith('.pt')
if pretrained:
with torch_distributed_zero_first(RANK):
weights = attempt_download(weights) # download if not found locally
ckpt = torch.load(weights, map_location=device) # load checkpoint
model = Model(cfg or ckpt['model'].yaml, ch=3, nc=nc, anchors=hyp.get('anchors')).to(device) # create
exclude = ['anchor'] if (cfg or hyp.get('anchors')) and not resume else [] # exclude keys
csd = ckpt['model'].float().state_dict() # checkpoint state_dict as FP32
csd = intersect_dicts(csd, model.state_dict(), exclude=exclude) # intersect
model.load_state_dict(csd, strict=False) # load
LOGGER.info(f'Transferred {len(csd)}/{len(model.state_dict())} items from {weights}') # report
else:
model = Model(cfg, ch=3, nc=nc, anchors=hyp.get('anchors')).to(device) # create
# Freeze
freeze = [f'model.{x}.' for x in range(freeze)] # layers to freeze
for k, v in model.named_parameters():
v.requires_grad = True # train all layers
if any(x in k for x in freeze):
print(f'freezing {k}')
v.requires_grad = False
# Optimizer
nbs = 64 # nominal batch size
accumulate = max(round(nbs / batch_size), 1) # accumulate loss before optimizing
hyp['weight_decay'] *= batch_size * accumulate / nbs # scale weight_decay
LOGGER.info(f"Scaled weight_decay = {hyp['weight_decay']}")
g0, g1, g2 = [], [], [] # optimizer parameter groups
for v in model.modules():
if hasattr(v, 'bias') and isinstance(v.bias, nn.Parameter): # bias
g2.append(v.bias)
if isinstance(v, nn.BatchNorm2d): # weight (no decay)
g0.append(v.weight)
elif hasattr(v, 'weight') and isinstance(v.weight, nn.Parameter): # weight (with decay)
g1.append(v.weight)
if opt.adam:
optimizer = Adam(g0, lr=hyp['lr0'], betas=(hyp['momentum'], 0.999)) # adjust beta1 to momentum
else:
optimizer = SGD(g0, lr=hyp['lr0'], momentum=hyp['momentum'], nesterov=True)
optimizer.add_param_group({'params': g1, 'weight_decay': hyp['weight_decay']}) # add g1 with weight_decay
optimizer.add_param_group({'params': g2}) # add g2 (biases)
LOGGER.info(f"{colorstr('optimizer:')} {type(optimizer).__name__} with parameter groups "
f"{len(g0)} weight, {len(g1)} weight (no decay), {len(g2)} bias")
del g0, g1, g2
# Scheduler
if opt.linear_lr:
lf = lambda x: (1 - x / (epochs - 1)) * (1.0 - hyp['lrf']) + hyp['lrf'] # linear
else:
lf = one_cycle(1, hyp['lrf'], epochs) # cosine 1->hyp['lrf']
scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf) # plot_lr_scheduler(optimizer, scheduler, epochs)
# EMA
ema = ModelEMA(model) if RANK in [-1, 0] else None
# Resume
start_epoch, best_fitness = 0, 0.0
if pretrained:
# Optimizer
if ckpt['optimizer'] is not None:
optimizer.load_state_dict(ckpt['optimizer'])
best_fitness = ckpt['best_fitness']
# EMA
if ema and ckpt.get('ema'):
ema.ema.load_state_dict(ckpt['ema'].float().state_dict())
ema.updates = ckpt['updates']
# Epochs
start_epoch = ckpt['epoch'] + 1
if resume:
assert start_epoch > 0, f'{weights} training to {epochs} epochs is finished, nothing to resume.'
if epochs < start_epoch:
LOGGER.info(f"{weights} has been trained for {ckpt['epoch']} epochs. Fine-tuning for {epochs} more epochs.")
epochs += ckpt['epoch'] # finetune additional epochs
del ckpt, csd
# Image sizes
gs = max(int(model.stride.max()), 32) # grid size (max stride)
nl = model.model[-1].nl # number of detection layers (used for scaling hyp['obj'])
imgsz = check_img_size(opt.imgsz, gs, floor=gs * 2) # verify imgsz is gs-multiple
# DP mode
if cuda and RANK == -1 and torch.cuda.device_count() > 1:
logging.warning('DP not recommended, instead use torch.distributed.run for best DDP Multi-GPU results.\n'
'See Multi-GPU Tutorial at https://github.com/ultralytics/yolov5/issues/475 to get started.')
model = torch.nn.DataParallel(model)
# SyncBatchNorm
if opt.sync_bn and cuda and RANK != -1:
model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model).to(device)
LOGGER.info('Using SyncBatchNorm()')
# Trainloader
train_loader, dataset = create_dataloader(train_path, imgsz, batch_size // WORLD_SIZE, gs, single_cls,
hyp=hyp, augment=True, cache=opt.cache, rect=opt.rect, rank=RANK,
workers=workers, image_weights=opt.image_weights, quad=opt.quad,
prefix=colorstr('train: '))
mlc = int(np.concatenate(dataset.labels, 0)[:, 0].max()) # max label class
nb = len(train_loader) # number of batches
assert mlc < nc, f'Label class {mlc} exceeds nc={nc} in {data}. Possible class labels are 0-{nc - 1}'
# Process 0
if RANK in [-1, 0]:
val_loader = create_dataloader(val_path, imgsz, batch_size // WORLD_SIZE * 2, gs, single_cls,
hyp=hyp, cache=None if noval else opt.cache, rect=True, rank=-1,
workers=workers, pad=0.5,
prefix=colorstr('val: '))[0]
if not resume:
labels = np.concatenate(dataset.labels, 0)
# c = torch.tensor(labels[:, 0]) # classes
# cf = torch.bincount(c.long(), minlength=nc) + 1. # frequency
# model._initialize_biases(cf.to(device))
if plots:
plot_labels(labels, names, save_dir)
# Anchors
if not opt.noautoanchor:
check_anchors(dataset, model=model, thr=hyp['anchor_t'], imgsz=imgsz)
model.half().float() # pre-reduce anchor precision
callbacks.run('on_pretrain_routine_end')
# DDP mode
if cuda and RANK != -1:
model = DDP(model, device_ids=[LOCAL_RANK], output_device=LOCAL_RANK)
# Model parameters
hyp['box'] *= 3. / nl # scale to layers
hyp['cls'] *= nc / 80. * 3. / nl # scale to classes and layers
hyp['obj'] *= (imgsz / 640) ** 2 * 3. / nl # scale to image size and layers
hyp['label_smoothing'] = opt.label_smoothing
model.nc = nc # attach number of classes to model
model.hyp = hyp # attach hyperparameters to model
model.class_weights = labels_to_class_weights(dataset.labels, nc).to(device) * nc # attach class weights
model.names = names
# Start training
t0 = time.time()
nw = max(round(hyp['warmup_epochs'] * nb), 1000) # number of warmup iterations, max(3 epochs, 1k iterations)
# nw = min(nw, (epochs - start_epoch) / 2 * nb) # limit warmup to < 1/2 of training
last_opt_step = -1
maps = np.zeros(nc) # mAP per class
results = (0, 0, 0, 0, 0, 0, 0) # P, R, [email protected], [email protected], val_loss(box, obj, cls)
scheduler.last_epoch = start_epoch - 1 # do not move
scaler = amp.GradScaler(enabled=cuda)
stopper = EarlyStopping(patience=opt.patience)
compute_loss = ComputeLoss(model) # init loss class
LOGGER.info(f'Image sizes {imgsz} train, {imgsz} val\n'
f'Using {train_loader.num_workers} dataloader workers\n'
f"Logging results to {colorstr('bold', save_dir)}\n"
f'Starting training for {epochs} epochs...')
for epoch in range(start_epoch, epochs): # epoch ------------------------------------------------------------------
model.train()
# Update image weights (optional, single-GPU only)
if opt.image_weights:
cw = model.class_weights.cpu().numpy() * (1 - maps) ** 2 / nc # class weights
iw = labels_to_image_weights(dataset.labels, nc=nc, class_weights=cw) # image weights
dataset.indices = random.choices(range(dataset.n), weights=iw, k=dataset.n) # rand weighted idx
# Update mosaic border (optional)
# b = int(random.uniform(0.25 * imgsz, 0.75 * imgsz + gs) // gs * gs)
# dataset.mosaic_border = [b - imgsz, -b] # height, width borders
mloss = torch.zeros(3, device=device) # mean losses
if RANK != -1:
train_loader.sampler.set_epoch(epoch)
pbar = enumerate(train_loader)
LOGGER.info(('\n' + '%10s' * 7) % ('Epoch', 'gpu_mem', 'box', 'obj', 'cls', 'labels', 'img_size'))
if RANK in [-1, 0]:
pbar = tqdm(pbar, total=nb) # progress bar
optimizer.zero_grad()
for i, (imgs, targets, paths, _) in pbar: # batch -------------------------------------------------------------
ni = i + nb * epoch # number integrated batches (since train start)
imgs = imgs.to(device, non_blocking=True).float() / 255.0 # uint8 to float32, 0-255 to 0.0-1.0
# Warmup
if ni <= nw:
xi = [0, nw] # x interp
# compute_loss.gr = np.interp(ni, xi, [0.0, 1.0]) # iou loss ratio (obj_loss = 1.0 or iou)
accumulate = max(1, np.interp(ni, xi, [1, nbs / batch_size]).round())
for j, x in enumerate(optimizer.param_groups):
# bias lr falls from 0.1 to lr0, all other lrs rise from 0.0 to lr0
x['lr'] = np.interp(ni, xi, [hyp['warmup_bias_lr'] if j == 2 else 0.0, x['initial_lr'] * lf(epoch)])
if 'momentum' in x:
x['momentum'] = np.interp(ni, xi, [hyp['warmup_momentum'], hyp['momentum']])
# Multi-scale
if opt.multi_scale:
sz = random.randrange(imgsz * 0.5, imgsz * 1.5 + gs) // gs * gs # size
sf = sz / max(imgs.shape[2:]) # scale factor
if sf != 1:
ns = [math.ceil(x * sf / gs) * gs for x in imgs.shape[2:]] # new shape (stretched to gs-multiple)
imgs = nn.functional.interpolate(imgs, size=ns, mode='bilinear', align_corners=False)
# Forward
with amp.autocast(enabled=cuda):
pred = model(imgs) # forward
loss, loss_items = compute_loss(pred, targets.to(device)) # loss scaled by batch_size
if RANK != -1:
loss *= WORLD_SIZE # gradient averaged between devices in DDP mode
if opt.quad:
loss *= 4.
# Backward
scaler.scale(loss).backward()
# Optimize
if ni - last_opt_step >= accumulate:
scaler.step(optimizer) # optimizer.step
scaler.update()
optimizer.zero_grad()
if ema:
ema.update(model)
last_opt_step = ni
# Log
if RANK in [-1, 0]:
mloss = (mloss * i + loss_items) / (i + 1) # update mean losses
mem = f'{torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0:.3g}G' # (GB)
pbar.set_description(('%10s' * 2 + '%10.4g' * 5) % (
f'{epoch}/{epochs - 1}', mem, *mloss, targets.shape[0], imgs.shape[-1]))
callbacks.run('on_train_batch_end', ni, model, imgs, targets, paths, plots, opt.sync_bn)
# end batch ------------------------------------------------------------------------------------------------
# Scheduler
lr = [x['lr'] for x in optimizer.param_groups] # for loggers
scheduler.step()
if RANK in [-1, 0]:
# mAP
callbacks.run('on_train_epoch_end', epoch=epoch)
ema.update_attr(model, include=['yaml', 'nc', 'hyp', 'names', 'stride', 'class_weights'])
final_epoch = (epoch + 1 == epochs) or stopper.possible_stop
if not noval or final_epoch: # Calculate mAP
results, maps, _ = val.run(data_dict,
batch_size=batch_size // WORLD_SIZE * 2,
imgsz=imgsz,
model=ema.ema,
single_cls=single_cls,
dataloader=val_loader,
save_dir=save_dir,
save_json=is_coco and final_epoch,
verbose=nc < 50 and final_epoch,
plots=plots and final_epoch,
callbacks=callbacks,
compute_loss=compute_loss)
# Update best mAP
fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, [email protected], [email protected]]
if fi > best_fitness:
best_fitness = fi
log_vals = list(mloss) + list(results) + lr
callbacks.run('on_fit_epoch_end', log_vals, epoch, best_fitness, fi)
# Save model
if (not nosave) or (final_epoch and not evolve): # if save
ckpt = {'epoch': epoch,
'best_fitness': best_fitness,
'model': deepcopy(de_parallel(model)).half(),
'ema': deepcopy(ema.ema).half(),
'updates': ema.updates,
'optimizer': optimizer.state_dict(),
'wandb_id': loggers.wandb.wandb_run.id if loggers.wandb else None}
# Save last, best and delete
torch.save(ckpt, last)
if best_fitness == fi:
torch.save(ckpt, best)
del ckpt
callbacks.run('on_model_save', last, epoch, final_epoch, best_fitness, fi)
# Stop Single-GPU
if RANK == -1 and stopper(epoch=epoch, fitness=fi):
break
# Stop DDP TODO: known issues shttps://github.com/ultralytics/yolov5/pull/4576
# stop = stopper(epoch=epoch, fitness=fi)
# if RANK == 0:
# dist.broadcast_object_list([stop], 0) # broadcast 'stop' to all ranks
# Stop DPP
# with torch_distributed_zero_first(RANK):
# if stop:
# break # must break all DDP ranks
# end epoch ----------------------------------------------------------------------------------------------------
# end training -----------------------------------------------------------------------------------------------------
if RANK in [-1, 0]:
LOGGER.info(f'\n{epoch - start_epoch + 1} epochs completed in {(time.time() - t0) / 3600:.3f} hours.')
if not evolve:
if is_coco: # COCO dataset
for m in [last, best] if best.exists() else [last]: # speed, mAP tests
results, _, _ = val.run(data_dict,
batch_size=batch_size // WORLD_SIZE * 2,
imgsz=imgsz,
model=attempt_load(m, device).half(),
iou_thres=0.7, # NMS IoU threshold for best pycocotools results
single_cls=single_cls,
dataloader=val_loader,
save_dir=save_dir,
save_json=True,
plots=False)
# Strip optimizers
for f in last, best:
if f.exists():
strip_optimizer(f) # strip optimizers
callbacks.run('on_train_end', last, best, plots, epoch)
LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}")
torch.cuda.empty_cache()
return results
import matplotlib.pyplot as plt
import torch
def draw_losses(losses):
t = np.arange(len(losses))
plt.plot(t, losses)
plt.show()
if __name__ == '__main__':
batch_size = 64
train_dataset = mnist.MNIST(root='./train', train=True, transform=ToTensor())
test_dataset = mnist.MNIST(root='./test', train=False, transform=ToTensor())
train_batch = DataLoader(train_dataset, batch_size=batch_size)
test_batch = DataLoader(test_dataset, batch_size=batch_size)
model = Model()
opti = SGD(model.parameters(), lr=1e-2)
cost = CrossEntropyLoss()
losses = []
for i in range(3):
for epoch, (train_x, train_label) in enumerate(train_batch):
label_np = np.zeros((train_label.shape[0], 10))
opti.zero_grad()
predict_y = model(train_x.float())
loss = cost(predict_y, train_label.long())
losses.append(loss.detach().numpy())
if epoch % 10 == 0:
print('epoch: {}, loss: {}'.format(epoch, loss.sum().item()))
loss.backward()
opti.step()
#torch.save(model, 'models.pkl')
draw_losses(losses)
def run(args):
with open(args.cnn_path, 'r') as f:
cnn = json.load(f)
if not os.path.exists(args.save_path):
os.mkdir(args.save_path)
with open(os.path.join(args.save_path, 'cnn.json'), 'w') as f:
json.dump(cnn, f, indent=1)
os.environ["CUDA_VISIBLE_DEVICES"] = args.device_ids
num_GPU = len(args.device_ids.split(','))
batch_size_train = cnn['batch_size'] * num_GPU
batch_size_valid = cnn['batch_size'] * num_GPU
num_workers = args.num_workers * num_GPU
model = chose_model(cnn)
fc_features = model.fc.in_features
model.fc = nn.Linear(fc_features, 1) # 须知
model = DataParallel(model, device_ids=None)
model = model.cuda()
loss_fn = BCEWithLogitsLoss().cuda()
optimizer = SGD(model.parameters(), lr=cnn['lr'], momentum=cnn['momentum'])
# dataset_train = ImageFolder(cnn['data_path_train'])
# dataset_valid = ImageFolder(cnn['data_path_valid'])
dataset_train = ImageDataset(cnn['data_path_train'],
cnn['image_size'],
cnn['crop_size'],
cnn['normalize'])
dataset_valid = ImageDataset(cnn['data_path_valid'],
cnn['image_size'],
cnn['crop_size'],
cnn['normalize'])
dataloader_train = DataLoader(dataset_train,
batch_size=batch_size_train,
num_workers=num_workers)
dataloader_valid = DataLoader(dataset_valid,
batch_size=batch_size_valid,
num_workers=num_workers)
summary_train = {'epoch': 0, 'step': 0}
summary_valid = {'loss': float('inf'), 'acc': 0}
summary_writer = SummaryWriter(args.save_path)
loss_valid_best = float('inf')
for epoch in range(cnn['epoch']):
summary_train = train_epoch(summary_train, summary_writer, cnn, model,
loss_fn, optimizer,
dataloader_train)
torch.save({'epoch': summary_train['epoch'],
'step': summary_train['step'],
'state_dict': model.module.state_dict()},
os.path.join(args.save_path, 'train.ckpt'))
time_now = time.time()
summary_valid = valid_epoch(summary_valid, model, loss_fn,
dataloader_valid)
time_spent = time.time() - time_now
logging.info('{}, Epoch: {}, step: {}, Validation Loss: {:.5f}, '
'Validation ACC: {:.3f}, Run Time: {:.2f}'
.format(time.strftime("%Y-%m-%d %H:%M:%S"), summary_train['epoch'],
summary_train['step'], summary_valid['loss'],
summary_valid['acc'], time_spent))
summary_writer.add_scalar('valid/loss',
summary_valid['loss'], summary_train['step'])
summary_writer.add_scalar('valid/acc',
summary_valid['acc'], summary_train['step'])
if summary_valid['loss'] < loss_valid_best:
loss_valid_best = summary_valid['loss']
torch.save({'epoch': summary_train['epoch'],
'step': summary_train['step'],
'state_dict': model.module.state_dict()},
os.path.join(args.save_path, 'best.ckpt'))
summary_writer.close()
parser.add_argument('--use-cuda', type=bool, default=True, metavar='CUDA',
help='use cuda (default: True)')
args = parser.parse_args()
batch_loader = BatchLoader('')
params = Parameters(batch_loader.max_word_len,
batch_loader.max_seq_len,
batch_loader.words_vocab_size,
batch_loader.chars_vocab_size)
neg_loss = NEG_loss(params.word_vocab_size, params.word_embed_size)
if args.use_cuda:
neg_loss = neg_loss.cuda()
# NEG_loss is defined over two embedding matrixes with shape of [params.word_vocab_size, params.word_embed_size]
optimizer = SGD(neg_loss.parameters(), 0.1)
for iteration in range(args.num_iterations):
input_idx, target_idx = batch_loader.next_embedding_seq(args.batch_size)
input = Variable(t.from_numpy(input_idx).long())
target = Variable(t.from_numpy(target_idx).long())
if args.use_cuda:
input, target = input.cuda(), target.cuda()
out = neg_loss(input, target, args.num_sample).mean()
optimizer.zero_grad()
out.backward()
optimizer.step()
def configure_optimizers(self):
return SGD(self.parameters(), lr=self.lr)
decoder_merge_policy="cat",
dropout=0.5,
)
pad_factor = 64
imread_library = "cv2" # can be cv2 or jpeg4py
optimizer = SGD(
[
{
"params": model.decoder.parameters(),
"lr": train_parameters["lr"]
},
# decrease lr for encoder in order not to permute
# pre-trained weights with large gradients on training start
{
"params": model.encoder.parameters(),
"lr": train_parameters["lr"] / 100
},
],
train_parameters["lr"],
weight_decay=1e-4,
nesterov=True,
momentum=0.9,
)
normalization = albu.Normalize(mean=mean, std=std, p=1)
train_augmentations = albu.Compose(
[
albu.RandomSizedCrop(
min_max_height=(
def run(train_batch_size, val_batch_size, epochs, lr, momentum, log_dir):
train_loader, val_loader = get_data_loaders(train_batch_size,
val_batch_size)
model = Net()
device = "cpu"
if torch.cuda.is_available():
device = "cuda"
optimizer = SGD(model.parameters(), lr=lr, momentum=momentum)
criterion = nn.CrossEntropyLoss()
trainer = create_supervised_trainer(model,
optimizer,
criterion,
device=device)
if sys.version_info > (3, ):
from ignite.contrib.metrics.gpu_info import GpuInfo
try:
GpuInfo().attach(trainer)
except RuntimeError:
print(
"INFO: By default, in this example it is possible to log GPU information (used memory, utilization). "
"As there is no pynvml python package installed, GPU information won't be logged. Otherwise, please "
"install it : `pip install pynvml`")
metrics = {"accuracy": Accuracy(), "loss": Loss(criterion)}
train_evaluator = create_supervised_evaluator(model,
metrics=metrics,
device=device)
validation_evaluator = create_supervised_evaluator(model,
metrics=metrics,
device=device)
@trainer.on(Events.EPOCH_COMPLETED)
def compute_metrics(engine):
train_evaluator.run(train_loader)
validation_evaluator.run(val_loader)
tb_logger = TensorboardLogger(log_dir=log_dir)
tb_logger.attach(
trainer,
log_handler=OutputHandler(
tag="training",
output_transform=lambda loss: {"batchloss": loss},
metric_names="all"),
event_name=Events.ITERATION_COMPLETED(every=100),
)
tb_logger.attach(
train_evaluator,
log_handler=OutputHandler(tag="training",
metric_names=["loss", "accuracy"],
another_engine=trainer),
event_name=Events.EPOCH_COMPLETED,
)
tb_logger.attach(
validation_evaluator,
log_handler=OutputHandler(tag="validation",
metric_names=["loss", "accuracy"],
another_engine=trainer),
event_name=Events.EPOCH_COMPLETED,
)
tb_logger.attach(trainer,
log_handler=OptimizerParamsHandler(optimizer),
event_name=Events.ITERATION_COMPLETED(every=100))
tb_logger.attach(trainer,
log_handler=WeightsScalarHandler(model),
event_name=Events.ITERATION_COMPLETED(every=100))
tb_logger.attach(trainer,
log_handler=WeightsHistHandler(model),
event_name=Events.EPOCH_COMPLETED(every=100))
tb_logger.attach(trainer,
log_handler=GradsScalarHandler(model),
event_name=Events.ITERATION_COMPLETED(every=100))
tb_logger.attach(trainer,
log_handler=GradsHistHandler(model),
event_name=Events.EPOCH_COMPLETED(every=100))
# kick everything off
trainer.run(train_loader, max_epochs=epochs)
tb_logger.close()
