def get_inception_score(args, images):
splits = args.num_splits
inps = []
input_transform = transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
for img in images:
img = img.astype(np.float32)
inps.append(np.expand_dims(img, 0))
preds = []
n_batches = int(math.ceil(float(len(inps)) / float(args.batch_size)))
n_preds = 0
net = ResNet18().cuda()
net.load_state_dict(torch.load(args.model_dir))
print("load model successfully")
for i in range(n_batches):
sys.stdout.write(".")
sys.stdout.flush()
inp = inps[(i * args.batch_size):min((i + 1) *
args.batch_size, len(inps))]
inp = np.concatenate(inp, 0)
inp = torch.from_numpy(np.rollaxis(inp, 3, 1)).cuda()
outputs = net(inp)
pred = outputs.data.tolist()
#pred = softmax(pred)
preds.append(pred)
n_preds += outputs.shape[0]
preds = np.concatenate(preds, 0)
preds = np.exp(preds) / np.sum(np.exp(preds), 1, keepdims=True)
mean_, std_ = preds2score(preds, splits)
return mean_, std_
def train_CIFAR(CIFAR10=True, n_epochs=100, noise_rate=0.0, model_path='./model/CIFAR.mdl'):
output_features = 10 if CIFAR10 else 100
dataset_name = 'CIFAR10' if CIFAR10 else 'CIFAR100'
layers_in_each_block_list = [2, 2, 2, 2]
model = ResNet18(layers_in_each_block_list, output_features).to(device)
"""Prepare data"""
print('==> Preparing data..')
if CIFAR10:
train_loader, test_loader = datasets.load_cifar10_dataset()
else:
train_loader, test_loader = datasets.load_cifar100_dataset()
"""training for 10 epochs"""
print(f'==> Start training {dataset_name} '
f'with noise level {noise_rate} '
f'for {n_epochs} epochs')
criterion = torch.nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=5e-4)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=200, eta_min=0.001)
(train_loss_per_epoch, test_loss_per_epoch,
correct_per_epoch, memorized_per_epoch, incorrect_per_epoch) = train(
model, criterion, optimizer, n_epochs=n_epochs,
train_loader=train_loader, test_loader=test_loader, scheduler=scheduler,
noise_rate=noise_rate)
"""Plot learning curve and accuracy"""
print(f'acc={correct_per_epoch[-1]}, memorized={memorized_per_epoch[-1]}')
plot_title = f'{dataset_name}, noise_level={noise_rate}'
plot_learning_curve_and_acc(train_loss_per_epoch, test_loss_per_epoch,
correct_per_epoch, memorized_per_epoch, incorrect_per_epoch, plot_title)
Path("./models").mkdir(parents=True, exist_ok=True)
torch.save(model.state_dict(), model_path)
def run(size, rank, epoch, batchsize):
#print('run')
if MODEL == 'CNN' and DATA_SET == 'KWS':
model = CNNKws()
if MODEL == 'CNN' and DATA_SET == 'Cifar10':
model = CNNCifar()
if MODEL == 'ResNet18' and DATA_SET == 'Cifar10':
model = ResNet18()
model = model.cuda()
optimizer = torch.optim.SGD(model.parameters(), lr=LR, weight_decay=1e-3)
loss_func = torch.nn.CrossEntropyLoss()
train_loader = get_local_data(size, rank, batchsize)
if rank == 0:
test_loader = get_testset(rank)
#fo = open("file_multi"+str(rank)+".txt", 'w')
group_list = [i for i in range(size)]
group = dist.new_group(group_list)
model, round = load_model(model, group, rank)
while round < MAX_ROUND:
sys.stdout.flush()
if rank == 0:
accuracy = 0
positive_test_number = 0
total_test_number = 0
for step, (test_x, test_y) in enumerate(test_loader):
test_x = test_x.cuda()
test_y = test_y.cuda()
test_output = model(test_x)
pred_y = torch.max(test_output, 1)[1].data.cpu().numpy()
positive_test_number += (
pred_y == test_y.data.cpu().numpy()).astype(int).sum()
# print(positive_test_number)
total_test_number += float(test_y.size(0))
accuracy = positive_test_number / total_test_number
print('Round: ', round, ' Rank: ', rank,
'| test accuracy: %.4f' % accuracy)
#fo.write(str(round) + " " + str(rank) + " " + str(accuracy) + "\n")
for epoch_cnt in range(epoch):
for step, (b_x, b_y) in enumerate(train_loader):
b_x = b_x.cuda()
b_y = b_y.cuda()
optimizer.zero_grad()
output = model(b_x)
loss = loss_func(output, b_y)
loss.backward()
optimizer.step()
model = all_reduce(model, size, group)
#if (round+1) % ROUND_NUMBER_FOR_REDUCE == 0:
#model = all_reduce(model, size, group)
if (round + 1) % ROUND_NUMBER_FOR_SAVE == 0:
save_model(model, round + 1, rank)
round += 1
def __init__(self, celeba_loader, config):
"""Initialize configurations."""
# Data loader.
self.celeba_loader = celeba_loader
# Model configurations.
self.c_dim = config.c_dim
self.image_size = config.image_size
self.g_conv_dim = config.g_conv_dim
self.d_conv_dim = config.d_conv_dim
self.g_repeat_num = config.g_repeat_num
self.d_repeat_num = config.d_repeat_num
self.lambda_cls = config.lambda_cls
self.lambda_rec = config.lambda_rec
# Training configurations.
self.dataset = config.dataset
self.batch_size = config.batch_size
self.num_iters = config.num_iters
self.num_iters_decay = config.num_iters_decay
self.g_lr = config.g_lr
self.d_lr = config.d_lr
self.beta1 = config.beta1
self.beta2 = config.beta2
self.resume_iters = config.resume_iters
self.selected_attrs = config.selected_attrs
# Test configurations.
self.test_iters = config.test_iters
# Miscellaneous.
self.use_tensorboard = config.use_tensorboard
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
# Directories.
self.log_dir = config.log_dir
self.sample_dir = config.sample_dir
self.model_save_dir = config.model_save_dir
self.result_dir = config.result_dir
# Step size.
self.log_step = config.log_step
self.fid_step = config.fid_step
self.sample_step = config.sample_step
self.model_save_step = config.model_save_step
self.lr_update_step = config.lr_update_step
# Classifier for computing FID
self.classifier = ResNet18().to(self.device)
self.classifier.eval()
# Build the model and tensorboard.
self.build_model()
if self.use_tensorboard:
self.build_tensorboard()
def __init__(self, X, Y, idxs_lb, net, handler, args, experiment):
super(RandomNetworkDistillation, self).__init__(X,
Y,
idxs_lb,
net,
handler,
args,
experiment)
self.predictor_net = ResNet18(in_channels=3).to(self.device)
self.predictor_state = self.predictor_net.state_dict()
def get_model():
if (USE_CUDA):
if args.use_half:
model = ResNet18(use_half=args.use_half).cuda().half()
else:
model = ResNet18(use_half=args.use_half).cuda()
else:
if args.use_half:
model = ResNet18(use_half=args.use_half).half()
else:
model = ResNet18(use_half=args.use_half)
if args.resume:
checkpoint_file = args.resume
if os.path.isfile(checkpoint_file):
print("loading checkpoint {}".format(args.resume))
checkpoint = torch.load(checkpoint_file)
model.load_state_dict(checkpoint['state_dict'])
print("loaded checkpoint {} )".format(checkpoint_file ))
# --
# Initialize optimizer
print('cifar10.py: initializing optimizer...', file=sys.stderr)
lr_scheduler = getattr(LRSchedule, args.lr_schedule)(lr_init=args.lr_init, epochs=args.epochs, extra=args.extra)
model.init_optimizer(
opt=torch.optim.SGD,
params=model.parameters(),
lr_scheduler=lr_scheduler,
momentum=args.momentum,
weight_decay=args.weight_decay,
nesterov=True )
return model
def run(fold_index):
train = pd.read_csv("/fred/oz138/test/data/train_folds.csv")
test = pd.read_csv("/fred/oz138/test/data/test.csv")
submission = pd.read_csv("/fred/oz138/test/data/sample_submission.csv")
# label encoding
train["ebird_label"] = LabelEncoder().fit_transform(train['ebird_code'])
train_df = train[~train.kfold.isin([fold_index])]
train_dataset = BirdDataset(df=train_df)
valid_df = train[train.kfold.isin([fold_index])]
valid_dataset = BirdDataset(df=valid_df, valid=True)
device = "cuda"
MX = ResNet18(pretrained=True)
model = MX.to(device)
train_loader = DataLoader(dataset=train_dataset,
batch_size=args.batch_size,
pin_memory=True,
drop_last=False)
valid_loader = DataLoader(dataset=valid_dataset,
batch_size=args.batch_size,
pin_memory=True,
drop_last=False)
optimizer = torch.optim.AdamW(model.parameters(),
lr=args.lr,
betas=args.betas,
eps=args.eps,
weight_decay=args.wd)
best_acc = 0
for epoch in range(args.epochs):
train_loss = Engine.train_fn(train_loader, model, optimizer, device,
epoch)
valid_loss, valid_acc = Engine.valid_fn(valid_loader, model, device,
epoch)
print(
f"Fold {fold_index} ** Epoch {epoch + 1} **==>** Accuracy = {valid_acc}"
)
if valid_acc > best_acc:
torch.save(model.state_dict(),
os.path.join(args.MODEL_PATH, f"fold_{fold_index}.bin"))
best_acc = valid_acc
def load_model(group, rank):
if MODEL == 'CNN' and DATA_SET == 'Mnist':
model = CNNMnist()
if MODEL == 'CNN' and DATA_SET == 'Cifar10':
model = CNNCifar()
if MODEL == 'ResNet18' and DATA_SET == 'Cifar10':
model = ResNet18()
if SAVE and os.path.exists('autoencoder'+str(rank)+'.t7'):
logging('===> Try resume from checkpoint')
checkpoint = torch.load('autoencoder'+str(rank)+'.t7')
model.load_state_dict(checkpoint['state'])
round = checkpoint['round']
print('===> Load last checkpoint data')
else:
round = 0
init_param(model, 0, group)
return model, round
def run_train(seed):
device = torch.device("cuda")
print(device)
model = ResNet18(2).to(device)
print('Num parameters: %d' % model.count_parameters())
#### read hyps here ####
cs = config_spaces.get_hyperparameter_search_space(seed)
hyps = cs.sample_configuration(1).get_dictionary()
lr = hyps['learning_rate_init']
mom = hyps['momentum']
batch_size = hyps['batch_size']
epochs = hyps['epochs']
weight_decay = hyps['weight_decay']
lr_decay = 1.0 / hyps['lr_decay']
patience = hyps['patience']
tolerance = hyps['tolerance']
resize_crop = hyps['resize_crop']
h_flip = hyps['h_flip']
v_flip = hyps['v_flip']
shuffle = hyps['shuffle']
train_loader, test_loader = load_data(shuffle, batch_size, resize_crop,
h_flip, v_flip)
optimizer = optim.SGD(model.parameters(),
lr=lr,
momentum=mom,
weight_decay=weight_decay)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='max',
factor=lr_decay,
patience=patience,
threshold=tolerance)
acc_list = []
loss_list = []
time_list = []
start = time()
for epoch in range(epochs):
train(model, device, train_loader, optimizer, epoch)
test_acc, test_loss = test(model, device, test_loader)
scheduler.step(test_acc / 100)
acc_list.append(test_acc)
loss_list.append(test_loss)
time_list.append(time() - start)
return acc_list, loss_list, time_list, hyps
def load_model(group, rank):
if MODEL == 'CNN' and DATASET == 'Mnist':
model = CNNMnist()
if MODEL == 'CNN' and DATASET == 'Cifar10':
model = CNNCifar()
if MODEL == 'ResNet18' and DATASET == 'Cifar10':
model = ResNet18()
if CUDA:
model.cuda()
if False and SAVE and os.path.exists('autoencoder' + str(rank) + '.t7'):
logging('===> Try resume from checkpoint')
checkpoint = torch.load('autoencoder' + str(rank) + '.t7')
model.load_state_dict(checkpoint['state'])
logging('model loaded')
else:
init_param(model, 0, group)
logging('model created')
return model
def get_model():
if args.model == 'ResNet18':
return ResNet18(p_dropout=args.dropout)
elif args.model == 'ResNet34':
return ResNet34(p_dropout=args.dropout)
elif args.model == 'ResNet50':
return ResNet50(p_dropout=args.dropout)
elif args.model == 'ResNet101':
return ResNet101(p_dropout=args.dropout)
elif args.model == 'ResNet152':
return ResNet152(p_dropout=args.dropout)
elif args.model == 'VGG11':
return VGG('VGG11', p_dropout=args.dropout)
elif args.model == 'VGG13':
return VGG('VGG13', p_dropout=args.dropout)
elif args.model == 'VGG16':
return VGG('VGG16', p_dropout=args.dropout)
elif args.model == 'VGG19':
return VGG('VGG19', p_dropout=args.dropout)
else:
raise 'Model Not found'
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
# Build train and test dataset
train_data = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train)
test_data = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)
#Build dataloader of train and test
train_loader = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, num_workers=2)
test_loader = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, num_workers=2)
# Build a Resnet18 model
if args.model == 'resnet18':
model = ResNet18()
elif args.model == 'resnet34':
model = ResNet34()
else:
raise Exception('Put \'resent18\' or \'resnet34\' for model argument')
model = model.to(device)
print(model)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=args.lr)
def train(epoch):
print('=============================================')
print('Epoch: %d' % epoch)
print('=============================================')
print('[Train Started]')
def load_model(model_file, device):
net = ResNet18().to(device)
model = torch.load(model_file)
net.load_state_dict(model.state_dict())
return net
def main():
args = parser.parse_args()
args.cuda = args.cuda == 'yes'
args.disable_pbar = args.disable_pbar == 'yes'
args.stable_sgd = args.stable_sgd == 'yes'
print(f"args={vars(args)}")
device = torch.device("cuda:0" if torch.cuda.is_available() and args.cuda else "cpu")
print(f'Using device: {device}')
# unique identifier
uid = uuid.uuid4().hex if args.uid is None else args.uid
now = str(datetime.datetime.now().date()) + "_" + ':'.join(str(datetime.datetime.now().time()).split(':')[:-1])
runname = 'T={}_id={}'.format(now, uid) if not args.resume else args.resume
# Paths
setupname = [args.strategy, args.exp_name, args.model, args.scenario]
parentdir = os.path.join(args.save_path, '_'.join(setupname))
results_path = Path(os.path.join(parentdir, runname))
results_path.mkdir(parents=True, exist_ok=True)
tb_log_dir = os.path.join(results_path, 'tb_run') # Group all runs
# Eval results
eval_metric = 'Top1_Acc_Stream/eval_phase/test_stream'
eval_results_dir = results_path / eval_metric.split('/')[0]
eval_results_dir.mkdir(parents=True, exist_ok=True)
eval_result_files = [] # To avg over seeds
seeds = [args.seed] if args.seed is not None else list(range(args.n_seeds))
for seed in seeds:
# initialize seeds
print("STARTING SEED {}/{}".format(seed, len(seeds) - 1))
set_seed(seed)
# create scenario
if args.scenario == 'smnist':
inputsize = 28 * 28
scenario = SplitMNIST(n_experiences=5, return_task_id=False, seed=seed,
fixed_class_order=[i for i in range(10)])
elif args.scenario == 'CIFAR10':
scenario = SplitCIFAR10(n_experiences=5, return_task_id=False, seed=seed,
fixed_class_order=[i for i in range(10)])
inputsize = (3, 32, 32)
elif args.scenario == 'miniimgnet':
scenario = SplitMiniImageNet(args.dset_rootpath, n_experiences=20, return_task_id=False, seed=seed,
fixed_class_order=[i for i in range(100)])
inputsize = (3, 84, 84)
else:
raise ValueError("Wrong scenario name.")
print(f"Scenario = {args.scenario}")
if args.model == 'simple_mlp':
model = MyMLP(input_size=inputsize, hidden_size=args.hs)
elif args.model == 'resnet18':
if not args.stable_sgd:
assert args.drop_prob == 0
model = ResNet18(inputsize, scenario.n_classes, drop_prob=args.drop_prob)
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr)
# Paths
eval_results_file = eval_results_dir / f'seed={seed}.csv'
# LOGGING
tb_logger = TensorboardLogger(tb_log_dir=tb_log_dir, tb_log_exp_name=f'seed={seed}.pt') # log to Tensorboard
print_logger = TextLogger() if args.disable_pbar else InteractiveLogger() # print to stdout
eval_logger = EvalTextLogger(metric_filter=eval_metric, file=open(eval_results_file, 'a'))
eval_result_files.append(eval_results_file)
# METRICS
eval_plugin = EvaluationPlugin(
accuracy_metrics(experience=True, stream=True),
loss_metrics(minibatch=True, experience=True),
ExperienceForgetting(), # Test only
StreamConfusionMatrix(num_classes=scenario.n_classes, save_image=True),
# LOG OTHER STATS
# timing_metrics(epoch=True, experience=False),
# cpu_usage_metrics(experience=True),
# DiskUsageMonitor(),
# MinibatchMaxRAM(),
# GpuUsageMonitor(0),
loggers=[print_logger, tb_logger, eval_logger])
plugins = None
if args.strategy == 'replay':
plugins = [RehRevPlugin(n_total_memories=args.mem_size,
mode=args.replay_mode, # STEP-BACK
aversion_steps=args.aversion_steps,
aversion_lr=args.aversion_lr,
stable_sgd=args.stable_sgd, # Stable SGD
lr_decay=args.lr_decay,
init_epochs=args.init_epochs # First task epochs
)]
# CREATE THE STRATEGY INSTANCE (NAIVE)
strategy = Naive(model, optimizer, criterion,
train_epochs=args.epochs, device=device,
train_mb_size=args.bs, evaluator=eval_plugin,
plugins=plugins
)
# train on the selected scenario with the chosen strategy
print('Starting experiment...')
for experience in scenario.train_stream:
if experience.current_experience == args.until_task:
print("CUTTING OF TRAINING AT TASK ", experience.current_experience)
break
else:
print("Start training on step ", experience.current_experience)
strategy.train(experience)
print("End training on step ", experience.current_experience)
print('Computing accuracy on the test set')
res = strategy.eval(scenario.test_stream[:args.until_task]) # Gathered by EvalLogger
final_results_file = eval_results_dir / f'seed_summary.pt'
stat_summarize(eval_result_files, final_results_file)
print(f"[FILE:TB-RESULTS]: {tb_log_dir}")
print(f"[FILE:FINAL-RESULTS]: {final_results_file}")
print("FINISHED SCRIPT")
# reproducibility is da best
np.random.seed(run)
torch.manual_seed(run)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
# fetch data
data = locate('data.get_%s' % args.dataset)(args)
# make dataloaders
train_loader, val_loader, test_loader = [
CLDataLoader(elem, args, train=t)
for elem, t in zip(data, [True, False, False])
]
model = ResNet18(args.n_classes, nf=20,
input_size=args.input_size).to(args.device)
opt = torch.optim.SGD(model.parameters(), lr=0.1)
gen = CVAE(20, args).cuda() # this is actually an autoencoder
opt_gen = torch.optim.Adam(gen.parameters())
# build buffer
if args.store_latents:
buffer = Buffer(args, input_size=(20 * 4 * 4, ))
else:
buffer = Buffer(args)
buffer.min_per_class = 0
print('multiple heads ', args.multiple_heads)
if run == 0:
def trainer(
train_set: torch.utils.data.Dataset,
test_set: torch.utils.data.Dataset,
size_dict: Dict[int, int],
model: str = "ResNet50",
device: torch.device = torch.device("cpu"),
batch_size: int = 500,
num_epochs: int = 10,
learning_rate: float = 0.001,
weight_decay: float = 0,
dropout: float = 0,
) -> float:
"""
Get the best test accuracy after training for `num_epochs` epochs.
"""
# create data-loaders
train_loader = DataLoader(
dataset=train_set,
batch_size=batch_size,
drop_last=True,
shuffle=True,
pin_memory=True,
)
test_loader = DataLoader(dataset=test_set,
batch_size=batch_size,
shuffle=False,
pin_memory=True)
# set model, loss function and optimizer
num_classes = len(size_dict)
if model == "ResNet50":
model = ResNet50(num_classes, dropout).to(device)
elif model == "ResNet18":
model = ResNet18(num_classes, dropout).to(device)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(params=model.parameters(),
lr=learning_rate,
weight_decay=weight_decay)
# here class encoding is necessary since we need the dimension
# of one-hot encoding identical to the number of classes
class_encoding = {
class_id: i
for i, (class_id, _) in enumerate(size_dict.items())
}
# start training
best_acc = 0
for epoch in range(num_epochs):
model.train()
for images, labels in train_loader:
labels = labels.apply_(lambda id: class_encoding[id])
images, labels = images.to(device), labels.to(device)
# forward pass
pred = model(images)
loss = criterion(pred, labels)
# backward pass
optimizer.zero_grad()
loss.backward()
optimizer.step()
# test after each epoch
model.eval()
accuracies = []
with torch.no_grad():
for images, labels in test_loader:
labels = labels.apply_(lambda id: class_encoding[id])
images, labels = images.to(device), labels.to(device)
# forward pass
pred = model(images)
batch_acc = accuracy(pred, labels)
accuracies.append(batch_acc)
epoch_acc = sum(accuracies) / len(accuracies)
best_acc = max(best_acc, epoch_acc)
return float(round(best_acc, 4))
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
trainset = torchvision.datasets.CIFAR10(
root=dataDir, train=True, download=True, transform=transform_train)
testset = torchvision.datasets.CIFAR10(
root=dataDir, train=False, download=True, transform=transform_test)
testloader = torch.utils.data.DataLoader(
testset, batch_size=batch_size, shuffle=True)
net = ResNet18().cuda()
optimizer = optim.SGD(net.parameters(), lr=0.1,momentum=0.9, weight_decay=5e-4)
checkpoint = torch.load(os.path.join(checkpointDir, 'forget_ckpt.pth'))
start_epoch = int(checkpoint['epoch'])+1
print ("Path : %s, Epoch : %d, Acc : %f" %(os.path.join(checkpointDir, 'forget_ckpt.pth'),start_epoch-1,checkpoint['acc']))
net.load_state_dict(checkpoint['net'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
criterion = nn.CrossEntropyLoss()
def compute_correct_incorrect():
net.eval()
correct_ones = torch.from_numpy(np.full(len(trainset), False)).cuda()
with torch.no_grad():
for i in range(0,len(trainset),batch_size):
batch_ind = torch.from_numpy(np.arange(i, min(i+batch_size,len(trainset))))
transformed_trainset=[]
# Begin Run Loop
for run in range(args.n_runs):
mir_tries, mir_success = 0, 0
# REPRODUCTIBILITY
if args.reproc:
np.random.seed(run)
torch.manual_seed(run)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
# CLASSIFIER
if args.use_conv:
cls = ResNet18(args.n_classes, nf=20, input_size=args.input_size).to(
args.device) # xuji added
else:
#cls = MLP(args)
cls = classifier(args).to(args.device)
opt = torch.optim.SGD(cls.parameters(), lr=args.lr)
if run == 0:
print("number of classifier parameters:",
sum([np.prod(p.size()) for p in cls.parameters()]))
# GENERATIVE MODELING
if (args.method is not 'no_rehearsal') or (args.gen_method
is not 'no_rehearsal'):
gen = VAE.VAE(args).to(args.device)
#if args.cuda:
# gen = gen.cuda()
def main():
train_loader, test_loader = get_dataloader()
if args.mode == "train":
args.round = 0
model = ResNet18(num_classes=10)
train_model(model, train_loader, test_loader)
elif args.mode == "prune":
previous_ckpt = "./checkpoints/resnet18-round%d.pth" % (args.round - 1)
print("Pruning round %d, load model from %s" % (args.round, previous_ckpt))
model = torch.load(previous_ckpt)
prune_model(model)
print(model)
params = sum([np.prod(p.size()) for p in model.parameters()])
print("Number of Parameters: %.1fM" % (params / 1e6))
train_model(model, train_loader, test_loader)
elif args.mode == "test":
ckpt = "./checkpoints/resnet18-round%d.pth" % (args.round)
print("Load model from %s" % (ckpt))
model = torch.load(ckpt)
params = sum([np.prod(p.size()) for p in model.parameters()])
print("Number of Parameters: %.1fM" % (params / 1e6))
acc = eval(model, test_loader)
print("Acc=%.4f\n" % (acc))
elif args.mode == "tensorrt":
ckpt = "./checkpoints/resnet18-round%d.pth" % (args.round)
print("Load model from %s" % (ckpt))
model = torch.load(ckpt)
params = sum([np.prod(p.size()) for p in model.parameters()])
print("Number of Parameters: %.1fM" % (params / 1e6))
torch_in = torch.ones((1, 3, 32, 32)).cuda()
torch.onnx.export(
model,
torch_in,
"./checkpoints/model_onnx.onnx",
verbose=False,
opset_version=12,
)
onnx_model = onnx.load("./checkpoints/model_onnx.onnx")
model_simp, check = simplify(onnx_model)
onnx.save(model_simp, "./checkpoints/model_onnx.onnx")
cmd = (
"onnx2trt "
+ "./checkpoints/model_onnx.onnx"
+ " -o "
+ "./checkpoints/tensorrt_engine.engine"
+ " -b "
+ "1"
+ " -w "
+ str(1024 * 1024 * 1024)
+ " -d 32"
)
os.system(cmd)
trt_model = TRT_Engine("./checkpoints/tensorrt_engine.engine", max_batch_size=1)
num_iter = 2000
total_time_list = []
with torch.no_grad():
for i in range(num_iter):
start = time.time()
trt_model(torch_in)
total_time_list.append(time.time() - start)
print(
"total FPS -> avg:{}, max:{}, min:{}".format(
1 / (sum(total_time_list[100:]) / (num_iter - 100)),
1 / (max(total_time_list[100:])),
1 / (min(total_time_list[100:])),
)
)
test_baseline = parser.baseline
num_epochs = parser.epochs
ngpu = parser.ngpu
iters = 0
# write into file
log_dir = parser.outpdir + "/resnetlogs"
ckpt_dir = parser.outpdir
if not os.path.exists(ckpt_dir):
os.makedirs(ckpt_dir)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# Load model
net = ResNet18().to(device)
if (device.type == 'cuda') and ngpu > 1:
net = nn.DataParallel(net, list(range(ngpu)))
#print(net)
# set up your loss function and optmizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(), lr=parser.lr) #, momentum=momentum)
print('#######################Start Training#######################')
with open('resnet_timelogs.csv', 'w') as timefile:
with open('resnet_acc.csv', 'w') as accfile:
##############################################
## Train at every epoch ##
##############################################
for epoch in range(num_epochs):
# --------------
# Begin Run Loop
for run in range(args.n_runs):
mir_tries, mir_success = 0, 0
# REPRODUCTIBILITY
if args.reproc:
np.random.seed(run)
torch.manual_seed(run)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
# CLASSIFIER
if args.use_conv:
cls = ResNet18(args.n_classes, nf=20, input_size=args.input_size)
else:
#cls = MLP(args)
cls = classifier(args).to(args.device)
opt = torch.optim.SGD(cls.parameters(), lr=args.lr)
if run == 0:
print("number of classifier parameters:",
sum([np.prod(p.size()) for p in cls.parameters()]))
# GENERATIVE MODELING
if (args.method is not 'no_rehearsal') or (args.gen_method
is not 'no_rehearsal'):
gen = VAE.VAE(args).to(args.device)
#if args.cuda:
# gen = gen.cuda()
generator.load_state_dict(ckpt["g"], strict=False)
discriminator_photo.load_state_dict(ckpt["d_p"])
discriminator_cari.load_state_dict(ckpt["d_c"])
discriminator_feat_p.load_state_dict(ckpt["d_feat_p"])
discriminator_feat_c.load_state_dict(ckpt["d_feat_c"])
g_ema.load_state_dict(ckpt["g_ema"], strict=False)
g_optim.load_state_dict(ckpt["g_optim"])
d_optim_p.load_state_dict(ckpt["d_optim_p"])
d_optim_c.load_state_dict(ckpt["d_optim_c"])
d_optim_fp.load_state_dict(ckpt["d_optim_fp"])
d_optim_fc.load_state_dict(ckpt["d_optim_fc"])
### Attribute classifier ###
c_cls = ResNet18().to(device)
p_cls = ResNet18().to(device)
ckpt_pa = torch.load(args.ckpt_pa)
ckpt_ca = torch.load(args.ckpt_ca)
p_cls.load_state_dict(ckpt_pa)
c_cls.load_state_dict(ckpt_ca)
p_cls.eval()
c_cls.eval()
id_net = InceptionResnetV1(pretrained='vggface2').to(device).eval()
with torch.no_grad():
mean_latent = generator.photo_generator.mean_latent(4096)
def main():
prior = torch.tensor(opt.prior)
input_size = 32
# prepare training data
transform = {
'train':
transforms.Compose([
transforms.Resize(input_size),
transforms.RandomCrop(input_size, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.4914, 0.4822, 0.4465],
[0.2023, 0.1994, 0.2010])
]),
'val':
transforms.Compose([
transforms.Resize(input_size),
transforms.CenterCrop(input_size),
transforms.ToTensor(),
transforms.Normalize([0.4914, 0.4822, 0.4465],
[0.2023, 0.1994, 0.2010])
]),
}
label_trainset = datasets.CIFAR10(root=opt.label_dir,
train=True,
download=True,
transform=transform['train'])
label_testset = datasets.CIFAR10(root=opt.label_dir,
train=False,
download=True,
transform=transform['val'])
labelset_index = []
classnum = np.ones(opt.num_classes) * opt.pos_num
i = 0
while (np.sum(classnum) > 0):
image = label_trainset[i][0]
label = label_trainset[i][1]
if (classnum[label] > 0):
labelset_index.append(i)
classnum[label] -= 1
i += 1
label_train_subset = torch.utils.data.Subset(label_trainset,
labelset_index)
target_transform = transforms.Lambda(
lambda target: target + opt.num_classes)
unlabel_set = datasets.ImageFolder(opt.unlabel_dir,
transform['train'],
target_transform=target_transform)
trainloader = torch.utils.data.DataLoader(label_train_subset + unlabel_set,
batch_size=opt.pu_batchsize,
shuffle=True,
num_workers=32)
testloader = torch.utils.data.DataLoader(label_testset,
batch_size=opt.pu_batchsize,
shuffle=False,
num_workers=32)
dataloader = {}
dataloader['train'] = trainloader
dataloader['val'] = testloader
# stage1: get positive data from unlabeled dataset
model_pu = nn.DataParallel(
initialize_model(use_pu=True, num_classes=opt.num_classes)).cuda()
optimizer_pu = optim.SGD(model_pu.parameters(),
lr=opt.pu_lr,
momentum=opt.momentum,
weight_decay=opt.pu_weight_decay)
scheduler_pu = optim.lr_scheduler.MultiStepLR(optimizer_pu,
milestones=[50, 100, 150],
gamma=0.1,
last_epoch=-1)
model_pu = train_pu(model_pu,
dataloader,
optimizer_pu,
scheduler_pu,
prior=prior,
num_classes=opt.num_classes,
num_epochs=opt.pu_num_epochs)
trainloader2 = torch.utils.data.DataLoader(unlabel_set,
batch_size=opt.pu_batchsize,
shuffle=False,
num_workers=32)
positive_index_all = get_positive(model_pu, trainloader2)
print("We have {} positive unlabeled images for all!".format(
len(positive_index_all)))
unlabel_positive_set = torch.utils.data.Subset(unlabel_set,
positive_index_all)
trainloader3 = torch.utils.data.DataLoader(label_train_subset +
unlabel_positive_set,
batch_size=opt.batchsize,
shuffle=True,
num_workers=32)
dataloader['train'] = trainloader3
# stage2: train student model with rkd method
teacher = nn.DataParallel(
initialize_model(use_pu=False, num_classes=opt.num_classes)).cuda()
student = nn.DataParallel(ResNet18(num_classes=opt.num_classes)).cuda()
class_weight = get_class_weight(teacher,
trainloader3,
num_classes=opt.num_classes)
print(class_weight)
class_weights = perturb(class_weight, opt.epsilon, opt.perturb_num)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(student.parameters(),
lr=opt.lr,
momentum=opt.momentum,
weight_decay=opt.weight_decay)
scheduler = optim.lr_scheduler.MultiStepLR(optimizer,
milestones=[50, 100, 150],
gamma=0.1,
last_epoch=-1)
model_s, hist = train(student,
teacher,
class_weights,
dataloader,
criterion,
optimizer,
scheduler,
num_epochs=opt.num_epochs)
axes[i][j].set_axis_off()
axes[i][j].imshow(X_train_orig[k].reshape((28, 28)))
#plt.show()
# Normalize image vectors
X_train = X_train_orig / 255.
X_test = X_test_orig / 255.
# Convert training and test labels to one hot matrices
label_binrizer = LabelBinarizer()
Y_train = label_binrizer.fit_transform(Y_train_orig)
Y_test = label_binrizer.fit_transform(Y_test_orig)
print("number of training examples = " + str(X_train.shape[0]))
print("number of test examples = " + str(X_test.shape[0]))
print("X_train shape: " + str(X_train.shape))
print("Y_train shape: " + str(Y_train.shape))
print("X_test shape: " + str(X_test.shape))
print("Y_test shape: " + str(Y_test.shape))
# train the neural network
model = ResNet18(input_shape=(28, 28, 1), classes=24)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(X_train, Y_train, epochs=2, batch_size=32)
# test the neural network
preds = model.evaluate(X_test, Y_test)
print("Loss = " + str(preds[0]))
print("Test Accuracy = " + str(preds[1]))
train_tfm = tfm.Compose([tfm.RandomCrop(28, padding=4),
tfm.RandomHorizontalFlip(),
tfm.ToTensor(),
tfm.Normalize(mean=(0.1307,), std=(0.3081,))
])
test_tfm = tfm.Compose([ tfm.ToTensor(),
tfm.Normalize(mean=(0.1307,), std=(0.3081,))
])
fmnist_train = FashionMNIST(args.save_dir, train=True, transform=train_tfm, download=True)
fmnist_test = FashionMNIST(args.save_dir, train=False, transform=test_tfm, download=True)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
clf = ResNet18(nc=1)
clf.to(device)
optimizer = optim.SGD(clf.parameters(), lr=args.lr, weight_decay=args.wd, momentum=args.mom)
criterion = nn.CrossEntropyLoss()
# Multiplies the LR with 0.1 at epoch 100 and 150 as mentioned in the paper
lmd = lambda x: 0.1 if x in [100,150] else 1
scheduler = lr_scheduler.MultiplicativeLR(optimizer, lr_lambda=lmd)
trainloader = DataLoader(fmnist_train, batch_size=args.batch_size, shuffle=True)
testloader = DataLoader(fmnist_test, batch_size=args.batch_size, shuffle=False)
best_loss = np.inf
for epoch in range(args.epochs):
