def main():
config = get_config(mode='test')
vocab = Vocab()
vocab.load(config.word2id_path, config.id2word_path)
print(f'Vocabulary size: {vocab.vocab_size}')
config.vocab_size = vocab.vocab_size
if config.users:
test_users = load_pickle(config.convs_users_path)
config.user_size = max([x for xx in test_users for x in xx]) + 1
print(f'User size: {config.user_size}')
else:
test_users = None
data_loader = get_loader(convs=load_pickle(config.convs_path),
convs_length=load_pickle(config.conversations_length_path),
utterances_length=load_pickle(config.utterances_length_path),
vocab=vocab, batch_size=config.batch_size, shuffle=False, convs_users=test_users)
model_solver = getattr(solvers, "Solver{}".format(config.model))
test_solver = model_solver(config, None, data_loader, vocab=vocab, is_train=False)
test_solver.build()
test_solver.export_samples()
def do_tsne(options):
options.shuffle = False
options.batch_size = 2048
options.cuda_device = f"cuda:{get_gpu_ids()[0]}"
# Load dataset into memory
dataset = getattr(datasets, options.dataset)(options).test_set
num = len(dataset)
# Get the model
model = get_model(options)
proj_dim = options.projection_dim
X = np.empty((num, proj_dim))
y = np.empty(num)
# Get the dataloader
loader = get_loader(dataset, options)
idx = 0
for (batch, labels) in loader:
num_batch = batch.shape[0]
X[idx:idx + num_batch] = model(batch).cpu().numpy()
y[idx:idx + num_batch] = labels.cpu().numpy()
idx += num_batch
X_new = TSNE().fit_transform(X)
return (X_new, y)
def _eval_open_set_helper(self,
discovered_samples,
discovered_classes,
roc_path,
goscr_path,
verbose=True,
do_plot=True):
dataset_loaders, _ = self.trainer_machine.get_trainloaders(
discovered_samples, shuffle=False)
train_loader = dataset_loaders['train']
test_loader = get_loader(self.dataset_info.test_dataset,
None,
shuffle=False,
batch_size=self.batch,
workers=self.workers)
test_features = self._get_features(test_loader,
self.trainer_machine,
verbose=verbose).cpu()
labeled_features = self._get_features(train_loader,
self.trainer_machine,
verbose=verbose).cpu()
self.min_dist = self.distance_function(
test_features, labeled_features).min(dim=1)[0].cpu().tolist()
return super()._eval_open_set_helper(discovered_samples,
discovered_classes,
roc_path,
goscr_path,
verbose=verbose,
do_plot=True)
def get_pre_classifier_pipeline(options, model):
full_train_set = getattr(datasets,
options.dataset)(options).plain_train_set
options.batch_size = 2048
loader = get_loader(full_train_set, options)
# standardize the scaler for all classifiers
scaler = StandardScaler(copy=False)
for batch, _ in loader:
out = model(batch).detach().cpu().numpy()
scaler.partial_fit(out)
if options.model == "MoCoModel":
steps = [('scaler', scaler)]
elif options.model == "SelfLabelModel":
if options.ipca:
ipca = IncrementalPCA(copy=False,
n_components=options.ipca_dim,
batch_size=options.batch_size)
for batch, _ in loader:
out = model(batch).detach().cpu().numpy()
ipca.partial_fit(scaler.transform(out))
steps = [('scaler', scaler), ('ipca', ipca)]
else:
steps = [('scaler', scaler)]
elif options.model == "SimCLRModel":
steps = [('scaler', scaler)]
else:
raise NotImplementedError(
f"Classifier pipeline for {model} not implemented")
pipeline = Pipeline(steps)
options.pre_classifier_pipeline = pipeline
return pipeline
def main(config):
prepare_dirs_and_logger(config)
rng = np.random.RandomState(config.random_seed)
tf.set_random_seed(config.random_seed)
if config.is_train:
data_path = config.data_path
batch_size = config.batch_size
do_shuffle = True
else:
setattr(config, 'batch_size', 16)
if config.test_data_path is None:
data_path = config.data_path
else:
data_path = config.test_data_path
batch_size = config.sample_per_image
do_shuffle = False
data_loader = get_loader(data_path, config.batch_size,
config.input_scale_size, config.data_format,
config.split)
trainer = Trainer(config, data_loader)
if config.is_train:
save_config(config)
trainer.train()
else:
if not config.load_path:
raise Exception(
"[!] You should specify `load_path` to load a pretrained model"
)
trainer.test()
def main(args):
model = utils.get_arch(arch=args.arch, dataset=args.dataset)
criterion = torch.nn.CrossEntropyLoss()
state_dict = torch.load(args.resume_path, map_location=torch.device('cpu'))
model.load_state_dict( state_dict['model_state_dict'] )
del state_dict
if not args.cpu:
model.cuda()
criterion = criterion.cuda()
attacker = utils.PGDAttacker(
radius = args.pgd_radius,
steps = args.pgd_steps,
step_size = args.pgd_step_size,
random_start = True,
norm_type = args.pgd_norm_type,
)
if args.grad_opt == 'get-mu':
''' calculate grad_mu '''
train_loader = utils.get_loader(
args.dataset, args.batch_size, train=True, training=False)
grad_mu = get_grad_mu(
model, criterion, train_loader, attacker, args.cpu)
with open('{}/{}-mu.pkl'.format(args.save_dir, args.eval_save_name), 'wb') as f:
pickle.dump(grad_mu, f)
elif args.grad_opt == 'get-noise':
''' calculate grad noise '''
with open(args.resume_grad_mu_path, 'rb') as f:
grad_mu = pickle.load(f)
train_loader = utils.get_loader(
args.dataset, args.batch_size, train=True, training=True)
grad_noise = get_grad_noise(
model, criterion, train_loader, attacker, grad_mu,
args.grad_rep_T, args.grad_samp_T, args.cpu)
with open('{}/{}-noise.pkl'.format(args.save_dir, args.eval_save_name), 'wb') as f:
pickle.dump(grad_noise, f)
def main():
model = YOLOv3(num_classes=config.NUM_CLASSES).to(config.DEVICE)
optimizer = optim.Adam(
model.parameters(), lr=config.LEARNING_RATE, weight_decay=config.WEIGHT_DECAY,
)
loss_fn = YoloLoss()
scalar = torch.cuda.amp.GradScaler()
train_loader, test_loader, train_eval_loader = get_loader(
train_csv_path=config.DATASET + "/train.csv", test_csv_path=config.DATASET+"/test.csv"
)
if config.LOAD_MODEL:
load_checkpoint(
config.CHECKPOINT_FILE, model, optimizer, config.LEARNING_RATE,
)
scaled_anchors = (
torch.tensor(config.ANCHORS)
+ torch.tensor(config.S).unsqueeze(1).unsqueeze(2).repeat(1, 3, 2)
).to(config.DEVICE)
for epoch in range(config.NUM_EPOCHS):
train_fn(test_loader, model, optimizer, loss_fn, scalar, scaled_anchors)
if config.SAVE_MODEL:
save_checkpoint(model, optimizer, filename=f"checkpoint.pth.tar")
print(f"Currently epoch {epoch}")
print("On Train Eval loader:")
print("On Train Loader:")
check_class_accuracy(model, train_loader, threshold=config.CONF_THRESHOLD)
if epoch % 10 == 0 and epoch > 0:
print("0n Test Loaded:")
check_class_accuracy(model, test_loader, threshold=config.CONF_THRESHOLD)
pred_boxes, true_boxes = get_evaluation_bboxes(
test_loader,
model,
iou_threshold = config.NMS_IOU_THRESH,
anchors = config.ANCHORS,
threshold = config.CONF_THRESHOLD,
)
mapval = mean_average_precision(
pred_boxes,
true_boxes,
iou_threshold = config.MAP_IOU_THRESH,
box_format = "midpoint",
num_classes = config.NUM_CLASSES,
)
print(f"MAP: {mapval.item()}")
model.train()
def load_dataset(self,
dataset_name,
batch_size_train=100,
batch_size_test=1000,
val_ratio=0.3,
n_cpu=8,
transforms=None):
"""
Method that return the data loaders of a torchvision dataset
:param dataset_name: name of the torchvision dataset
batch_size_train: batch size to use for train data loader
batch_size_test: batch size to use for test data loader
val_ratio: percent of the data train to use for validation data loader
n_cpu: num workers to use
transforms: transformation to apply
:return: train, validation and test torch data loader
"""
assert dataset_name == "CIFAR-10" or dataset_name == "CIFAR-100", \
"Pytorch datasets permitted are: CIFAR-10, CIFAR-100"
if not os.path.exists(self.root_path):
os.makedirs(self.root_path)
train_dataset, test_dataset = None, None
if dataset_name == "CIFAR-10":
train_dataset = torchvision.datasets.CIFAR10(
self.root_path,
train=True,
download=self.download,
transform=transforms["train"])
test_dataset = torchvision.datasets.CIFAR10(
self.root_path,
train=False,
download=self.download,
transform=transforms["test"])
elif dataset_name == "CIFAR-100":
train_dataset = torchvision.datasets.CIFAR100(
self.root_path,
train=True,
download=self.download,
transform=transforms["train"])
test_dataset = torchvision.datasets.CIFAR100(
self.root_path,
train=False,
download=self.download,
transform=transforms["test"])
train_loader, validation_loader, test_loader = utils.get_loader(
train_dataset=train_dataset,
test_dataset=test_dataset,
batch_size_train=batch_size_train,
batch_size_test=batch_size_test,
n_cpu=n_cpu,
val_ratio=val_ratio)
return train_loader, validation_loader, test_loader
def generate(config, epoch):
step_time = time.time()
loader = utils.get_loader(config, config.sample_dir)
sample_time = time.time()
pool = utils.Image_Pool(config, config.sample_step, batch_size=1)
print('-- Loading Images')
for _, (image, _) in enumerate(loader):
pool.add(image)
print('-- Start Generating Images')
idx = 0
count = 0
while True:
images = pool()
generated = net(variable(images))
start = (config.pool_size - config.sample_step) // 2
end = start + config.sample_step
for i in range(start, end):
image_a = images[0, :, i, :, :]
image_b = generated.data[0, :, i, :, :].cpu()
sample = torch.cat([image_a, image_b], dim=2)
sample = utils.denorm(sample)
if not os.path.isdir(config.result_dir):
os.mkdir(config.result_dir)
name = os.path.join(config.result_dir,
'Epoch-%d-%05d.jpg' % (epoch, count))
torchvision.utils.save_image(sample, name)
count += 1
if not pool.check():
break
if idx % config.log_frequency == 0:
speed = (time.time() - step_time) / config.log_frequency
step_time = time.time()
format_str = 'Generating Images: Step %d Speed %.2f sec/frame'
print(format_str % (idx, speed))
idx += 1
def main(config):
prepare_dirs_and_logger(config)
rng = np.random.RandomState(config.random_seed)
tf.set_random_seed(config.random_seed)
if config.is_train:
data_path = config.data_path
batch_size = config.batch_size
do_shuffle=True
else:
setattr(config,'batch_size',64)
if config.test_data_path is None:
data_path = config.data_path
else:
data_path = config.test_data_path
batch_size = config.sample_per_image
do_shuffle=False
if config.dataset == 'mnist' :
mnist = tf.keras.datasets.mnist
(x_train,y_train),(x_test,y_test) = mnist.load_data()
data_loader = batch_generator(x_train, config.batch_size, config.data_format)
elif config.dataset == 'cifar10':
cifar = tf.keras.datasets.cifar10
(x_train,y_train),(x_test,y_test) = cifar.load_data()
data_loader = batch_generator(x_train, config.batch_size, config.data_format)
else:
data_loader = get_loader(
data_path, config.batch_size,config.scale_size, config.data_format)
trainer = Trainer(config,data_loader)
if config.is_train:
save_config(config)
trainer.train()
else:
if not config.load_path:
raise Exception("[!] You should specify load_path to load a pretrained model")
trainer.test()
def main():
start_time = time()
args = get_args()
if args.checkpoint_dir_name:
dir_name = args.checkpoint_dir_name
else:
dir_name = datetime.datetime.now().strftime('%y%m%d%H%M%S')
path_to_dir = Path(__file__).resolve().parents[1]
path_to_dir = os.path.join(path_to_dir, *['log', dir_name])
os.makedirs(path_to_dir, exist_ok=True)
# tensorboard
path_to_tensorboard = os.path.join(path_to_dir, 'tensorboard')
os.makedirs(path_to_tensorboard, exist_ok=True)
writer = SummaryWriter(path_to_tensorboard)
# model saving
os.makedirs(os.path.join(path_to_dir, 'model'), exist_ok=True)
path_to_model = os.path.join(path_to_dir, *['model', 'model.tar'])
# csv
os.makedirs(os.path.join(path_to_dir, 'csv'), exist_ok=True)
path_to_results_csv = os.path.join(path_to_dir, *['csv', 'results.csv'])
path_to_args_csv = os.path.join(path_to_dir, *['csv', 'args.csv'])
if not args.checkpoint_dir_name:
with open(path_to_args_csv, 'a') as f:
args_dict = vars(args)
param_writer = csv.DictWriter(f, list(args_dict.keys()))
param_writer.writeheader()
param_writer.writerow(args_dict)
# logging using hyperdash
if not args.no_hyperdash:
from hyperdash import Experiment
exp = Experiment('Classification task on CIFAR10 dataset with CNN')
for key in vars(args).keys():
exec("args.%s = exp.param('%s', args.%s)" % (key, key, key))
else:
exp = None
path_to_dataset = os.path.join(
Path(__file__).resolve().parents[2], 'datasets')
os.makedirs(path_to_dataset, exist_ok=True)
train_loader, eval_loader, classes = get_loader(
batch_size=args.batch_size,
num_workers=args.num_workers,
path_to_dataset=path_to_dataset)
# show some of the training images, for fun.
dataiter = iter(train_loader)
images, labels = dataiter.next()
img_grid = torchvision.utils.make_grid(images)
matplotlib_imshow(img_grid)
writer.add_image('four_CIFAR10_images', img_grid)
# define a network, loss function and optimizer
model = CNN()
writer.add_graph(model, images)
model = torch.nn.DataParallel(model)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum)
start_epoch = 0
# resume training
if args.checkpoint_dir_name:
print('\nLoading the model...')
checkpoint = torch.load(path_to_model)
model.state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
start_epoch = checkpoint['epoch'] + 1
summary(model, input_size=(3, 32, 32))
model.to(args.device)
# train the network
print('\n--------------------')
print('Start training and evaluating the CNN')
for epoch in range(start_epoch, args.n_epoch):
start_time_per_epoch = time()
train_loss, train_acc = train(train_loader, model, criterion,
optimizer, args.device, writer, epoch,
classes)
eval_loss, eval_acc = eval(eval_loader, model, criterion, args.device)
elapsed_time_per_epoch = time() - start_time_per_epoch
result_dict = {
'epoch': epoch,
'train_loss': train_loss,
'eval_loss': eval_loss,
'train_acc': train_acc,
'eval_acc': eval_acc,
'elapsed time': elapsed_time_per_epoch
}
with open(path_to_results_csv, 'a') as f:
result_writer = csv.DictWriter(f, list(result_dict.keys()))
if epoch == 0: result_writer.writeheader()
result_writer.writerow(result_dict)
# checkpoint
torch.save(
{
'epoch': epoch,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, path_to_model)
if exp:
exp.metric('train loss', train_loss)
exp.metric('eval loss', eval_loss)
exp.metric('train acc', train_acc)
exp.metric('eval acc', eval_acc)
else:
print(result_dict)
writer.add_scalar('loss/train_loss', train_loss,
epoch * len(train_loader))
writer.add_scalar('loss/eval_loss', eval_loss,
epoch * len(eval_loader))
writer.add_scalar('acc/train_acc', train_acc,
epoch * len(train_loader))
writer.add_scalar('acc/eval_acc', eval_acc, epoch * len(eval_loader))
elapsed_time = time() - start_time
print('\nFinished Training, elapsed time ===> %f' % elapsed_time)
if exp:
exp.end()
writer.close()
import torch
import torch.nn.functional as F
import os, sys, csv
import yaml, time
from model import Net
from utils import get_loader
with open(sys.argv[1]) as f:
config = yaml.load(f, Loader=yaml.Loader)
print('Preparing datasets...')
loader, names = get_loader(if_label=False,
imgs_folder=config['imgs_folder'],
csv_file=config['csv_file'],
resize_size=config['resize_size'],
crop_size=config['crop_size'],
is_train=config['is_train'],
batch_size=config['batch_size'],
num_workers=config['num_workers'])
print('Constructing network...')
net = Net(config)
device = 'cuda:%d' % config['device'] if torch.cuda.is_available() else 'cpu'
net.to(device)
net.ready()
print('Test start!')
with torch.no_grad():
tot_pred = None
for i, x in enumerate(loader):
x = x.to(device) # x is 5D tensor
# 增加程序的运行速度
cudnn.benchmark = True
# Create directories if not exist.
if not os.path.exists(opts.log_path):
os.makedirs(opts.log_path)
if not os.path.exists(opts.model_save_dir):
os.makedirs(opts.model_save_dir)
if not os.path.exists(opts.output_path):
os.makedirs(opts.output_path)
if not os.path.exists(opts.result_dir):
os.makedirs(opts.result_dir)
# Load experiment setting
config = get_config(opts.config)
#为当前GPU设置随机种子，使得结果是确定的
torch.manual_seed(opts.seed)
#为所有的GPU设置种子
torch.cuda.manual_seed(opts.seed)
face_loader = None
face_loader = get_loader(config['data_root'], config['face_crop_size'],
config['face_image_size'], config['batch_size'],
'train', config['num_workers'])
solver = Solver(face_loader, config, opts)
if opts.mode == 'train':
solver.train()
elif opts.mode == 'test':
solver.test()
