loss_fn = multiClassHingeLoss() if args.clf == 'svm' else F.nll_loss
loss_type = 'hinge' if args.clf == 'svm' else 'nll'
print('+'*80)
best = 0
x_ax = []
acc_train = []
acc_test = []
l_train = []
l_test = []
print('Pre-Training')
test(args, model, device, test_loader, best, 1, loss_type)
grad = []
grad_angle_prev = []
grad_angle_strt = []
grad_init = []
wait = 0
for epoch in range(1, args.epochs + 1):
gradi = []
correcti = 0
for data, target in train_loader:
data, target = data.to(device), target.to(device)
output = model(data)
loss = loss_fn(output, target)
optim.zero_grad()
def main(eval_args):
# ensures that weight initializations are all the same
logging = utils.Logger(eval_args.local_rank, eval_args.save)
# load a checkpoint
logging.info('loading the model at:')
logging.info(eval_args.checkpoint)
checkpoint = torch.load(eval_args.checkpoint, map_location='cpu')
args = checkpoint['args']
if not hasattr(args, 'ada_groups'):
logging.info('old model, no ada groups was found.')
args.ada_groups = False
if not hasattr(args, 'min_groups_per_scale'):
logging.info('old model, no min_groups_per_scale was found.')
args.min_groups_per_scale = 1
if not hasattr(args, 'num_mixture_dec'):
logging.info('old model, no num_mixture_dec was found.')
args.num_mixture_dec = 10
logging.info('loaded the model at epoch %d', checkpoint['epoch'])
arch_instance = utils.get_arch_cells(args.arch_instance)
model = AutoEncoder(args, None, arch_instance)
# Loading is not strict because of self.weight_normalized in Conv2D class in neural_operations. This variable
# is only used for computing the spectral normalization and it is safe not to load it. Some of our earlier models
# did not have this variable.
model.load_state_dict(checkpoint['state_dict'], strict=False)
model = model.cuda()
logging.info('args = %s', args)
logging.info('num conv layers: %d', len(model.all_conv_layers))
logging.info('param size = %fM ', utils.count_parameters_in_M(model))
if eval_args.eval_mode == 'evaluate':
# load train valid queue
args.data = eval_args.data
train_queue, valid_queue, num_classes = datasets.get_loaders(args)
if eval_args.eval_on_train:
logging.info('Using the training data for eval.')
valid_queue = train_queue
# get number of bits
num_output = utils.num_output(args.dataset)
bpd_coeff = 1. / np.log(2.) / num_output
valid_neg_log_p, valid_nelbo = test(
valid_queue,
model,
num_samples=eval_args.num_iw_samples,
args=args,
logging=logging)
logging.info('final valid nelbo %f', valid_nelbo)
logging.info('final valid neg log p %f', valid_neg_log_p)
logging.info('final valid nelbo in bpd %f', valid_nelbo * bpd_coeff)
logging.info('final valid neg log p in bpd %f',
valid_neg_log_p * bpd_coeff)
else:
bn_eval_mode = not eval_args.readjust_bn
num_samples = 16
with torch.no_grad():
n = int(np.floor(np.sqrt(num_samples)))
set_bn(model,
bn_eval_mode,
num_samples=36,
t=eval_args.temp,
iter=500)
for ind in range(10): # sampling is repeated.
torch.cuda.synchronize()
start = time()
with autocast():
logits = model.sample(num_samples, eval_args.temp)
output = model.decoder_output(logits)
output_img = output.mean if isinstance(output, torch.distributions.bernoulli.Bernoulli) \
else output.sample()
torch.cuda.synchronize()
end = time()
# save images to 'results/eval-x/images/epochn' where x is exp id and n is epoch muber
# print("tensor shape: {}".format(output_img.shape))
# try saving the images one my one
path_to_images = '/content/gdrive/MyDrive/pipeline_results/NVAE/results/eval-1/images'
if not os.path.exists(path_to_images):
os.makedirs(path_to_images)
for i in range(output_img.size(0)):
vutils.save_image(output_img[i, :, :, :],
'%s/sample_batch%03d_img%03d.png' %
(path_to_images, ind + 1, i + 1),
normalize=True)
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum)
else:
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum)
best_valid_loss = np.inf
iteration = 0
epoch = 1
# train with early stopping
while (epoch < args.epochs + 1) and (iteration < args.patience):
train(train_loader, model, optimizer, epoch, args.cuda, args.log_interval)
valid_loss = test(valid_loader, model, args.cuda)
if valid_loss > best_valid_loss:
iteration += 1
print('Loss was not improved, iteration {0}'.format(str(iteration)))
else:
print('Saving model...')
iteration = 0
best_valid_loss = valid_loss
state = {
'net': model.module if args.cuda else model,
'acc': valid_loss,
'epoch': epoch,
}
if not os.path.isdir('checkpoint'):
os.mkdir('checkpoint')
torch.save(state, './checkpoint/ckpt.t7')
def main():
global args
args = parser.parse_args()
modelpath = os.path.join(os.path.abspath('../pixel_Exps'),args.exp)
train_data = np.load(os.path.join(modelpath,'train_split.npy'))
val_data = np.load(os.path.join(modelpath,'val_split.npy'))
with open(os.path.join(modelpath,'train_attack.txt'),'r') as f:
train_attack = f.readlines()
train_attack = [attack.split(' ')[0].split(',')[0].split('\n')[0] for attack in train_attack]
sys.path.append(modelpath)
model = import_module('model')
config, net = model.get_model()
start_epoch = args.start_epoch
save_dir = args.save_dir
if args.resume:
checkpoint = torch.load(args.resume)
if start_epoch == 0:
start_epoch = checkpoint['epoch'] + 1
if not save_dir:
save_dir = checkpoint['save_dir']
else:
save_dir = os.path.join(modelpath,'results', save_dir)
net.load_state_dict(checkpoint['state_dict'])
else:
if start_epoch == 0:
start_epoch = 1
if not save_dir:
exp_id = time.strftime('%Y%m%d-%H%M%S', time.localtime())
save_dir = os.path.join(modelpath,'results', exp_id)
else:
save_dir = os.path.join(modelpath,'results', save_dir)
if args.test == 1:
net = net.net
if args.debug:
net = net.cuda()
else:
net = DataParallel(net).cuda()
cudnn.benchmark = True
if args.test == 1:
with open(os.path.join(modelpath,'test_attack.txt'),'r') as f:
test_attack = f.readlines()
test_attack = [attack.split(' ')[0].split(',')[0].split('\n')[0] for attack in test_attack]
test_data = np.load(os.path.join(modelpath,'test_split.npy'))
dataset = DefenseDataset(config, 'test', test_data, test_attack)
test_loader = DataLoader(
dataset,
batch_size = args.batch_size,
shuffle = False,
num_workers = args.workers,
pin_memory = True)
resumeid = args.resume.split('.')[-2].split('/')[-1]
print(args.defense)
args.defense = args.defense==1
if args.defense:
name = 'result_%s_%s'%(args.exp,resumeid)
else:
name = 'result_%s_%s_nodefense'%(args.exp,resumeid)
test(net, test_loader, name, args.defense)
return
dataset = DefenseDataset(config, 'train', train_data, train_attack)
train_loader = DataLoader(
dataset,
batch_size = args.batch_size,
shuffle = True,
num_workers = args.workers,
pin_memory = True)
dataset = DefenseDataset(config, 'val', val_data, train_attack)
val_loader = DataLoader(
dataset,
batch_size = args.batch_size,
shuffle = True,
num_workers = args.workers,
pin_memory = True)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logfile = os.path.join(save_dir,'log')
sys.stdout = Logger(logfile)
pyfiles = [f for f in os.listdir('./') if f.endswith('.py')]
for f in pyfiles:
shutil.copy(f, os.path.join(save_dir, f))
if isinstance(net, DataParallel):
params = net.module.net.denoise.parameters()
else:
params = net.net.denoise.parameters()
if args.optimizer == 'sgd':
optimizer = optim.SGD(
params,
lr = args.lr,
momentum = 0.9,
weight_decay = args.weight_decay)
elif args.optimizer == 'adam':
optimizer = optim.Adam(
params,
lr = args.lr,
weight_decay = args.weight_decay)
else:
exit('Wrong optimizer')
def get_lr(epoch):
if epoch <= args.epochs * 0.6:
return args.lr
elif epoch <= args.epochs * 0.9:
return args.lr * 0.1
else:
return args.lr * 0.01
for epoch in range(start_epoch, args.epochs + 1):
requires_control = epoch == start_epoch
train(epoch, net, train_loader, optimizer, get_lr, config['loss_idcs'], requires_control = requires_control)
val(epoch, net, val_loader, requires_control = requires_control)
if epoch % args.save_freq == 0:
try:
state_dict = net.module.state_dict()
except:
state_dict = net.state_dict()
for key in state_dict.keys():
state_dict[key] = state_dict[key].cpu()
torch.save({
'epoch': epoch,
'save_dir': save_dir,
'state_dict': state_dict,
'args': args},
os.path.join(save_dir, '%03d.ckpt' % epoch))
# setup checkpoint directory and load from checkpoint as needed
model, best_acc, ckpt_output_fname = setup_and_load()
torch.set_num_threads(10) # number of MKL threads for evaluation
# download dataset if not found
logging.debug('Downloading')
datasets.CIFAR10(f'{args.tmp_dir}/data/', train=True, download=True)
# Determine initial checkpoint accuracy
# necessary to get initial confidences
logging.info('Testing')
val_loss, val_accuracy = test(args,
model,
device,
dataloader_kwargs,
etime=-1)
logging.info('Eval acc: %.3f', val_accuracy)
torch.set_num_threads(3) # number of MKL threads for evaluation
start_time = time.time()
# when simulating, attack process is the first to run
if simulating:
if args.attack_checkpoint_path != 'train':
logging.warning('Checkpoint path ignored during simulation')
step = launch_atk_proc()
# attack finished, allow for recovery if more than one worker
if args.num_processes > 1:
# with open("data.json", "r") as fp:
# data = json.load(fp)
data = {}
for key in model_dict.keys():
model = model_dict[key]()
stats = []
print(key, "\n")
for epoch in tqdm(range(1, n_epochs + 1), leave=False):
start_time = time()
if "SG" not in key:
train(model, epoch, train_loader, log_interval, verbose=False)
else:
train_sg(model, epoch, train_loader, log_interval, verbose=False)
end_time = time()
time_taken = end_time - start_time
metrics = test(model, test_loader)
metrics.append(time_taken)
stats.append(metrics)
data[key] = stats
# clear gpu memory
torch.cuda.empty_cache()
# with open("data.json", "w") as fp:
# json.dump(data, fp)
model.load_state_dict(checkpoint['state_dict'])
if opt.is_train:
optimizer.load_state_dict(checkpoint['optimizer'])
for epoch in range(opt.begin_epoch, opt.n_epochs + 1):
if opt.is_train:
train_epoch(epoch, train_logger, train_batch_logger, trainloader,
optimizer, criterion, model, opt, device)
if opt.is_val:
validation_loss = val_epoch(epoch, val_logger, valloader,
optimizer, criterion, model, device)
if opt.is_train and opt.is_val:
scheduler.step(validation_loss)
if opt.is_test:
testset = FrankaDataset(opt,
'test',
data_length,
transform_input=transform_i,
transform_target=transform_t)
testloader = DataLoader(testset,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.n_threads,
pin_memory=True)
test(trainloader, model, device)
import pickle
import torch
from LeavesDataset import dataLoaders
from train import test
from directories import listDir
pickleFilenames = listDir('.', '.pickle')
for filename in pickleFilenames:
print('Patching', filename)
with open(filename, 'rb') as f:
info = pickle.load(f)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
accuracy, labels, predictions = test(info['model'], dataLoaders['testing'], device)
info['accuracy'] = accuracy
info['labels'] = labels
info['predictions'] = predictions
patchFilename = 'patched-' + filename
print('Patch in', patchFilename)
with open(patchFilename, 'wb') as f:
pickle.dump(info, f, pickle.HIGHEST_PROTOCOL)
test_loader = torch.utils.data.DataLoader(test_dataset_full,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=1)
if args.PRETRAIN:
print('pretraining...')
net = LeNet5().cuda()
loss_func = torch.nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(net.parameters(), lr=5e-2, momentum=0.5)
get_accuracy(net, train_loader, loss_func)
val_accuracy = validate(net, val_loader, loss_func)
best_acc = val_accuracy[0]
test(net, test_loader, loss_func)
save_model_ori(args.model_ori, net, optimizer)
for epoch in range(100):
if epoch % 30 == 0:
optimizer.param_groups[0]['lr'] *= 0.2
train_fullprecision(net, train_loader, loss_func, optimizer, epoch)
val_accuracy = validate(net, val_loader, loss_func)
if val_accuracy[0] > best_acc:
best_acc = val_accuracy[0]
test(net, test_loader, loss_func)
save_model_ori(args.model_ori, net, optimizer)
if args.ALQ:
print('adaptive loss-aware quantization...')
if __name__ == '__main__':
dataset = 'THUCNews'
model_name = args.model
print('model_name: ', model_name)
# 动态化加载模型
x = import_module('models.' + model_name)
# 加载模型的参数
config = x.Config(dataset)
# 保持每次运行取得的数据集是一样的
np.random.seed(5)
torch.manual_seed(5)
torch.cuda.manual_seed_all(4)
torch.backends.cudnn.deterministic = True
# 加载数据
start_time = time.time()
print('加载数据')
train_data, test_data, dev_data = utils.bulid_dataset(config)
train_iter = utils.build_iteration(train_data, config)
test_iter = utils.build_iteration(test_data, config)
dev_iter = utils.build_iteration(dev_data, config)
time_idf = utils.get_time_idf(start_time)
print('数据加载完成, 用时: ', time_idf)
# 模型的训练, 验证, 测试
model = x.Model(config).to(config.device)
train.train(config, model, train_iter, dev_iter)
train.test(config, model, test_iter)
best_prec1 = 0.
for epoch in range(args.start_epoch, args.epochs):
if epoch in [args.epochs * 0.5, args.epochs * 0.75]:
for param_group in optimizer.param_groups:
param_group['lr'] *= 0.1
avg_loss, train_acc = train(model,
optimizer,
epoch=epoch,
device=device,
train_loader=train_loader,
valid=args.valid,
valid_size=valid_size,
log_interval=args.log_interval)
if args.valid:
prec1 = valid(model, device, valid_loader, valid_size=valid_size)
test(model, device, test_loader)
else:
prec1 = test(model, device, test_loader)
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
'cfg': model.cfg
},
is_best,
filepath=args.save)
def main(eval_args):
# ensures that weight initializations are all the same
logging = utils.Logger(eval_args.local_rank, eval_args.save)
# load a checkpoint
logging.info('loading the model at:')
logging.info(eval_args.checkpoint)
checkpoint = torch.load(eval_args.checkpoint, map_location='cpu')
args = checkpoint['args']
if not hasattr(args, 'ada_groups'):
logging.info('old model, no ada groups was found.')
args.ada_groups = False
if not hasattr(args, 'min_groups_per_scale'):
logging.info('old model, no min_groups_per_scale was found.')
args.min_groups_per_scale = 1
if not hasattr(args, 'num_mixture_dec'):
logging.info('old model, no num_mixture_dec was found.')
args.num_mixture_dec = 10
logging.info('loaded the model at epoch %d', checkpoint['epoch'])
arch_instance = utils.get_arch_cells(args.arch_instance)
model = AutoEncoder(args, None, arch_instance)
# Loading is not strict because of self.weight_normalized in Conv2D class in neural_operations. This variable
# is only used for computing the spectral normalization and it is safe not to load it. Some of our earlier models
# did not have this variable.
model.load_state_dict(checkpoint['state_dict'], strict=False)
model = model.cuda()
logging.info('args = %s', args)
logging.info('num conv layers: %d', len(model.all_conv_layers))
logging.info('param size = %fM ', utils.count_parameters_in_M(model))
if eval_args.eval_mode == 'evaluate':
# load train valid queue
args.data = eval_args.data
train_queue, valid_queue, num_classes = datasets.get_loaders(args)
if eval_args.eval_on_train:
logging.info('Using the training data for eval.')
valid_queue = train_queue
# get number of bits
num_output = utils.num_output(args.dataset)
bpd_coeff = 1. / np.log(2.) / num_output
valid_neg_log_p, valid_nelbo = test(
valid_queue,
model,
num_samples=eval_args.num_iw_samples,
args=args,
logging=logging)
logging.info('final valid nelbo %f', valid_nelbo)
logging.info('final valid neg log p %f', valid_neg_log_p)
logging.info('final valid nelbo in bpd %f', valid_nelbo * bpd_coeff)
logging.info('final valid neg log p in bpd %f',
valid_neg_log_p * bpd_coeff)
else:
bn_eval_mode = not eval_args.readjust_bn
num_samples = 16
with torch.no_grad():
n = int(np.floor(np.sqrt(num_samples)))
set_bn(model,
bn_eval_mode,
num_samples=36,
t=eval_args.temp,
iter=500)
for ind in range(10): # sampling is repeated.
torch.cuda.synchronize()
start = time()
with autocast():
logits = model.sample(num_samples, eval_args.temp)
output = model.decoder_output(logits)
output_img = output.mean if isinstance(output, torch.distributions.bernoulli.Bernoulli) \
else output.sample()
torch.cuda.synchronize()
end = time()
output_tiled = utils.tile_image(output_img,
n).cpu().numpy().transpose(
1, 2, 0)
logging.info('sampling time per batch: %0.3f sec',
(end - start))
output_tiled = np.asarray(output_tiled * 255, dtype=np.uint8)
output_tiled = np.squeeze(output_tiled)
plt.imshow(output_tiled)
plt.show()
total_num_of_tasks=args.nb_task)
val_task_loader = tasks.TaskLoader(is_train=False, total_num_of_tasks=100)
for i in pbar:
if i % 10000 == 10000 - 1:
torch.save(M, 'particles-{}.p'.format(i))
lr_scheduler.step()
#args.step_size = GetInnerStepSize(i)
train_task = next(train_task_loader)
#ret_dic = TrainOneTask(task, M, SVGD, optimizer, DEVICE, args.steps, args.step_size)
if i % args.test_interval == args.test_interval - 1:
logp = test(val_task_loader, M, SVGD, DEVICE, args.steps,
args.step_size)
writer.add_scalar('TestLogP', logp, i // args.test_interval)
print(logp)
ret_dic = TrainOneTaskWithChaserLoss(train_task, M, SVGD, optimizer,
DEVICE, args.steps, args.step_size)
torch.cuda.empty_cache()
pbar.set_description("Training")
pbar.set_postfix(ret_dic)
# For tensorboard
LogpTrainDic = {}
LogpJointDic = {}
for key, var in ret_dic.items():
if key.endswith('train'):
LogpTrainDic.update({key: var})
def main():
now = datetime.datetime.now()
logger = Logger(args.save_path + '/logs_{}'.format(now.isoformat()))
model = getModel(args)
cudnn.benchmark = True
optimizer = torch.optim.SGD(model.parameters(), args.LR,
momentum=args.momentum,
weight_decay=args.weight_decay)
valSource_dataset = SourceDataset('test', ref.nValViews)
valTarget_dataset = TargetDataset('test', ref.nValViews)
valSource_loader = torch.utils.data.DataLoader(valSource_dataset, batch_size = 1,
shuffle=False, num_workers=1, pin_memory=True, collate_fn=collate_fn_cat)
valTarget_loader = torch.utils.data.DataLoader(valTarget_dataset, batch_size = 1,
shuffle=False, num_workers=1, pin_memory=True, collate_fn=collate_fn_cat)
if args.test:
f = {}
for split in splits:
f['{}'.format(split)] = open('{}/{}.txt'.format(args.save_path, split), 'w')
test(args, valSource_loader, model, None, f['valSource'], 'valSource')
test(args, valTarget_loader, model, None, f['valTarget'], 'valTarget')
return
train_dataset = Fusion(SourceDataset, TargetDataset, nViews = args.nViews, targetRatio = args.targetRatio, totalTargetIm = args.totalTargetIm)
trainTarget_dataset = train_dataset.targetDataset
train_loader = torch.utils.data.DataLoader(
train_dataset, batch_size=args.batchSize, shuffle=not args.test,
num_workers=args.workers if not args.test else 1, pin_memory=True, collate_fn=collate_fn_cat)
trainTarget_loader = torch.utils.data.DataLoader(
trainTarget_dataset, batch_size=args.batchSize, shuffle=False,
num_workers=args.workers if not args.test else 1, pin_memory=True, collate_fn=collate_fn_cat)
M = None
if args.shapeWeight > ref.eps:
print 'getY...'
Y = getY(train_dataset.sourceDataset)
M = initLatent(trainTarget_loader, model, Y, nViews = args.nViews, S = args.sampleSource, AVG = args.AVG)
print 'Start training...'
for epoch in range(1, args.epochs + 1):
adjust_learning_rate(optimizer, epoch, args.dropLR)
train_mpjpe, train_loss, train_unSuploss = train(args, train_loader, model, optimizer, M, epoch)
valSource_mpjpe, valSource_loss, valSource_unSuploss = validate(args, 'Source', valSource_loader, model, None, epoch)
valTarget_mpjpe, valTarget_loss, valTarget_unSuploss = validate(args, 'Target', valTarget_loader, model, None, epoch)
train_loader.dataset.targetDataset.shuffle()
if args.shapeWeight > ref.eps and epoch % args.intervalUpdateM == 0:
M = stepLatent(trainTarget_loader, model, M, Y, nViews = args.nViews, lamb = args.lamb, mu = args.mu, S = args.sampleSource)
logger.write('{} {} {}\n'.format(train_mpjpe, valSource_mpjpe, valTarget_mpjpe))
logger.scalar_summary('train_mpjpe', train_mpjpe, epoch)
logger.scalar_summary('valSource_mpjpe', valSource_mpjpe, epoch)
logger.scalar_summary('valTarget_mpjpe', valTarget_mpjpe, epoch)
logger.scalar_summary('train_loss', train_loss, epoch)
logger.scalar_summary('valSource_loss', valSource_loss, epoch)
logger.scalar_summary('valTatget_loss', valTarget_loss, epoch)
logger.scalar_summary('train_unSuploss', train_unSuploss, epoch)
logger.scalar_summary('valSource_unSuploss', valSource_unSuploss, epoch)
logger.scalar_summary('valTarget_unSuploss', valTarget_unSuploss, epoch)
if epoch % 10 == 0:
torch.save({
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'optimizer' : optimizer.state_dict(),
}, args.save_path + '/checkpoint_{}.pth.tar'.format(epoch))
logger.close()
def main(mode, args):
# build vocab
word2index, index2word = build_all_vocab(init_vocab={
UNK_WORD: 0,
BOS_WORD: 1
})
args.vocab, args.vocab_size, args.index2word = word2index, len(
word2index), index2word
# get data_from_train from only_label = True, for same as train baseline
args.only_label = True
train_dataset = BindingDataset('train', args=args)
data_from_train = (train_dataset.tokenize_max_len,
train_dataset.columns_token_max_len,
train_dataset.columns_split_marker_max_len,
train_dataset.cells_token_max_len,
train_dataset.cells_split_marker_max_len,
train_dataset.pos_tag_vocab,
train_dataset.bert_tokenize_max_len,
train_dataset.bert_tokenize_marker_max_len,
train_dataset.bert_columns_split_max_len,
train_dataset.bert_columns_split_marker_max_len,
train_dataset.bert_cells_split_max_len,
train_dataset.bert_cells_split_marker_max_len)
args.tokenize_max_len, args.columns_token_max_len, args.columns_split_marker_max_len, \
args.cells_token_max_len, args.cells_split_marker_max_len, args.pos_tag_vocab,\
args.bert_tokenize_max_len, args.bert_tokenize_marker_max_len, args.bert_columns_split_max_len, args.bert_columns_split_marker_max_len,\
args.bert_cells_split_max_len, args.bert_cells_split_marker_max_len = data_from_train
logger.info('data_from_train'), logger.info(data_from_train)
# set only_label
if mode == 'train baseline':
args.only_label = True
elif mode == 'policy gradient':
args.only_label = False
elif mode == 'test model':
args.only_label = True
elif mode == 'add feature':
args.only_label = False
elif mode == 'write cases':
args.only_label = True
elif mode == 'anonymous':
args.only_label = False
# build train_dataloader
train_dataset = BindingDataset('train',
args=args,
data_from_train=data_from_train)
train_dataloader = DataLoader(dataset=train_dataset,
batch_size=args.batch_size,
shuffle=args.shuffle)
# build dev_dataloader
args.shuffle = False
dev_dataset = BindingDataset('dev',
args=args,
data_from_train=data_from_train)
dev_dataloader = DataLoader(dataset=dev_dataset,
batch_size=args.batch_size,
shuffle=args.shuffle)
# build test_dataloader
# test_dataset = BindingDataset('test', args=args, data_from_train=data_from_train)
# test_dataloader = DataLoader(dataset=test_dataset, batch_size=args.batch_size, shuffle=args.shuffle)
# load word embedding
if args.load_w2v:
args.embed_matrix = load_word_embedding(args.word_dim, word2index)
# train
if mode == 'train baseline':
if args.model == 'baseline':
model = Baseline(args=args)
elif args.model == 'gate':
if args.bert_model is None:
model = Gate(args=args)
else:
model = BertGate(args=args)
else:
raise NotImplementedError
train(train_dataloader, dev_dataloader, args=args, model=model)
elif mode == 'policy gradient':
model = torch.load('./res/' + args.model +
'/2816_False_True_True_726425',
map_location=lambda storage, loc: storage.cuda(0))
train_rl(train_dataloader, dev_dataloader, args=args, model=model)
elif mode == 'test model':
# also need the correct 'model' for dataloader
model = torch.load(
'./res/policy_gradient/0.819928_True_True_True_412532',
map_location=lambda storage, loc: storage.cuda(0))
eval(dev_dataloader, args, model, epoch=0)
eval_rl(dev_dataloader, args, model, epoch=0)
elif mode == 'add feature':
model = torch.load('./res/policy_gradient/0.804922_22-16-28',
map_location=lambda storage, loc: storage.cuda(0))
res = test(dev_dataloader, args, model)
add_abstraction('dev', res=res, args=args)
elif mode == 'write cases':
model = torch.load(
'./res/policy_gradient/0.819928_True_True_True_412532',
map_location=lambda storage, loc: storage.cuda(0))
res_pg = test(dev_dataloader, args, model, sep=' ')
model = torch.load('./res/gate/epoch100',
map_location=lambda storage, loc: storage.cuda(0))
res_gate = test(dev_dataloader, args, model, sep=' ')
with open('cases.txt', 'w', encoding='utf-8') as f:
for key in res_pg.keys():
# diff between gate and policy
if res_gate[key]['pred'] != res_pg[key]['pred']:
if res_gate[key]['pred'] == res_gate[key]['label']:
f.write(key + '\n')
f.write('Pred_Gate:\t\t\t\t' +
json.dumps(res_gate[key]['pred']) + '\n')
f.write('Pred_Policy_Gradient:\t' +
json.dumps(res_pg[key]['pred']) + '\n')
f.write('Label:\t\t\t\t\t' +
json.dumps(res_pg[key]['label']) + '\n')
f.write('SQL_Labels:\t\t\t\t' +
json.dumps(res_pg[key]['sql_labels']) + '\n' +
'\n')
elif mode == 'anonymous':
model = torch.load(
'./res/policy_gradient/0.819928_True_True_True_412532',
map_location=lambda storage, loc: storage.cuda(0))
res = test(train_dataloader, args, model, sep='')
anonymous('train', res, args)
best = 0
# Fire the engines
model = FCN(args.input_size, args.output_size).to(device)
model.load_state_dict(torch.load(init_path))
print('Load init: {}'.format(init_path))
best = 0
x_ax = []
y_ax = []
l_test = []
grad_tr = []
for epoch in range(1, args.epochs + 1):
grad = train(args, model, fog_graph, workers, X_trains, y_trains,
device, epoch, loss_fn='hinge')
acc, loss = test(args, model, device, test_loader, best, epoch,
loss_fn='hinge')
y_ax.append(acc)
x_ax.append(epoch)
l_test.append(loss)
grad_tr.append(grad)
if args.save_model and acc > best:
best = acc
torch.save(model.state_dict(), best_path)
print('Model best @ {}, acc {}: {}\n'.format(
epoch, acc, best_path))
if (args.save_model):
torch.save(model.state_dict(), stop_path)
print('Model stop: {}'.format(stop_path))
model = network.WideResNet(depth=28,
num_classes=args.n_classes,
widen_factor=10,
dropRate=0.0).to(device)
else:
raise ValueError('invalid network architecture {}'.format(args.arch))
# Load the pretrained model
model.load_state_dict(torch.load(pretrained_model, map_location='cpu'))
# Set the model in evaluation mode. In this case this is for the Dropout
# layers
model.eval()
print('test accuracy w/ input transform. ')
clean_accuracy = test(None, model, device, test_loader)
accuracies = []
# Run test for each epsilon
for eps in epsilons:
if args.attack_method == "FGSM":
# TODO: magic number
attacker = attack_func.FGSM(epsilon=eps,
clip_min=-1.,
clip_max=1.,
targeted=args.targeted)
else:
raise NotImplementedError(
'attack method {} not implemented. '.format(
args.attack_method))
args = parser.parse_args()
if __name__ == '__main__':
with open(args.model, 'rb') as f:
model = torch.load(f)
if args.cuda:
model.cuda()
optimizer = torch.optim.Adamax(model.parameters(), lr=2e-4)
with open(opts["data"] + 'train.pickle', 'rb') as f:
train_data = pickle.load(f)
with open(opts["data"] + 'dev.pickle', 'rb') as f:
dev_data = pickle.load(f)
dev_data = sorted(dev_data, key=lambda x: len(x[1]))
print('train data size {:d}, dev data size {:d}'.format(
len(train_data), len(dev_data)))
print("testing...")
best = test(net=model, valid_data=dev_data)
print("best: {}".format(best))
print("training...")
for epoch in range(opts["epoch"]):
train(epoch, model, train_data, optimizer, best)
acc = test(net=model, valid_data=dev_data)
if acc > best:
best = acc
with open(args.save, 'wb') as f:
torch.save(model, f)
print('epcoh {:d} dev acc is {:f}, best dev acc {:f}'.format(
epoch, acc, best))
legend=['Train Loss', 'Valid Loss']))
epoch_fig = viz.line(X=torch.zeros((1, )).cpu(),
Y=100 * torch.ones((1, )).cpu(),
opts=dict(xlabel='Iteration',
ylabel='Loss',
title='Current Epoch Training Loss',
legend=['Loss']))
train_losses = []
valid_losses = []
for epoch in range(1, args.n_epochs + 1):
#training + validation
train_loss = train(train_loader, epoch, model, args, epoch_fig)
valid_loss = test(valid_loader, model, args)
# save train valid loss
train_losses.append(train_loss)
valid_losses.append(valid_loss)
#plot
viz.line(X=torch.ones((1)).cpu() * epoch,
Y=torch.Tensor([train_loss, valid_loss]).unsqueeze(0).cpu(),
win=fig,
update='append')
#saving model
if epoch % args.save_freq == 1:
fn = 'vrnn_state_dict_' + str(epoch) + '.pth'
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
def launch_procs(eval_counter=0, s_rank=0):
'''Launch normal workers.
If no workers would be spawned, just return. This will happen if
simulating with a single worker --- no recovery time is allowed. '''
if s_rank == args.num_processes:
_, val_acc = test(args,
model,
device,
dataloader_kwargs,
etime=eval_counter)
return val_acc
# Spawn the worker processes. Each runs an independent call of the train
# function
processes = []
for rank in range(s_rank, args.num_processes):
p = mp.Process(target=train,
args=(rank, args, model, device, dataloader_kwargs))
p.start()
processes.append(p)
logging.info('Started %s', p.pid)
log = []
# While any process is alive, continuously evaluate accuracy - the master
# thread is the evaluation thread
with tqdm(inf_iter(processes),
position=0,
desc='Testing',
total=float("inf"),
unit='Validation') as tbar:
for p_status in tbar:
if p_status is False:
break
# log in test
_, val_acc = test(args,
model,
device,
dataloader_kwargs,
etime=eval_counter)
log.append({'vacc': val_acc, 'time': eval_counter})
# tqdm.write(f'Accuracy is {vacc}')
logging.info('Accuracy is %s', val_acc)
eval_counter += 1
tbar.set_postfix(acc=val_acc)
# update checkpoint
torch.save({
'net': model.state_dict(),
'acc': val_acc
}, ckpt_output_fname)
time.sleep(60)
# open eval log as append in case we're simulating and the attack thread
# added some data
with open(f"{outdir}/eval", 'a') as eval_f:
writer = csv.DictWriter(eval_f, fieldnames=['time', 'vacc'])
for dat in log:
writer.writerow(dat)
# There should be no processes left alive by this point, but do this anyway
# to make sure no orphaned processes are left behind
for proc in processes:
os.system("kill -9 {}".format(proc.pid))
return val_acc
test_loader = DataLoader(test_opt)
logger.info('Loading model: %s', opt.model_file)
checkpoint = torch.load(opt.model_file)
checkpoint_opt = checkpoint['opt']
opt.model_type = checkpoint_opt.model_type
opt.vocab = checkpoint_opt.vocab
opt.vocab_size = checkpoint_opt.vocab_size
opt.seq_length = checkpoint_opt.seq_length
opt.feat_dims = checkpoint_opt.feat_dims
assert opt.vocab_size == test_loader.get_vocab_size()
assert opt.seq_length == test_loader.get_seq_length()
assert opt.feat_dims == test_loader.get_feat_dims()
logger.info('Building model...')
model = CaptionModel(opt)
logger.info('Loading state from the checkpoint...')
model.load_state_dict(checkpoint['model'])
xe_criterion = CrossEntropyCriterion()
if torch.cuda.is_available():
model.cuda()
xe_criterion.cuda()
logger.info('Start testing...')
test(model, xe_criterion, test_loader, opt)
logger.info('Time: %s', datetime.now() - start)
shuffle=False)
# Instantiate the PyTorch model
x, y = dataset[args.dataset_split][0]
model_cfg['input_length'] = len(x)
model_cfg['output_length'] = len(y)
model = JetSubstructureNeqModel(model_cfg)
# Load the model weights
checkpoint = torch.load(options_cfg['checkpoint'], map_location='cpu')
model.load_state_dict(checkpoint['model_dict'])
# Test the PyTorch model
print("Running inference on baseline model...")
model.eval()
baseline_accuracy = test(model, test_loader, cuda=False)
print("Baseline accuracy: %f" % (baseline_accuracy))
# Instantiate LUT-based model
lut_model = JetSubstructureLutModel(model_cfg)
lut_model.load_state_dict(checkpoint['model_dict'])
# Generate the truth tables in the LUT module
print("Converting to NEQs to LUTs...")
generate_truth_tables(lut_model, verbose=True)
# Test the LUT-based model
print("Running inference on LUT-based model...")
lut_inference(lut_model)
lut_model.eval()
lut_accuracy = test(lut_model, test_loader, cuda=False)
train=True,
download=True,
transform=transform)
dataset2 = datasets.MNIST('../data', train=False, transform=transform)
kwargs = {'batch_size': args.batch_size, 'shuffle': True}
if use_cuda:
kwargs.update({
'num_workers': 1,
'pin_memory': True,
})
torch.manual_seed(args.seed)
mp.set_start_method('spawn')
model = Net().to(device)
model.share_memory(
) # gradients are allocated lazily, so they are not shared here
processes = []
for rank in range(args.num_processes):
p = mp.Process(target=train,
args=(rank, args, model, device, dataset1, kwargs))
# We first train the model across `num_processes` processes
p.start()
processes.append(p)
for p in processes:
p.join()
# Once training is complete, we can test the model
test(args, model, device, dataset2, kwargs)
parser.add_argument("--lr", default=3.16e-5, type=float, dest="lr")
parser.add_argument("--num_epoch", default=150000, type=int, dest="num_epoch")
parser.add_argument("--batch_size", default=40, type=int, dest="batch_size")
parser.add_argument("--use_gpu", default=False, type=bool, dest="use_gpu")
parser.add_argument("--ckpt_dir",
default='./checkpoint',
type=str,
dest="ckpt_dir")
parser.add_argument("--log_dir", default='./log', type=str, dest="log_dir")
parser.add_argument("--result_dir",
default="./result/",
type=str,
dest="result_dir")
parser.add_argument('--mode',
default='train',
choices=['train', 'test'],
dest='mode')
parser.add_argument('--train_continue',
default='off',
choices=['on', 'off'],
dest='train_continue')
args = parser.parse_args()
if __name__ == "__main__":
if args.mode == "train":
train(args)
elif args.mode == "test":
test(args)
def run_test(self, classifier, transforms, showimage=True):
res = {"acc": [], "act_correct": [], "act_incorrect": []}
res_list = []
act_all_list = []
act_correct_list = []
act_incorrect_list = []
roc_list = []
brier_list = []
img = []
for i, t in enumerate(transforms):
ds = TransformDS(self.testset, t)
dl = DataLoader(ds, batch_size=self.batch_size, num_workers=0)
# Run evaluation
acc, act, metrics = train.test(classifier,
dl,
self.device,
num_classes=self.num_classes)
res["acc"].append(acc)
res["act_correct"].append(metrics["activation_correct_mean"])
res["act_incorrect"].append(metrics["activation_incorrect_mean"])
res_list.append(acc)
act_all_list.append(act.item())
act_correct_list.append(metrics["activation_correct_mean"].item())
act_incorrect_list.append(
metrics["activation_incorrect_mean"].item())
roc_list.append(metrics["roc"])
brier_list.append(metrics["brier"])
# Print test images
if showimage:
#print("Level "+ str(i+1))
for counter, data in enumerate(dl):
x, y = data[0].to(self.device), data[1].to(self.device)
#utils.imshow(torchvision.utils.make_grid(x[:8,:,:,:], 8))
img.append(x[:8, :, :, :])
break
print("Accuracy:", acc, "Correct:", metrics["n_correct"],
"Incorrect:", metrics["n_incorrect"])
print("ROC:", metrics["roc"], "PR IN:", metrics["pr_in"],
"PR OUT:", metrics["pr_out"])
print("Brier:", metrics["brier"])
sns.distplot(metrics["activations_correct"],
hist=False,
label="Correct activations")
sns.distplot(metrics["activations_incorrect"],
hist=False,
label="Incorrect activations")
plt.show()
plt.clf()
# Res order - acc lvl 1-5, act lvl 1-5
for act in act_all_list:
res_list.append(act)
for act in act_correct_list:
res_list.append(act)
for act in act_incorrect_list:
res_list.append(act)
for roc in roc_list:
res_list.append(roc)
for brier in brier_list:
res_list.append(brier)
if showimage:
utils.gridshow(torch.cat(img))
return res, res_list
word_vec_list.append(torch.from_numpy(vector))
wordvec_matrix = torch.cat(word_vec_list)
print('oov: %s' %str(oov))
print(args.embed_num)
args.pretrained_weight = wordvec_matrix
args.kernel_sizes = [int(k) for k in args.kernel_sizes.split(',')]
cnn = model.CNN_Sim(args)
args.cuda = (not args.no_cuda) and torch.cuda.is_available(); del args.no_cuda
# if args.cuda:
# torch.cuda.set_device(args.device)
# cnn = cnn.cuda()
if args.snapshot is not None:
print('\nLoading model from {}...'.format(args.snapshot))
cnn.load_state_dict(torch.load(args.snapshot))
# if args.cuda:
# torch.cuda.set_device(args.device)
# cnn = cnn.cuda()
else:
try:
train(train_iter=train_iter, dev_iter=dev_iter, model=cnn, args=args)
except KeyboardInterrupt:
print(traceback.print_exc())
print('\n' + '-' * 89)
print('Exiting from training early')
test(test_iter=test_iter, model=cnn, args=args)
train_table,
model,
model_bert,
opt,
bert_config,
tokenizer,
config["max_seq_length"],
config["num_target_layers"],
config["accumulate_gradients"],
opt_bert=opt_bert,
st_pos=0,
path_db=path_wikisql,
dset_name='train')
# check DEV
with torch.no_grad():
acc_dev, results_dev, count_list = test(dev_loader,
dev_table,
model,
model_bert,
bert_config,
tokenizer,
config["max_seq_length"],
config["num_target_layers"],
detail=False,
path_db=path_wikisql,
st_pos=0,
dset_name='dev', EG=config["EG"])
print_result(epoch, acc_train, 'train')
print_result(epoch, acc_dev, 'dev')
for ax in ax_cmap:
ax.set(xticklabels=[])
ax.set(yticklabels=[])
# TRAIN MODELS AND PLOT
time_steps = 50
epochs = 15
num_predict = 30
start_time = time.time()
model = rnn_ode.RNN_ODE(1, 4, 1, device_params, time_steps)
losses = train.train(train_data, model, epochs)
model.node_rnn.observe(True)
# model.use_cb(True)
output, times = train.test(test_start[0][0], test_start[0][1], num_predict,
model)
# model.use_cb(False)
#ax1.scatter(torch.cat((x.view(-1)[cutoff + tw - 1].view(-1), times.view(-1)), axis=0),
# torch.cat((y.view(-1)[cutoff + tw - 1].view(-1), output.view(-1)), axis=0),
# edgecolors='k',
# facecolors='none')
# Predictions
ax1.plot(
torch.cat((x.view(-1)[cutoff + tw - 1].view(-1), times.view(-1)), axis=0),
torch.cat((y.view(-1)[cutoff + tw - 1].view(-1), output.view(-1)), axis=0),
'o-',
linewidth=0.5,
color='deepskyblue',
start = time.time()
patience = args.patience
savePath = args.savePath
train_path = "../data/dummy/classification_train.csv"
test_path = "../data/dummy/classification_test.csv"
for ei in range(args.max_epochs):
bad_counter = 0
best_loss = -1.0
train_main(args, model, train_path, criterion, optimizer)
dayloss, valid_loss = test(args, model, test_path, criterion)
print("valid loss : {}".format(valid_loss))
print(dayloss)
if valid_loss < best_loss or best_loss < 0:
print("find best")
bad_counter = 0
torch.save(model, args.saveModel)
else:
bad_counter += 1
if bad_counter > patience:
print('Early Stopping')
break
# Draw a sample
import matplotlib
) # gradients are allocated lazily, so they are not shared here
# ================================================================================
processes = []
# Following loop creates 2 processes
# and they process training in parallel lazily (?)
for rank in range(args.num_processes):
p = mp.Process(target=train,
args=(rank, args, model, device, dataloader_kwargs))
# print("p",p)
# <Process(Process-1, initial)>
# We first train the model across `num_processes` processes
p.start()
# print("p",p)
# <Process(Process-1, started)>
processes.append(p)
# ================================================================================
# print("processes",processes)
# [<Process(Process-1, started)>, <Process(Process-2, started)>]
for p in processes:
p.join()
# Once training is complete, we can test the model
test(args, model, device, dataloader_kwargs)
y_ax = []
l_test = []
for epoch in range(1, args.epochs + 1):
train(args,
model,
fog_graph,
workers,
X_trains,
y_trains,
device,
epoch,
loss_fn='averaging',
rounds=0,
radius=0,
d2d=0)
acc, loss = test(args, model, device, test_loader, best, epoch)
y_ax.append(acc)
x_ax.append(epoch)
l_test.append(loss)
if args.save_model and acc > best:
best = acc
torch.save(model.state_dict(), best_path)
print('Model best @ {}, acc {}: {}\n'.format(
epoch, acc, best_path))
if (args.save_model):
torch.save(model.state_dict(), stop_path)
print('Model stop: {}'.format(stop_path))
hist_file = '../ckpts/history_{}.pkl'
x = x.view(-1, 320)
x = F.relu(self.fc1(x))
x = F.dropout(x, training=self.training)
x = self.fc2(x)
return F.log_softmax(x, dim=1)
if __name__ == '__main__':
args = parser.parse_args()
use_cuda = args.cuda and torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
dataloader_kwargs = {'pin_memory': True} if use_cuda else {}
torch.manual_seed(args.seed)
mp.set_start_method('spawn')
model = Net().to(device)
model.share_memory() # gradients are allocated lazily, so they are not shared here
processes = []
for rank in range(args.num_processes):
p = mp.Process(target=train, args=(rank, args, model, device, dataloader_kwargs))
# We first train the model across `num_processes` processes
p.start()
processes.append(p)
for p in processes:
p.join()
# Once training is complete, we can test the model
test(args, model, device, dataloader_kwargs)
import train
R = [[3, 3, 3, 3, 3],
[2, 1, 3, 1, 2],
[5, 5, 3, 1, 2],
[1, 1, 2, 4, 5],
[4, 5, 4, 5, 4],
[2, 2, 3, 4, 4],
[5, 5, 1, 1, 1]]
R = np.array(R)
users = np.array([1,4,3,2,6,4,3,7,5], dtype=np.uint32)
movies = np.array([2,5,4,1,3,2,1,2,5], dtype=np.uint16)
ratings = np.array([3,5,1,2,3,1,5,5,4], dtype=np.uint8)
data = (users, movies, ratings)
tusers = np.array([1,6,3,2,5], dtype=np.uint32)
tmovies = np.array([5,2,1,3,2], dtype=np.uint16)
tratings = np.array([3,2,5,3,5], dtype=np.uint8)
tdata = (tusers, tmovies, tratings)
train.train(data)
gratings = train.test(tdata)
print gratings
print tratings
errs = tratings - gratings
errs = errs * errs
print sum(errs)/len(tratings)
print