def resize_images(imagePath, resizedPath, size=[800, 800]):
cv = Convnet(False, False, size, form=True)
sizeTxt = '{}x{}'.format(size[0], size[1])
lastResized = getLastFromDir(resizedPath + '/' + sizeTxt)
for subdir, dirs, files in os.walk(imagePath):
lenFiles = len(files)
break
nameList = list(x + lastResized for x in range(lenFiles))
x = 0
for subdir, dirs, files in os.walk(imagePath):
for file in files:
path = subdir.replace("""\ """, '/') + '/' + file
try:
img = imread(path).astype(np.float32)
if img.shape[2] != 3:
newImg = np.zeros((img.shape[0], img.shape[1], 3))
for x in range(len(newImg)):
for y in range(len(newImg[x])):
for z in range(len(newImg[x][y])):
newImg[x][y][z] = img[x][y][z]
img = newImg
except Exception as e:
print(e)
print(file)
continue
img = img.reshape(img.shape[0], img.shape[1], 3)
image = cv.format_images([img])[0]
name = nameList[random.randrange(len(nameList))]
scipy.misc.imsave(
resizedPath + '/' + sizeTxt + '/' + '{}.png'.format(x), image)
x += 1
nameList.remove(name)
lastResized += 1
def __init__(self, memory_size=1000, crop_size=192, z_size=32, lr=LR):
self.dqn, self.target = Convnet().cuda(), Convnet().cuda()
self.crop_size = crop_size
self.z_size = z_size
self.memory_size = memory_size
self.mb_pool = np.zeros([memory_size, crop_size, crop_size, z_size])
self.maft_pool = np.zeros([memory_size, crop_size, crop_size, z_size])
self.reward_pool = np.zeros([memory_size, 1])
self.action_pool = np.zeros([memory_size, 1])
self.count = 0
self.learn_step_counter = 0
self.optimizer = torch.optim.Adam(self.dqn.parameters(), lr=lr)
self.loss_func = nn.MSELoss()
def create_model(model_opts, sys_opts, device):
if model_opts.classifier == 'maml':
model = ModelConvMiniImagenet(5, hidden_size=64)
model.load_state_dict(
torch.load(os.path.join(sys_opts.root, sys_opts.load_path)))
model.classifier = nn.Linear(12544, 1000)
return model
if model_opts.backbone == 'resnet-18':
backbone = models.resnet18(pretrained=model_opts.pretrained)
if model_opts.path_to_model is not None:
pretrained_model_dict = torch.load(model_opts.path_to_model)
backbone.load_state_dict(pretrained_model_dict)
elif model_opts.backbone == 'resnet-34':
backbone = models.resnet34(pretrained=model_opts.pretrained)
elif model_opts.backbone == 'resnet-50':
backbone = models.resnet50(pretrained=model_opts.pretrained)
if model_opts.path_to_model is not None:
pretrained_model_dict = torch.load(model_opts.path_to_model)
backbone.load_state_dict(pretrained_model_dict)
elif model_opts.backbone == 'mobilenetv2':
backbone = models.mobilenet_v2(pretrained=model_opts.pretrained)
elif model_opts.backbone == 'densenet-161':
backbone = models.densenet161(pretrained=model_opts.pretrained)
elif model_opts.backbone == 'convnet':
backbone = Convnet()
else:
sys.exit("Given model not in predefined set of models")
if model_opts.classifier == 'knn':
backbone = extract_backbone(backbone)
if model_opts.similarity_measure == 'euclidean':
measure = euclidean_metric
elif model_opts.similarity_measure == 'cosine':
measure = cosine_sim
model = KNN(backbone, measure)
elif model_opts.classifier == 'linear':
model = backbone
elif model_opts.classifier == 'split':
#TODO consider refactoring this to take the base classes as array rather than loading them from file
model = SplitModel(backbone, model_opts.split_layers,
sys_opts.sequence_num, sys_opts.root,
model_opts.num_classes, device)
elif model_opts.classifier == 'hybrid':
model = backbone
backbone = extract_backbone(backbone)
if model_opts.similarity_measure == 'euclidean':
measure = euclidean_metric
elif model_opts.similarity_measure == 'cosine':
measure = cosine_sim
elif model_opts.similarity_measure == 'dot':
measure = dot_product
model = Hybrid(backbone, measure, model)
elif model_opts.classifier == 'ptn':
measure = euclidean_metric
backbone = extract_backbone(backbone)
model = KNN(backbone, measure)
model.load_state_dict(
torch.load(os.path.join(sys_opts.root, sys_opts.load_path)))
else:
sys.exit("Given classifier not in predefined set of classifiers")
return model
def test_convnet_dropout(self):
#Test that dropout is performed correctly
tensorflow.reset_default_graph()
network = Convnet(self.params,
tensorflow.placeholder(tensorflow.float32,
(None, 128, 128, 3),
name="x"),
training=True)
assert sum(["dropout" in v.name
for v in network.var_dict.values()]) == 2
tensorflow.reset_default_graph()
network = Convnet(self.params,
tensorflow.placeholder(tensorflow.float32,
(None, 128, 128, 3),
name="x"),
training=False)
assert sum(["dropout" in v.name
for v in network.var_dict.values()]) == 0
def save_as_array(imagePath, arrayPath, size=[800, 800]):
cv = Convnet(False, False, size, form=True)
sizeTxt = '{}x{}'.format(size[0], size[1])
lastArray = getLastFromDir(arrayPath + '/' + sizeTxt)
batch = []
for subdir, dirs, files in os.walk(imagePath):
for file in files:
path = subdir.replace("""\ """, '/') + '/' + file
img = imread(path).astype(np.float32)
batch.append(img)
if len(batch) >= batch_size:
with open(
arrayPath + '/' + sizeTxt + '/' + str(lastArray + 1) +
'.p', "wb") as f:
batch = cv.format_images(batch)
lastArray += 1
pickle.dump(batch, f)
batch = []
with open(arrayPath + '/' + str(lastArray + 1) + '.p', "wb") as f:
pickle.dump(batch, f)
def build_graph(params, training): """ Build the convnet and the placeholder variables """ tf.reset_default_graph() x = tf.placeholder(tf.float32, (None, 128, 128, 3), name="x") #the input variable y = tf.placeholder(tf.float32, (None, 2), name="y") #the output variable #Build the network and get the updatw network = Convnet(params, x, training=training) cost, updt = ch.get_cost_updt(params, network.pred, y) return x, y, network, cost, updt
def main(opts):
pprint(vars(opts))
device = torch.device(opts.device)
print('using device: {}'.format(device))
encoder = Convnet().to(device)
encoder.load_state_dict(torch.load(opts.weights))
encoder.eval()
pipeline = protoPipe(encoder, opts.shot_k, opts.query_k)
test_set = MNIST(mode='test')
test_sampler = TaskSampler(test_set.label, 1000, opts.n_way,
opts.shot_k + opts.query_k)
test_loader = DataLoader(dataset=test_set,
batch_sampler=test_sampler,
num_workers=8,
pin_memory=True)
test_acc = []
for episode, batch in enumerate(test_loader, 0):
task = batch[0].view(opts.n_way * (opts.shot_k + opts.query_k), 3, 84,
84)
loss, acc = pipeline(task.to(device), opts.n_way)
test_acc.append(acc)
m, h = mean_confidence_interval(test_acc)
print('TEST set acc: {:.4f}, h: {:.4f}'.format(m, h))
def test_convnet_naming(self):
#Test that layers are name scoped properly
tensorflow.reset_default_graph()
network = Convnet(self.params,
tensorflow.placeholder(tensorflow.float32,
(None, 128, 128, 3),
name="x"),
training=True)
for k, v in network.var_dict.items():
if "conv" in k:
assert k in v.name
tvars = tensorflow.trainable_variables()
for t in tvars:
assert "conv" in t.name or "fc" in t.name or "prediction" in t.name
def test_gradient_connectivity(self):
# Make sure that all trainable variables have gradients computed when we backprop from error
tensorflow.reset_default_graph()
network = Convnet(self.params,
tensorflow.placeholder(tensorflow.float32,
(None, 128, 128, 3),
name="x"),
training=True)
y = tensorflow.placeholder(tensorflow.float32, (None, 2),
name="y") #the output variable
cost, updt = ch.get_cost_updt(self.params, network.pred, y)
gvs = tensorflow.train.AdamOptimizer(
learning_rate=self.params["learning_rate"]).compute_gradients(cost)
gvnames = set([g[1] for g in gvs])
for v in tensorflow.trainable_variables():
assert v in gvnames
def main(opts):
pprint(vars(opts))
device = torch.device(opts.device)
print('using device: {}'.format(device))
encoder = Convnet().to(device)
encoder.load_state_dict(torch.load(opts.weights))
encoder.eval()
pipeline = protoPipe(encoder, opts.shot_k, opts.query_k)
if opts.seed is not None:
torch.manual_seed(opts.seed)
torch.cuda.manual_seed_all(opts.seed)
np.random.seed(opts.seed)
print("\nrandom seed: {}".format(opts.seed))
if opts.dataset == 'mini':
test_set = MiniImageNet(mode='test')
elif opts.dataset == 'MNIST':
test_set = MNIST(mode='test')
elif opts.dataset == 'CIFAR':
test_set = CIFAR(mode='test')
elif opts.dataset == 'FashionMNIST':
test_set = Fashion_MNIST(mode='test')
test_sampler = TaskSampler(test_set.label, 1000, opts.n_way,
opts.shot_k + opts.query_k)
test_loader = DataLoader(dataset=test_set,
batch_sampler=test_sampler,
num_workers=8,
pin_memory=True)
test_acc = []
for episode, batch in enumerate(test_loader, 0):
task = batch[0].view(opts.n_way * (opts.shot_k + opts.query_k), 3, 84,
84)
loss, acc = pipeline(task.to(device), opts.n_way)
test_acc.append(acc)
m, h = mean_confidence_interval(test_acc)
print('TEST set acc: {:.4f}, h: {:.4f}'.format(m, h))
def main(args):
device = torch.device(args.device)
ensure_path(args.save_path)
data = Data(args.dataset, args.n_batches, args.train_way, args.test_way, args.shot, args.query)
train_loader = data.train_loader
val_loader = data.valid_loader
model = Convnet(x_dim=2).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.5)
def save_model(name):
torch.save(model.state_dict(), osp.join(args.save_path, name + '.pth'))
trlog = dict(
args=vars(args),
train_loss=[],
val_loss=[],
train_acc=[],
val_acc=[],
max_acc=0.0,
)
timer = Timer()
for epoch in range(1, args.max_epoch + 1):
lr_scheduler.step()
model.train()
tl = Averager()
ta = Averager()
for i, batch in enumerate(train_loader, 1):
data, _ = [_.to(device) for _ in batch]
data = data.reshape(-1, 2, 105, 105)
p = args.shot * args.train_way
embedded = model(data)
embedded_shot, embedded_query = embedded[:p], embedded[p:]
proto = embedded_shot.reshape(args.shot, args.train_way, -1).mean(dim=0)
label = torch.arange(args.train_way).repeat(args.query).to(device)
logits = euclidean_metric(embedded_query, proto)
loss = F.cross_entropy(logits, label)
acc = count_acc(logits, label)
print('epoch {}, train {}/{}, loss={:.4f} acc={:.4f}'
.format(epoch, i, len(train_loader), loss.item(), acc))
tl.add(loss.item())
ta.add(acc)
optimizer.zero_grad()
loss.backward()
optimizer.step()
tl = tl.item()
ta = ta.item()
model.eval()
vl = Averager()
va = Averager()
for i, batch in enumerate(val_loader, 1):
data, _ = [_.cuda() for _ in batch]
data = data.reshape(-1, 2, 105, 105)
p = args.shot * args.test_way
data_shot, data_query = data[:p], data[p:]
proto = model(data_shot)
proto = proto.reshape(args.shot, args.test_way, -1).mean(dim=0)
label = torch.arange(args.test_way).repeat(args.query).to(device)
logits = euclidean_metric(model(data_query), proto)
loss = F.cross_entropy(logits, label)
acc = count_acc(logits, label)
vl.add(loss.item())
va.add(acc)
vl = vl.item()
va = va.item()
print('epoch {}, val, loss={:.4f} acc={:.4f}'.format(epoch, vl, va))
if va > trlog['max_acc']:
trlog['max_acc'] = va
save_model('max-acc')
trlog['train_loss'].append(tl)
trlog['train_acc'].append(ta)
trlog['val_loss'].append(vl)
trlog['val_acc'].append(va)
torch.save(trlog, osp.join(args.save_path, 'trlog'))
save_model('epoch-last')
if epoch % args.save_epoch == 0:
save_model('epoch-{}'.format(epoch))
print('ETA:{}/{}'.format(timer.measure(), timer.measure(epoch / args.max_epoch)))
logfile = open(osp.join(args.save_path, logname + '.txt'), 'w+')
pprint(vars(args))
set_gpu(args.gpu)
valset = MiniImageNet2('trainvaltest')
val_loader = DataLoader(dataset=valset, batch_size = 128,
num_workers=8, pin_memory=True)
valset2 = MiniImageNet2('trainval')
val_loader2 = DataLoader(dataset=valset2, batch_size = 128,
num_workers=8, pin_memory=True)
valset3 = MiniImageNet2('test')
val_loader3 = DataLoader(dataset=valset3, batch_size = 128,
num_workers=8, pin_memory=True)
model_cnn = Convnet().cuda()
model_cnn.load_state_dict(torch.load('./100way_pn_basenovel.pth'))
global_proto = torch.load('./global_proto_basenovel_PN_5shot_500.pth')
global_base =global_proto[:args.n_base_class,:]
global_novel = global_proto[args.n_base_class:,:]
global_base = [Variable(global_base.cuda(),requires_grad=True)]
global_novel = [Variable(global_novel.cuda(),requires_grad=True)]
def log(out_str):
print(out_str)
logfile.write(out_str+'\n')
logfile.flush()
model_cnn.eval()
for epoch in range(1, args.max_epoch + 1):
from data.datamgr import SetDataManager
from parser_util import parse_args, get_best_file
from utils import set_device, euclidean_dist
if __name__ == '__main__':
params = parse_args('test')
device = set_device(params)
acc_all = []
image_size = 84
iter_num = 600
few_shot_params = dict(n_way=params.test_n_way, n_support=params.n_shot)
model = Convnet()
model = model.to(device)
checkpoint_dir = '%s/checkpoints/%s_%dway_%dshot' % (
configs.save_dir, params.dataset, params.train_n_way, params.n_shot)
if params.train_aug:
checkpoint_dir += '_aug'
modelfile = get_best_file(checkpoint_dir)
if modelfile is not None:
tmp = torch.load(modelfile)
model.load_state_dict(tmp['state'])
split = params.split
if params.save_iter != -1:
split_str = split + "_" + str(params.save_iter)
ss_sampler = CategoriesSampler(ssdata.slabel, 100, args.train_way,
2 * args.shot + args.query)
ss_loader = DataLoader(dataset=ssdata,
batch_sampler=ss_sampler,
num_workers=args.num_workers,
pin_memory=True)
valset = MiniImageNet('val')
val_sampler = CategoriesSampler(valset.label, 400, args.test_way,
args.shot + args.query)
val_loader = DataLoader(dataset=valset,
batch_sampler=val_sampler,
num_workers=args.num_workers,
pin_memory=True)
model = Convnet().cuda()
if args.load is not 'na':
print('Loading Model')
model.load_state_dict(torch.load(args.load))
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=20,
gamma=0.5)
def save_model(name):
torch.save(model.state_dict(), osp.join(args.save_path, name + '.pth'))
timer = Timer()
# s_label = torch.arange(args.train_way).repeat(args.shot).view(args.shot * args.train_way)
# s_onehot = torch.zeros(s_label.size(0), 20)
ensure_path(args.save_path)
writer = SummaryWriter()
#noise = torch.distributions.Normal(loc=0, scale=.02)
trainset = MiniImageNet('train')
train_sampler = CategoriesSampler(trainset.label, 100,
args.train_way, args.shot + args.query)
train_loader = DataLoader(dataset=trainset, batch_sampler=train_sampler,
num_workers=args.num_workers, pin_memory=True)
valset = MiniImageNet('val')
val_sampler = CategoriesSampler(valset.label, 400,
args.test_way, args.shot + args.query)
val_loader = DataLoader(dataset=valset, batch_sampler=val_sampler,
num_workers=args.num_workers, pin_memory=True)
model = torch.nn.DataParallel(Convnet())
if args.load is not 'na':
model.load_state_dict(torch.load(args.load))
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.5)
def save_model(name):
torch.save(model.state_dict(), osp.join(args.save_path, name + '.pth'))
timer = Timer()
s_label = torch.arange(args.train_way).repeat(args.shot).view(args.shot * args.train_way)
s_onehot = torch.zeros(s_label.size(0), 20)
s_onehot = s_onehot.scatter_(1, s_label.unsqueeze(dim=1), 1).cuda()
trainset = DiabeticRetinopathy('train')
train_sampler_ = PrototypicalBatchSampler(trainset.label, 3, train_way,
shot + query)
train_loader = DataLoader(dataset=trainset,
batch_sampler=train_sampler_,
num_workers=8)
valset = DiabeticRetinopathy('val')
val_sampler = PrototypicalBatchSampler(valset.label, 4, test_way,
shot + query)
val_loader = DataLoader(dataset=valset,
batch_sampler=val_sampler,
num_workers=8,
pin_memory=True)
model = Convnet()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=20,
gamma=0.5)
def save_model(name):
torch.save(model.state_dict(), osp.join(save_path, name + '.pth'))
trlog = {}
trlog['args'] = vars(args)
trlog['train_loss'] = []
trlog['val_loss'] = []
trlog['train_acc'] = []
trlog['val_acc'] = []
trlog['max_acc'] = 0.0
train_loader = DataLoader(dataset=trainset,
batch_sampler=train_sampler,
num_workers=8,
pin_memory=True)
#valset = MiniImageNet('test')
valset = MiniImageNet('trainvaltest')
val_sampler = CategoriesSampler_val_100way(valset.label, 400,
args.test_way, args.shot,
args.query_val)
val_loader = DataLoader(dataset=valset,
batch_sampler=val_sampler,
num_workers=8,
pin_memory=True)
model_cnn = Convnet().cuda()
model_reg = Registrator().cuda()
model_cnn.load_state_dict(
torch.load(
'./iterative_G3_trainval_lr150epoch_dataaugumentation_2epoch-175_backbone.pth'
))
noise_dim = 128
model_gen = Hallucinator(noise_dim).cuda()
model_gen.load_state_dict(
torch.load(
'./iterative_G3_trainval_lr150epoch_dataaugumentation_2epoch-175_gen.pth'
))
global_proto = torch.load('./global_proto_all_new.pth')
global_base = global_proto[:args.n_base_class, :]
num_workers=8,
pin_memory=True)
valset = MiniImageNet('val')
val_sampler = CategoriesSampler(valset.label, 400, args.test_way,
args.shot + args.query)
val_loader = DataLoader(dataset=valset,
batch_sampler=val_sampler,
num_workers=8,
pin_memory=True)
if args.multi == False:
gen = generator(1600, 1).cuda()
else:
gen = generator(1600, 1600).cuda()
model = Convnet().cuda()
#gradient_mean=Bufferswitch()
optimizer = torch.optim.Adam(list(model.parameters()) +
list(gen.parameters()),
lr=0.001)
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=20,
gamma=0.5)
def save_model(name):
torch.save(model.state_dict(), osp.join(save_path, name + '.pth'))
def save_gen(name):
torch.save(gen.state_dict(), osp.join(save_path, name + '.pth'))
dis = torch.pow(cls_proto_ext-query_feats_ext, 2).sum(dim=2)
relations = F.log_softmax(-dis, dim=1)
# print(dis.mean())
loss = -relations.gather(1, query_labels.view(-1,1)).mean()
pred = torch.argmax(relations, dim=1)
acc = (pred==query_labels.view(-1)).float().mean().item()
# print((pred==query_labels.view(-1)).float().sum().item())
return loss, acc
if '__main__' == __name__:
from torch.utils.data import DataLoader
from data import MiniImageNet, MiniImageNetSampler
trainset = MiniImageNet('train')
train_sampler = MiniImageNetSampler(trainset.label, 2,
20, 5+15)
train_loader = DataLoader(dataset=trainset, batch_sampler=train_sampler,
num_workers=8, pin_memory=True)
convnet = Convnet().cuda()
pipe = protoPipe(convnet)
for epoch, batch in enumerate(train_loader, 0):
print(epoch)
batch = batch[0]
loss, acc = pipe(batch.cuda(), 20)
loss.backward()
print(loss)
return history
if __name__ == '__main__':
args = parser.parse_args()
# limit gpu memory
if args.limit > 0:
utils.limit_gpu_memory(args.limit)
# prepare model
if args.resume:
model = load_model(args.model_path)
else:
model = Convnet(args.max_len, args.win_size)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['acc'])
# prepare data
# see more in [exmaple.csv](https://github.com/j40903272/MalConv-keras/blob/master/example.csv) (1:benign, 0:malicious)
#
# preprocess is handled in utils.data_generator
df = pd.read_csv(args.csv, header=None)
data, label = df[0].values, df[1].values
x_train, x_test, y_train, y_test = utils.train_test_split(
data, label, args.val_size)
print('Train on %d data, test on %d data' % (len(x_train), len(x_test)))
history = train(model, args.max_len, args.batch_size, args.verbose,
def visualize_convnet():
sal_map_type = "Deconv_maxlogit" # change it to get different visualizations
image_name = 'tabby' # or using a list to deal with multiple images
max_pool = True #indicate whether to add a max-pooling layer
data_dir = "../data"
save_dir = "results"
if not os.path.exists(save_dir):
os.mkdir(save_dir)
n_labels = 1000
n_input = 224
fns = []
image_list = []
label_list = []
# load in the original image and its adversarial examples
for image_path in glob.glob(os.path.join(data_dir, '{}.png'.format(image_name))):
fns.append(os.path.basename(image_path).split('.')[0])
image = imread(image_path, mode='RGB')
image = imresize(image, (n_input, n_input)).astype(np.float32)
image_list.append(image)
onehot_label = np.array([1 if i == image_dict[image_name] else 0 for i in range(n_labels)])
label_list.append(onehot_label)
batch_img = np.array(image_list)
batch_label = np.array(label_list)
batch_size = batch_img.shape[0]
# tf session
sess = tf.Session()
# construct the graph based on the gradient type we want
# plain relu vs guidedrelu
if sal_map_type.split('_')[0] == 'GuidedBackprop':
eval_graph = tf.get_default_graph()
with eval_graph.gradient_override_map({'Relu': 'GuidedRelu'}):
conv_model = Convnet(sess,max_pool)
# ADD DECONV
elif sal_map_type.split('_')[0] == 'Deconv':
eval_graph = tf.get_default_graph()
with eval_graph.gradient_override_map({'Relu': 'DeconvRelu'}):
conv_model = Convnet(sess,max_pool)
elif sal_map_type.split('_')[0] == 'PlainSaliency':
conv_model = Convnet(sess,max_pool)
else:
raise Exception("Unknown saliency type")
# saliency gradient to input layer
if sal_map_type.split('_')[1] == "cost":
sal_map = tf.gradients(conv_model.cost, conv_model.imgs)[0]
elif sal_map_type.split('_')[1] == 'maxlogit':
sal_map = tf.gradients(conv_model.maxlogit, conv_model.imgs)[0]
elif sal_map_type.split('_')[1] == 'randlogit':
sal_map = tf.gradients(conv_model.logits[:, random.randint(0, 999)], conv_model.imgs)[0]
else:
raise Exception("Unknown logit gradient type")
# predict
probs = sess.run(conv_model.probs, feed_dict={conv_model.images: batch_img})
# sal_map
sal_map_val = sess.run(sal_map, feed_dict={conv_model.images: batch_img, conv_model.labels: batch_label})
for idx in range(batch_size):
print_prob(probs[idx])
visualize(sal_map_val[idx], sal_map_type, save_dir, fns[idx])
def main(opts):
pprint(vars(opts))
ensure_path(opts.name)
with open(osp.join(opts.name, 'settings.txt'), 'w') as f:
dic = vars(opts)
f.writelines(["{}: {}\n".format(k, dic[k]) for k in dic])
if opts.seed is not None:
torch.manual_seed(opts.seed)
torch.cuda.manual_seed_all(opts.seed)
np.random.seed(opts.seed)
print("\nrandom seed: {}".format(opts.seed))
writer = SummaryWriter('./{}/run'.format(opts.name))
device = torch.device(opts.device)
print('using device: {}'.format(device))
train_set = MiniImageNet(mode='train')
train_sampler = MiniImageNetSampler(train_set.label, 100, opts.c_sampled,
opts.shot_k + opts.query_k)
train_loader = DataLoader(dataset=train_set,
batch_sampler=train_sampler,
num_workers=8,
pin_memory=True)
val_set = MiniImageNet(mode='val')
val_sampler = MiniImageNetSampler(val_set.label, 400, opts.n_way,
opts.shot_k + opts.query_k)
val_loader = DataLoader(dataset=val_set,
batch_sampler=val_sampler,
num_workers=8,
pin_memory=True)
encoder = Convnet().to(device)
# print(encoder.state_dict()['block4.0.bias'])
pipeline = protoPipe(encoder, opts.shot_k, opts.query_k)
pipeline.train()
optimizer = optim.Adam(
encoder.parameters(),
opts.lr,
)
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=2000,
gamma=0.5)
max_val_acc = 0
for epoch in range(opts.epoch // 100):
for episode, batch in enumerate(train_loader, 0):
lr_scheduler.step()
task = batch[0].view(opts.c_sampled * (opts.shot_k + opts.query_k),
3, 84, 84)
loss, acc = pipeline(task.to(device), opts.c_sampled)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if episode % 10 == 0:
writer.add_scalar("train_loss", loss.item(),
epoch * 100 + episode)
print(
'epoch: {}, episode: {} ,loss: {:.4f},acc: {:.4f}'.format(
epoch, episode, loss.item(), acc))
val_acc = []
pipeline.eval()
for episode, batch in enumerate(val_loader, 0):
task = batch[0].view(opts.n_way * (opts.shot_k + opts.query_k), 3,
84, 84)
loss, acc = pipeline(task.to(device), opts.n_way)
val_acc.append(acc)
m, h = mean_confidence_interval(val_acc)
writer.add_scalar("val_acc", m, epoch * 100)
print('VAL set acc: {:.4f}, h: {:.4f}'.format(m, h))
if epoch % (opts.epoch // 20) == 0:
torch.save(pipeline.encoder.state_dict(),
osp.join(opts.name, 'epoch_{}.pth'.format(epoch)))
if m > max_val_acc:
max_val_acc = m
torch.save(pipeline.encoder.state_dict(),
osp.join(opts.name, 'max_acc.pth'))
pipeline.train()
def main():
sal_map_type = "GuidedBackprop_maxlogit"
# sal_map_type = "PlainSaliency_maxlogit"
data_dir = "../VGGImagenet/data_imagenet"
save_dir = "results/10222017/convnet"
# TODO: extend this part to a list
image_name = 'tabby'
n_labels = 10
n_input = 224
layers = [
'conv1_1', 'conv1_2', 'pool1', 'conv2_1', 'conv2_2', 'pool2',
'conv3_1', 'conv3_2', 'conv3_3', 'pool3', 'conv4_1', 'conv4_2',
'conv4_3', 'pool4', 'conv5_1', 'conv5_2', 'conv5_3', 'pool5', 'fc1',
'fc2', 'fc3'
]
fns = []
image_list = []
label_list = []
# load in the original image and its adversarial examples
for image_path in glob.glob(
os.path.join(data_dir, '{}.png'.format(image_name))):
fns.append(os.path.basename(image_path).split('.')[0])
image = imread(image_path, mode='RGB')
image = imresize(image, (n_input, n_input)).astype(np.float32)
image_list.append(image)
onehot_label = np.array(
[1 if i == image_dict[image_name] else 0 for i in range(n_labels)])
label_list.append(onehot_label)
batch_img = np.array(image_list)
batch_label = np.array(label_list)
batch_size = batch_img.shape[0]
# tf session
sess = tf.Session()
# construct the graph based on the gradient type we want
# plain relu vs guidedrelu
if sal_map_type.split('_')[0] == 'GuidedBackprop':
eval_graph = tf.get_default_graph()
with eval_graph.gradient_override_map({'Relu': 'GuidedRelu'}):
conv_model = Convnet(sess)
elif sal_map_type.split('_')[0] == 'NGuidedBackprop':
eval_graph = tf.get_default_graph()
with eval_graph.gradient_override_map({'Relu': 'NGuidedRelu'}):
# load the vgg graph
# plain_init = true -> load the graph with random weights
# plain_init = false -> load the graph with pre-trained weights
conv_model = Convnet(sess)
elif sal_map_type.split('_')[0] == 'PlainSaliency':
# load the vgg graph
# plain_init = true -> load the graph with random weights
# plain_init = false -> load the graph with pre-trained weights
conv_model = Convnet(sess)
else:
raise Exception("Unknown saliency_map type - 1")
# --------------------------------------------------------------------------
# Visualize grad-camp and its adversarial examples
# --------------------------------------------------------------------------
# Get last convolutional layer gradient for generating gradCAM visualization
target_conv_layer = conv_model.convnet_out
if sal_map_type.split('_')[1] == "cost":
conv_grad = tf.gradients(conv_model.cost, target_conv_layer)[0]
elif sal_map_type.split('_')[1] == 'maxlogit':
conv_grad = tf.gradients(conv_model.maxlogit, target_conv_layer)[0]
elif sal_map_type.split('_')[1] == 'randlogit':
conv_grad = tf.gradients(conv_model.logits[0], target_conv_layer)[0]
# conv_grad = tf.gradients(conv_model.logits[random.randint(0, 999)], target_conv_layer)[0]
else:
raise Exception("Unknown saliency_map type - 2")
# normalization
conv_grad_norm = tf.div(conv_grad, tf.norm(conv_grad) + tf.constant(1e-5))
# saliency gradient to input layer
if sal_map_type.split('_')[1] == "cost":
sal_map = tf.gradients(conv_model.cost, conv_model.imgs)[0]
elif sal_map_type.split('_')[1] == 'maxlogit':
sal_map = tf.gradients(conv_model.maxlogit, conv_model.imgs)[0]
elif sal_map_type.split('_')[1] == 'randlogit':
sal_map = tf.gradients(conv_model.logits[0], conv_model.imgs)[0]
# sal_map = tf.gradients(conv_model.logits[random.randint(0, 999)], conv_model.imgs)[0]
else:
raise Exception("Unknown saliency_map type - 2")
# predict
probs = sess.run(conv_model.probs,
feed_dict={conv_model.images: batch_img})
# sal_map and conv_grad
sal_map_val, target_conv_layer_val, conv_grad_norm_val =\
sess.run([sal_map, target_conv_layer, conv_grad_norm],
feed_dict={conv_model.images: batch_img, conv_model.labels: batch_label})
for idx in range(batch_size):
print_prob(probs[idx])
visualize(batch_img[idx], target_conv_layer_val[idx],
conv_grad_norm_val[idx], sal_map_val[idx], sal_map_type,
save_dir, fns[idx], probs[idx])
else:
params.stop_epoch = 600 # default
train_few_shot_params = dict(n_way=params.train_n_way,
n_support=params.n_shot)
base_datamgr = SetDataManager(image_size,
n_query=params.query,
**train_few_shot_params)
base_loader = base_datamgr.get_data_loader(base_file, aug=params.train_aug)
test_few_shot_params = dict(n_way=params.test_n_way,
n_support=params.n_shot)
val_datamgr = SetDataManager(image_size,
n_query=params.query,
**test_few_shot_params)
val_loader = val_datamgr.get_data_loader(val_file, aug=False)
model = Convnet()
optimization = 'Adam'
params.checkpoint_dir = '%s/checkpoints/%s_%dway_%dshot' % (
configs.save_dir, params.dataset, params.train_n_way, params.n_shot)
if params.train_aug:
params.checkpoint_dir += '_aug'
ensure_path(params.checkpoint_dir)
start_epoch = params.start_epoch
stop_epoch = params.stop_epoch
model = train(base_loader, val_loader, model, optimization, start_epoch,
stop_epoch, params)
parser.add_argument('--way', type=int, default=5)
parser.add_argument('--shot', type=int, default=1)
parser.add_argument('--query', type=int, default=30)
parser.add_argument('--folds', type=int, default=2)
args = parser.parse_args()
pprint(vars(args))
set_gpu(args.gpu)
dataset = MiniImageNet('test')
sampler = CategoriesSampler(dataset.label,
args.batch, args.way, args.folds * args.shot + args.query)
loader = DataLoader(dataset, batch_sampler=sampler,
num_workers=8, pin_memory=True)
model = Convnet().cuda()
model.load_state_dict(torch.load(args.load))
model.eval()
ave_acc = Averager()
s_label = torch.arange(args.train_way).repeat(args.shot).view(args.shot * args.train_way)
s_onehot = torch.zeros(s_label.size(0), 20)
s_onehot = s_onehot.scatter_(1, s_label.unsqueeze(dim=1), 1).cuda()
for i, batch in enumerate(loader, 1):
data, _ = [_.cuda() for _ in batch]
k = args.way * args.shot
data_shot, meta_support, data_query = data[:k], data[k:2*k], data[2*k:]
#p = inter_fold(model, args, data_shot)
class Agent():
def __init__(self, memory_size=1000, crop_size=192, z_size=32, lr=LR):
self.dqn, self.target = Convnet().cuda(), Convnet().cuda()
self.crop_size = crop_size
self.z_size = z_size
self.memory_size = memory_size
self.mb_pool = np.zeros([memory_size, crop_size, crop_size, z_size])
self.maft_pool = np.zeros([memory_size, crop_size, crop_size, z_size])
self.reward_pool = np.zeros([memory_size, 1])
self.action_pool = np.zeros([memory_size, 1])
self.count = 0
self.learn_step_counter = 0
self.optimizer = torch.optim.Adam(self.dqn.parameters(), lr=lr)
self.loss_func = nn.MSELoss()
def setFixed(self, fixed):
self.fixed = fixed
def chooseAction(self, m):
m = torch.unsqueeze(torch.Tensor(m).cuda().float(), 0)
# input only one sample
if np.random.uniform() < EPSILON: # greedy
Q_pre = self.dqn(self.fixed, m)
action = torch.argmax(Q_pre)[1].cpu().numpy()
else: # random
action = np.random.randint(0, N_ACTIONS)
return action
def saveState(self, m_bf, a, reward, m_aft):
if self.count == self.memory_size:
self.flushMem()
self.mb_pool[self.count, ...] = m_bf
self.reward_pool[self.count, ...] = reward
self.maft_pool[self.count, ...] = m_aft
self.action_pool[self.count, ...] = a
self.count += 1
def flushMem(self):
self.count = 0
def saveCKPT(self, path):
torch.save(self.dqn.state_dict(), path)
def learnDQN(self):
if self.learn_step_counter % TARGET_REPLACE_ITER == 0:
self.target.load_state_dict(self.dqn.state_dict())
self.learn_step_counter += 1
# sample batch transitions
sample_index = np.random.choice(self.memory_size, BATCH_SIZE)
b_memory = self.mb_pool[sample_index]
aft_memory = self.maft_pool[sample_index]
r_memory = self.reward_pool[sample_index]
a_memory = self.action_pool[sample_index]
b_s = torch.Tensor(b_memory).float().cuda()
b_a = torch.Tensor(a_memory.astype(int)).long().cuda()
b_r = torch.Tensor(r_memory).float().cuda()
b_aft = torch.Tensor(aft_memory).float().cuda()
# q_eval w.r.t the action in experience
q_eval = self.dqn(b_s).gather(1, b_a) # shape (batch, 1)
q_next = self.target(
b_aft).detach() # detach from graph, don't backpropagate
q_target = b_r + GAMMA * q_next.max(1)[0].view(BATCH_SIZE,
1) # shape (batch, 1)
loss = self.loss_func(q_eval, q_target)
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
