def test_save_obj(self):
vertices, faces = neural_renderer.load_obj('./tests/data/teapot.obj')
neural_renderer.save_obj('./tests/data/teapot2.obj', vertices, faces)
vertices2, faces2 = neural_renderer.load_obj('./tests/data/teapot.obj')
os.remove('./tests/data/teapot2.obj')
assert np.allclose(vertices, vertices2)
assert np.allclose(faces, faces2)
def test_save_obj(self):
vertices, faces = neural_renderer.load_obj('./tests/data/teapot.obj')
neural_renderer.save_obj('./tests/data/teapot2.obj', vertices, faces)
vertices2, faces2 = neural_renderer.load_obj('./tests/data/teapot.obj')
os.remove('./tests/data/teapot2.obj')
assert np.allclose(vertices, vertices2)
assert np.allclose(faces, faces2)
def test_save_obj(self):
teapot = os.path.join(data_dir, 'teapot.obj')
teapot2 = os.path.join(data_dir, 'teapot2.obj')
vertices, faces = nr.load_obj(teapot)
nr.save_obj(teapot2, vertices, faces)
vertices2, faces2 = nr.load_obj(teapot2)
os.remove(teapot2)
assert torch.allclose(vertices, vertices2)
assert torch.allclose(faces, faces2)
def run():
# arguments
parser = argparse.ArgumentParser()
parser.add_argument('-eid', '--experiment_id', type=str)
parser.add_argument('-d', '--directory', type=str, default=DIRECTORY)
parser.add_argument('-i', '--input_image', type=str)
parser.add_argument('-oi', '--output_image', type=str)
parser.add_argument('-oo', '--output_obj', type=str)
parser.add_argument('-s', '--seed', type=int, default=RANDOM_SEED)
parser.add_argument('-g', '--gpu', type=int, default=GPU)
args = parser.parse_args()
directory_output = os.path.join(args.directory, args.experiment_id)
# set random seed, gpu
random.seed(args.seed)
np.random.seed(args.seed)
cp.random.seed(args.seed)
chainer.cuda.get_device(args.gpu).use()
# load dataset
image_in = skimage.io.imread(args.input_image).astype('float32') / 255
if image_in.ndim != 3 or image_in.shape[-1] != 4:
raise Exception('Input must be a RGBA image.')
images_in = image_in.transpose((2, 0, 1))[None, :, :, :]
images_in = chainer.cuda.to_gpu(images_in)
# setup model & optimizer
model = model_nview.Model(img_size=64)
model.to_gpu()
chainer.serializers.load_npz(os.path.join(directory_output, 'model.npz'),
model)
# reconstruct .obj
with chainer.configuration.using_config('train', False):
vertices, faces = model.reconstruct(images_in)
neural_renderer.save_obj(args.output_obj,
vertices.data.get()[0],
faces.get()[0])
# render reconstructed shape
ones = chainer.cuda.to_gpu(np.ones((16, ), 'float32'))
distances = 2.732 * ones
elevations = 30. * ones
azimuths = chainer.cuda.to_gpu(
np.arange(0, 360, 360. / 16.).astype('float32')) * ones
viewpoints = neural_renderer.get_points_from_angles(
distances, elevations, azimuths)
images_out = model.reconstruct_and_render(
chainer.functions.tile(images_in, (16, 1, 1, 1)), viewpoints)
image_out = tile_images(images_out.data.get())
image_out = (image_out * 255).clip(0, 255).astype('uint8')
skimage.io.imsave(args.output_image, image_out)
for i in range(4):
for j in range(4):
img_tmp = image_out[i * 64:(i + 1) * 64, j * 64:(j + 1) * 64, :]
skimage.io.imsave(args.output_image + '%d%d.png' % (i, j), img_tmp)
def run():
parser = argparse.ArgumentParser()
parser.add_argument('-io', '--filename_obj', type=str, default='./examples/data/teapot.obj')
parser.add_argument('-ir', '--filename_ref', type=str, default='./examples/data/example2_ref.png')
parser.add_argument('-oo', '--filename_output_optimization', type=str, default='./examples/data/example2_optimization.gif')
parser.add_argument('-or', '--filename_output_result', type=str, default='./examples/data/example2_result.gif')
parser.add_argument('-g', '--gpu', type=int, default=0)
parser.add_argument('-ni', '--num_iters', type=int, default=300)
args = parser.parse_args()
working_directory = os.path.dirname(args.filename_output_result)
if not os.path.exists(working_directory):
os.makedirs(working_directory)
model = Model(args.filename_obj, args.filename_ref)
model.to_gpu()
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
loop = tqdm.tqdm(range(args.num_iters))
for i in loop:
loop.set_description('Optimizing')
optimizer.target.cleargrads()
loss = model()
loss.backward()
optimizer.update()
images = model.renderer.render_silhouettes(model.vertices, model.faces)
#images = model.renderer.render_silhouettes(model.vertices, model.faces, model.image_size)
image = images.data.get()[0]
scipy.misc.toimage(image, cmin=0, cmax=1).save('%s/_tmp_%04d.png' % (working_directory, i))
make_gif(working_directory, args.filename_output_optimization)
# save obj file
#vertices,faces,textures = model.mesh.get_batch(args.batch_size)
## fill back
#textures_1 = chainer.functions.concat((textures, textures.transpose((0, 1, 4, 3, 2, 5))), axis=1)
faces_1 = chainer.functions.concat((model.faces, model.faces[:, :, ::-1]), axis=1).data
obj_fn = args.filename_output_result.split('/')[-1].split('.')[0]
#output_directory = os.path.split(args.filename_output_result)[0]#'/'.join(args.filename_output.split('/')[-3:-1])
#import pdb
#pdb.set_trace()
neural_renderer.save_obj('%s/%s.obj'% (working_directory,obj_fn), model.vertices[0], faces_1[0])
#neural_renderer.save_obj('%s/%s.obj'% (output_directory,obj_fn), vertices[0], faces[0], textures[0].array)
# draw object
loop = tqdm.tqdm(range(0, 360, 4))
for num, azimuth in enumerate(loop):
loop.set_description('Drawing')
model.renderer.eye = neural_renderer.get_points_from_angles(2.732, 0, azimuth)
images = model.renderer.render(model.vertices, model.faces, model.textures)
image = images.data.get()[0].transpose((1, 2, 0))
scipy.misc.toimage(image, cmin=0, cmax=1).save('%s/_tmp_%04d.png' % (working_directory, num))
make_gif(working_directory, args.filename_output_result)
def run():
# set random seed, gpu
seed = RANDOM_SEED
random.seed(seed)
np.random.seed(seed)
cp.random.seed(seed)
chainer.cuda.get_device(GPU).use()
# load dataset
image_in = skimage.io.imread('data/3d_input.png').astype('float32') / 255
if image_in.ndim != 3 or image_in.shape[-1] != 4:
raise Exception('Input must be a RGBA image.')
images_in = image_in.transpose((2, 0, 1))[None, :, :, :]
images_in = chainer.cuda.to_gpu(images_in)
# setup model & optimizer
model = models.Model()
model.to_gpu()
chainer.serializers.load_npz('reconstruction.npz', model)
# reconstruct .obj
vertices, faces = model.reconstruct(images_in)
neural_renderer.save_obj('data/target.obj',
vertices.data.get()[0],
faces.get()[0])
# render reconstructed shape
ones = chainer.cuda.to_gpu(np.ones((16, ), 'float32'))
distances = 2.732 * ones
elevations = 30. * ones
azimuths = chainer.cuda.to_gpu(
np.arange(0, 360, 360. / 16.).astype('float32')) * ones
viewpoints = neural_renderer.get_points_from_angles(
distances, elevations, azimuths)
images_out = model.reconstruct_and_render(
chainer.functions.tile(images_in, (16, 1, 1, 1)), viewpoints)
image_out = tile_images(images_out.data.get())
image_out = (image_out * 255).clip(0, 255).astype('uint8')
skimage.io.imsave('output_3d.jpg', image_out)
def reconstruct(self,
images,
viewpoints=None,
save_path=None,
fill_back=False):
vertices, faces = self.decoder(self.encoder(images))
textures = self.encoder_color(images)[0].view(-1, faces.size(1),
TEXTURE_SIZE,
TEXTURE_SIZE,
TEXTURE_SIZE, 3)
textures = textures.permute((0, 1, 4, 3, 2, 5))
if save_path is not None:
if not fill_back:
nr.save_obj(save_path, vertices[0], faces[0])
else:
faces = torch.cat(
(faces,
faces[:, :,
list(reversed(list(range(faces.shape[-1]))))]),
dim=1)
textures = torch.cat(
(textures, textures.permute((0, 1, 4, 3, 2, 5))), dim=1)
nr.save_obj(save_path, vertices[0], faces[0])
if viewpoints is None:
if fill_back:
faces = torch.cat(
(faces,
faces[:, :,
list(reversed(list(range(faces.shape[-1]))))]),
dim=1)
textures = torch.cat(
(textures, textures.permute((0, 1, 4, 3, 2, 5))), dim=1)
return vertices, faces, textures
else:
self.renderer.eye = viewpoints
textures = torch.ones(viewpoints.size(0), faces.size(1), 2, 2, 2,
3).float().cuda()
images_out = self.renderer.render(vertices, faces, textures)
return images_out, textures
def test_case1(self):
data = [[
'./tests/data/4e49873292196f02574b5684eaec43e9/model.obj',
neural_renderer.get_points_from_angles(2.5, 10, -90),
'./tests/data/4e49873292196f02574b5684eaec43e9.png',
],
[
'./tests/data/1cde62b063e14777c9152a706245d48/model.obj',
neural_renderer.get_points_from_angles(2.5, 10, 60),
'./tests/data/1cde62b063e14777c9152a706245d48.png',
]]
filename_tmp = './tests/data/tmp.obj'
renderer = neural_renderer.Renderer()
renderer.draw_backside = False
for i, (filename, viewpoint, reference) in enumerate(data):
renderer.viewpoints = viewpoint
ref = neural_renderer.imread(reference)
vertices, faces, vertices_t, faces_t, textures = neural_renderer.load_obj(
filename, load_textures=True)
neural_renderer.save_obj(filename_tmp, vertices, faces, vertices_t,
faces_t, textures)
vertices, faces, vertices_t, faces_t, textures = neural_renderer.load_obj(
filename_tmp, load_textures=True)
vertices, faces, vertices_t, faces_t, textures = neural_renderer.to_gpu(
(vertices[None, :, :], faces, vertices_t[None, :, :], faces_t,
textures[None, :, :, :]))
images = renderer.render(vertices, faces, vertices_t, faces_t,
textures).data
image = images[0].transpose((1, 2, 0))
chainer.testing.assert_allclose(ref, image, atol=1e-2, rtol=1e-2)
for f in glob.glob('./tests/data/tmp*'):
os.remove(f)
def run():
# load settings
parser = argparse.ArgumentParser()
parser.add_argument('-im', '--filename_mesh', type=str)
parser.add_argument('-is', '--filename_style', type=str)
parser.add_argument('-o', '--filename_output', type=str)
parser.add_argument('-ls', '--lambda_style', type=float, default=1.)
parser.add_argument('-lc', '--lambda_content', type=float, default=2e9)
parser.add_argument('-ltv', '--lambda_tv', type=float, default=1e7)
parser.add_argument('-emax', '--elevation_max', type=float, default=40.)
parser.add_argument('-emin', '--elevation_min', type=float, default=20.)
parser.add_argument('-lrv', '--lr_vertices', type=float, default=0.01)
parser.add_argument('-lrt', '--lr_textures', type=float, default=1.0)
parser.add_argument('-cd', '--camera_distance', type=float, default=2.732)
parser.add_argument('-cdn',
'--camera_distance_noise',
type=float,
default=0.1)
parser.add_argument('-ts', '--texture_size', type=int, default=4)
parser.add_argument('-lr', '--adam_lr', type=float, default=0.05)
parser.add_argument('-ab1', '--adam_beta1', type=float, default=0.9)
parser.add_argument('-ab2', '--adam_beta2', type=float, default=0.999)
parser.add_argument('-bs', '--batch_size', type=int, default=4)
parser.add_argument('-im_s', '--image_size', type=int, default=400)
parser.add_argument('-ni', '--num_iteration', type=int, default=1000)
parser.add_argument('-g', '--gpu', type=int, default=0)
args = parser.parse_args()
# create output directory
directory_output = os.path.dirname(args.filename_output)
if not os.path.exists(directory_output):
os.makedirs(directory_output)
# setup chainer
#os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" # see issue #152
#os.environ["CUDA_VISIBLE_DEVICES"]=str(args.gpu)
chainer.cuda.get_device_from_id(args.gpu).use()
cp.random.seed(0)
np.random.seed(0)
# setup scene
model = style_transfer_3d.StyleTransferModel(
filename_mesh=args.filename_mesh,
filename_style=args.filename_style,
lambda_style=args.lambda_style,
lambda_content=args.lambda_content,
lambda_tv=args.lambda_tv,
elevation_max=args.elevation_max,
elevation_min=args.elevation_min,
lr_vertices=args.lr_vertices,
lr_textures=args.lr_textures,
camera_distance=args.camera_distance,
camera_distance_noise=args.camera_distance_noise,
texture_size=args.texture_size,
image_size=args.image_size)
model.to_gpu()
optimizer = neural_renderer.Adam(alpha=args.adam_lr, beta1=args.adam_beta1)
optimizer.setup(model)
# optimization
#import pdb
#pdb.set_trace()
loop = tqdm.tqdm(range(args.num_iteration))
for _ in loop:
optimizer.target.cleargrads()
loss = model(args.batch_size)
loss.backward()
optimizer.update()
loop.set_description('Optimizing. Loss %.4f' % loss.data)
# save obj
##pdb.set_trace()
#model.textures_1 = chainer.functions.concat((model.mesh.textures, model.mesh.textures.transpose((0, 1, 4, 3, 2, 5))), axis=1)
##model.textures_1 = chainer.functions.concat((model.mesh.textures, model.mesh.textures), axis=1)
#obj_fn = args.filename_output.split('/')[-1].split('.')[0]
#output_directory = os.path.split(args.filename_output)[0]#'/'.join(args.filename_output.split('/')[-3:-1])
##neural_renderer.save_obj('%s/%s.obj'% (output_directory,obj_fn), model.mesh.vertices, model.mesh.faces, chainer.functions.tanh(model.textures_1).array)
#neural_renderer.save_obj('%s/%s.obj'% (output_directory,obj_fn), model.mesh.vertices[0], model.mesh.faces[0], chainer.functions.tanh(model.textures_1[0]).array)
#
vertices, faces, textures = model.mesh.get_batch(args.batch_size)
## fill back
textures_1 = chainer.functions.concat(
(textures, textures.transpose((0, 1, 4, 3, 2, 5))), axis=1)
faces_1 = chainer.functions.concat((faces, faces[:, :, ::-1]), axis=1).data
#
obj_fn = args.filename_output.split('/')[-1].split('.')[0]
output_directory = os.path.split(args.filename_output)[
0] #'/'.join(args.filename_output.split('/')[-3:-1])
#neural_renderer.save_obj('%s/%s.obj'% (output_directory,obj_fn), model.mesh.vertices.array, model.mesh.faces, model.mesh.textures.array)
neural_renderer.save_obj('%s/%s.obj' % (output_directory, obj_fn),
vertices[0], faces[0], textures[0].array)
#neural_renderer.save_obj('%s/%s.obj'% (output_directory,obj_fn), model.mesh.vertices[0], model.mesh.faces[0], chainer.functions.tanh(model.textures_1[0]).array)
# draw object
model.renderer.background_color = (1, 1, 1)
#model.renderer.background_color = (0, 0, 0)
loop = tqdm.tqdm(range(0, 360, 4))
for num, azimuth in enumerate(loop):
loop.set_description('Drawing')
model.renderer.eye = neural_renderer.get_points_from_angles(
2.732, 30, azimuth)
images = model.renderer.render(*model.mesh.get_batch(1))
image = images.data.get()[0].transpose((1, 2, 0))
scipy.misc.toimage(image, cmin=0, cmax=1).save('%s/_tmp_%04d.png' %
(directory_output, num))
make_gif(directory_output, args.filename_output)
def main():
args = parse_args()
cfg = Config.fromfile(args.config)
# set cudnn_benchmark
if cfg.get('cudnn_benchmark', False):
torch.backends.cudnn.benchmark = True
# update configs according to CLI args
if args.work_dir is not None:
cfg.work_dir = args.work_dir
if args.resume_from is not None:
cfg.resume_from = args.resume_from
cfg.gpus = args.gpus
cfg.train_cfg.rcnn.sampler.add_gt_as_proposals = False # Actually it doesn't matter.
if args.ckpt:
cfg.resume_from = args.ckpt
if args.imgs_per_gpu > 0:
cfg.data.imgs_per_gpu = args.imgs_per_gpu
if args.nms_thr:
cfg.test_cfg.rcnn.nms.iou_thr = args.nms_thr
FOCAL_LENGTH = cfg.get('FOCAL_LENGTH', 1000)
# init distributed env first, since logger depends on the dist info.
if args.launcher == 'none':
distributed = False
else:
distributed = True
init_dist(args.launcher, **cfg.dist_params)
# init logger before other steps
logger = get_root_logger(cfg.log_level)
logger.info('Distributed training: {}'.format(distributed))
# set random seeds
if args.seed is not None:
logger.info('Set random seed to {}'.format(args.seed))
set_random_seed(args.seed)
model = build_detector(cfg.model,
train_cfg=cfg.train_cfg,
test_cfg=cfg.test_cfg)
# train_dataset = get_dataset(cfg.datasets[0].train)
train_dataset = get_dataset(cfg.datasets[1].train)
if cfg.checkpoint_config is not None:
# save mmdet version, config file content and class names in
# checkpoints as meta data
cfg.checkpoint_config.meta = dict(mmdet_version=__version__,
config=cfg.text,
CLASSES=train_dataset.CLASSES)
# add an attribute for visualization convenience
model.CLASSES = train_dataset.CLASSES
model = MMDataParallel(model, device_ids=range(cfg.gpus)).cuda()
# build runner
optimizer = build_optimizer(model, cfg.optimizer)
runner = Runner(model, lambda x: x, optimizer, cfg.work_dir, cfg.log_level)
runner.resume(cfg.resume_from)
model = runner.model
# ONLY FOR DEBUG
# print('remove DDP for debug!')
# model = model._modules['module']
model.eval()
dataset_cfg = eval_dataset_mapper[args.dataset]
dataset_cfg.update(cfg.common_val_cfg)
dataset_cfg.pop('max_samples')
dataset = get_dataset(dataset_cfg)
# dataset.debugging = True
shuffle = False if args.dataset in stable_list else True
data_loader = build_dataloader_fuse(
dataset,
1,
0,
cfg.gpus,
dist=False,
shuffle=shuffle,
drop_last=False,
)
dump_dir = os.path.join(cfg.work_dir, f'eval_{args.dataset}')
os.makedirs(dump_dir, exist_ok=True)
if args.viz_dir:
os.makedirs(args.viz_dir, exist_ok=True)
eval_handler = eval_handler_mapper[args.dataset](
writer=tqdm.write,
viz_dir=args.viz_dir,
FOCAL_LENGTH=FOCAL_LENGTH,
work_dir=cfg.work_dir) # type: EvalHandler
with torch.no_grad():
for i, data_batch in enumerate(tqdm(data_loader)):
file_name = data_batch['img_meta'].data[0][0]['file_name']
try:
bbox_results, pred_results = model(**data_batch,
return_loss=False,
use_gt_bboxes=args.use_gt)
pred_results['bboxes'] = bbox_results
if args.paper_dir:
os.makedirs(args.paper_dir, exist_ok=True)
img = denormalize(data_batch['img'].data[0][0].numpy())
verts = pred_results['pred_vertices'] + pred_results[
'pred_translation']
dump_folder = osp.join(args.paper_dir, file_name)
os.makedirs(dump_folder, exist_ok=True)
plt.imsave(osp.join(dump_folder, 'img.png'), img)
for obj_i, vert in enumerate(verts):
nr.save_obj(osp.join(dump_folder, f'{obj_i}.obj'),
vert,
torch.tensor(smpl.faces.astype(np.int64)))
save_pack = eval_handler(data_batch,
pred_results,
use_gt=args.use_gt)
save_pack.update({'bbox_results': pred_results['bboxes']})
if args.dump_pkl:
with open(
osp.join(dump_dir,
f"{save_pack['file_name']}.pkl"),
'wb') as f:
pickle.dump(save_pack, f)
except Exception as e:
tqdm.write(f"Fail on {file_name}")
tqdm.write(str(e))
eval_handler.finalize()
def fgsm(rank, args, shared_model, number):
torch.cuda.set_device(args.gpus.index(args.gpus[rank % len(args.gpus)]))
all_n = 0
# torch.cuda.set_device(args.gpus.index(args.gpus[rank % len(args.gpus)]))
model_kwargs = {'vocab': load_vocab(args.vocab_json)}
model = VqaLstmCnnAttentionModel(**model_kwargs)
device_ids = [0,1]
model = model.cuda(device_ids[0])
model = torch.nn.DataParallel(model, device_ids=device_ids)
# torch.backends.cudnn.benchmark = True
lossFn = torch.nn.CrossEntropyLoss().cuda()
eval_loader_kwargs = {
'questions_h5': getattr(args, args.eval_split + '_h5'),
'data_json': args.data_json,
'vocab': args.vocab_json,
'target_obj_conn_map_dir': args.target_obj_conn_map_dir,
'batch_size': args.batch_size,
'input_type': args.input_type,
'num_frames': args.num_frames,
'split': args.eval_split,
'max_threads_per_gpu': args.max_threads_per_gpu,
'gpu_id': args.gpus[rank%len(args.gpus)],
'to_cache': args.cache
}
eval_loader = EqaDataLoader(**eval_loader_kwargs)
# for ijcai in range(number):
# eval_loader.dataset._load_envs()
eval_loader.dataset._load_envs(start_idx=number)
print('eval_loader has %d samples' % len(eval_loader.dataset))
args.output_log_path = os.path.join(args.log_dir,
'eval_' + str(rank) + '.json')
model.load_state_dict(handle_load(shared_model.state_dict()))
model.eval()
metrics = VqaMetric(
info={'split': args.eval_split},
metric_names=[
'loss', 'accuracy', 'mean_rank', 'mean_reciprocal_rank'
],
log_json=args.output_log_path)
all_envs_loaded = eval_loader.dataset._check_if_all_envs_loaded()
done = False
print(number, ' begin')
import copy
import torch.nn as nn
from PIL import Image
softmax = nn.Softmax()
allcor = []
while done == False:
for ii,batch in enumerate(eval_loader):
# model.cuda()
if(ii>=0):
idx, questions, answers, house, v, f, vt, pos, _, _, _ = batch
print('all size:',v.size(),f.size(),vt.size())
#print(house)
print(questions, answers)
questions_var = Variable(questions.cuda())
print(questions_var.size())
answers_var = Variable(answers.cuda())
v_var = Variable(v.cuda(),requires_grad=True)
f_var = Variable(f.cuda())
vt_var = Variable(vt.cuda(),requires_grad=True)
begin, end = get_info(idx[0], house[0])
#print(vt_var[0][0][40010][0][0])
#print(vt_var[0][0][39506][0][0])
vt_test = copy.deepcopy(vt_var)
epsilon = 32.0/256.0
vt_grad = torch.zeros(vt_var.size()).cuda()
#vt_var.retain_grad()
scores, att_probs,img_clean = model(v_var, f_var, vt_var, pos, questions_var)
i1, i2 = torch.max(scores[0],0)
mi = i2.cpu().numpy().tolist()
ms = int(answers)
print(mi)
print(mi==int(ms))
if(mi==int(ms)):
allcor.append(1.0)
else:
allcor.append(0.0)
print(softmax(scores[0]))
print(softmax(scores[0])[ms])
img_clean = img_clean[0]
for iii in range(img_clean.size()[0]):
imggg = img_clean[iii].detach().cpu().numpy()
imggg = imggg * 255.0
imggg = imggg.transpose((1,2,0))
imggg = Image.fromarray(imggg.astype('uint8'))
imggg.save('result_test/'+str(ii)+'_'+str(iii)+'_clean.jpg')
loss = lossFn(scores, answers_var)
loss.backward()
#print(torch.max(vt_grad))
vt_grad = vt_var.grad
v_grad = v_var.grad
print('max grad',torch.max(vt_grad.data))
#print(vt_grad[0][0][40010][0][0])
#print(vt_grad[0][0][39506][0][0])
vt_var = vt_var.detach() + epsilon * torch.sign(vt_grad)
#v_var = v_var.detach() + 1.0 * torch.sign(v_grad)
vt_var = torch.clamp(vt_var, 0, 1)
#vt_var = Variable(vt_var.data, requires_grad=True).cuda()
with torch.no_grad():
model.eval()
begin, end = get_info(idx[0], house[0])
#for iii in range(begin,end):
#if(vt_test[0][0][iii][0][0][0][0] != vt_var[0][0][iii][0][0][0][0] or vt_test[0][0][iii][0][0][1][0] != vt_var[0][0][iii][0][0][1][0] or vt_test[0][0][iii][0][1][1][0] != vt_var[0][0][iii][0][1][1][0]):
#print(iii)
vt_test[0][0][begin:end] = vt_var[0][0][begin:end]
#print(vt_test[0][0][40010][0][0])
#print(vt_test[0][0][39506][0][0])
#print((vt_test[0][0] == vt_var[0][0]).sum())
#print((vt_test[0][0].size()),(vt_var[0][0].size()))
scores, att_probs,imgg = model(v_var, f_var, vt_test, pos, questions_var)
imgg = imgg[0]
#print(imgg.size())
for iii in range(imgg.size()[0]):
imggg = imgg[iii].detach().cpu().numpy()
imggg = imggg * 255.0
imggg = imggg.transpose((1,2,0))
imggg = Image.fromarray(imggg.astype('uint8'))
imggg.save('result_test/'+str(ii)+'_'+str(iii)+'_adv.jpg')
i1, i2 = torch.max(scores[0],0)
mi = i2.cpu().numpy().tolist()
ms = int(answers)
print(mi)
print(mi==int(ms))
print(softmax(scores[0]))
print(softmax(scores[0])[ms])
for k in range(idx.shape[0]):
begin, end = get_info(idx[k], house[k])
#print(begin, end)
#begin, end = find_index('167', house[k])
v_m = v_var[k][0]
f_m = f_var[k][0][begin:end]
vt_m = vt_var[k][0][begin:end]
nr.save_obj('/media/trs1/dataset/suncg_data/house/' + house[k] + '/attack_' + str(int(idx[k])) + '.obj', v_m, f_m, vt_m)
idx = idx.cpu()
questions = questions.cpu()
answers = answers.cpu()
questions_var = questions_var.cpu()
answers_var = answers_var.cpu()
v = v.cpu()
f = f.cpu()
vt = vt.cpu()
v_m = v_m.cpu()
f_m = f_m.cpu()
vt_m = vt_m.cpu()
v_var = v_var.detach().cpu()
f_var = f_var.cpu()
vt_var = vt_var.detach().cpu()
vt_grad = vt_grad.cpu()
print(house[k] + ' ' + str(int(idx[k])) + ' ok')
all_n += 1
# handle_file(path)
if all_envs_loaded == False:
eval_loader.dataset._load_envs()
if len(eval_loader.dataset.pruned_env_set) == 0:
done = True
else:
done = True
print(allcor)
print(number, ' over')
def run():
# settings
parser = argparse.ArgumentParser()
parser.add_argument('-i', '--filename_obj', type=str)
parser.add_argument('-o', '--filename_output', type=str)
parser.add_argument('-d', '--output_directory', type=str)
parser.add_argument('-al', '--adam_lr', type=float, default=0.01)
parser.add_argument('-ab1', '--adam_beta1', type=float, default=0.9)
parser.add_argument('-bs', '--batch_size', type=int, default=4)
parser.add_argument('-ni', '--num_iteration', type=int, default=1000)
parser.add_argument('-cd', '--camera_distance', type=float, default=2.5)
parser.add_argument('-ib', '--init_bias', type=str, default='(0,0,0)')
parser.add_argument('-g', '--gpu', type=int, default=0)
args = parser.parse_args()
args.init_bias = tuple([float(v) for v in args.init_bias[1:-1].split(',')])
# create output directory
if not os.path.exists(args.output_directory):
os.makedirs(args.output_directory)
# setup chainer
chainer.cuda.get_device_from_id(args.gpu).use()
cp.random.seed(0)
np.random.seed(0)
# setup scene & optimizer
model = deep_dream_3d.DeepDreamModel(
args.filename_obj,
camera_distance=args.camera_distance,
init_bias=args.init_bias)
model.to_gpu()
optimizer = neural_renderer.Adam(alpha=args.adam_lr, beta1=args.adam_beta1)
optimizer.setup(model)
# optimization
loop = tqdm.tqdm(range(args.num_iteration))
for _ in loop:
optimizer.target.cleargrads()
loss = model(args.batch_size)
loss.backward()
optimizer.update()
loop.set_description('Optimizing. Loss %.4f' % loss.data)
## save obj
#pdb.set_trace()
vertices,faces,textures = model.mesh.get_batch(args.batch_size)
## fill back
textures_1 = chainer.functions.concat((textures, textures.transpose((0, 1, 4, 3, 2, 5))), axis=1)
faces_1 = chainer.functions.concat((faces, faces[:, :, ::-1]), axis=1).data
#
obj_fn = args.filename_output.split('/')[-1].split('.')[0]
output_directory = os.path.split(args.filename_output)[0]#'/'.join(args.filename_output.split('/')[-3:-1])
#neural_renderer.save_obj('%s/%s.obj'% (output_directory,obj_fn), model.mesh.vertices.array, model.mesh.faces, model.mesh.textures.array)
neural_renderer.save_obj('%s/%s.obj'% (output_directory,obj_fn), vertices[0], faces[0], textures[0].array)
#neural_renderer.save_obj('%s/%s.obj'% (output_directory,obj_fn), model.mesh.vertices[0], model.mesh.faces[0], chainer.functions.tanh(model.textures_1[0]).array)
# draw object
model.renderer.background_color = (1, 1, 1)
loop = tqdm.tqdm(range(0, 360, 4))
for num, azimuth in enumerate(loop):
loop.set_description('Drawing')
model.renderer.eye = neural_renderer.get_points_from_angles(2.732, 30, azimuth)
images = model.renderer.render(*model.mesh.get_batch(1))
image = images.data.get()[0].transpose((1, 2, 0))
scipy.misc.toimage(image, cmin=0, cmax=1).save('%s/_tmp_%04d.png' % (args.output_directory, num))
make_gif(args.output_directory, args.filename_output)
def attack_fgsm(rank, args, shared_model, number): #?
torch.cuda.set_device(args.gpus.index(args.gpus[rank % len(args.gpus)]))
all_n = 0
# torch.cuda.set_device(args.gpus.index(args.gpus[rank % len(args.gpus)]))
model_kwargs = {'vocab': load_vocab(args.vocab_json)}
model = VqaLstmCnnAttentionModel(**model_kwargs)
device_ids = [0, 1]
model = model.cuda(device_ids[0])
model = torch.nn.DataParallel(model, device_ids=device_ids)
# torch.backends.cudnn.benchmark = True
lossFn = torch.nn.CrossEntropyLoss().cuda()
eval_loader_kwargs = {
'questions_h5': getattr(args, args.eval_split + '_h5'),
'data_json': args.data_json,
'vocab': args.vocab_json,
'batch_size': args.batch_size,
'input_type': args.input_type,
'num_frames': args.num_frames,
'split': args.eval_split,
'max_threads_per_gpu': args.max_threads_per_gpu,
'gpu_id': args.gpus[rank % len(args.gpus)],
'to_cache': args.cache
}
eval_loader = EqaDataLoader(**eval_loader_kwargs)
# for ijcai in range(number):
# eval_loader.dataset._load_envs()
eval_loader.dataset._load_envs(start_idx=number)
print('eval_loader has %d samples' % len(eval_loader.dataset))
args.output_log_path = os.path.join(args.log_dir,
'eval_' + str(rank) + '.json')
model.load_state_dict(handle_load(shared_model.state_dict()))
model.eval()
metrics = VqaMetric(
info={'split': args.eval_split},
metric_names=['loss', 'accuracy', 'mean_rank', 'mean_reciprocal_rank'],
log_json=args.output_log_path)
all_envs_loaded = eval_loader.dataset._check_if_all_envs_loaded()
done = False
while done == False:
for batch in eval_loader:
idx, questions, answers, house, v, f, vt, pos, _, _, _ = batch
questions_var = Variable(questions.cuda())
answers_var = Variable(answers.cuda())
v_var = Variable(v.cuda(), requires_grad=True)
f_var = Variable(f.cuda())
vt_var = Variable(vt.cuda(), requires_grad=True)
# noise level
epsilon = 12.0 / 255.0
scores, att_probs = model(v_var, f_var, vt_var, pos, questions_var,
0, '0')
loss = lossFn(scores, answers_var)
loss.backward()
# get grad for attack
vt_grad = vt_var.grad
vt_detach = vt_var.detach()
begin, end, oid = get_info(idx[0], house[0])
if (begin == 1000 and end == 2000):
print(str(int(idx[0])), 'error')
vt_grad[0][0][:begin] = 0
vt_grad[0][0][end:] = 0
vt_var = vt_detach + epsilon * torch.sign(vt_grad)
for k in range(idx.shape[0]):
begin, end, oid = get_info(idx[k], house[k])
v_m = v_var[k][0]
f_m = f_var[k][0][begin:end]
vt_m = vt_var[k][0][begin:end]
# save changed object to .obj file
nr.save_obj(
'/path/to/data/house/' + house[k] + '/attack_' +
str(int(idx[k])) + '_' + str(oid) + '.obj', v_m, f_m, vt_m)
with torch.no_grad():
model.eval()
scores, att_probs = model(v_var, f_var, vt_var, pos,
questions_var, 0, '0')
accuracy, ranks = metrics.compute_ranks(
scores.data.cpu(), answers)
if all_envs_loaded == False:
eval_loader.dataset._load_envs()
if len(eval_loader.dataset.pruned_env_set) == 0:
done = True
else:
done = True
def attack_pgd(rank, args, shared_model, number):
torch.cuda.set_device(args.gpus.index(args.gpus[rank % len(args.gpus)]))
all_n = 0
model_kwargs = {'vocab': load_vocab(args.vocab_json)}
model = VqaLstmCnnAttentionModel(**model_kwargs)
device_ids = [0, 1]
model = model.cuda(device_ids[0])
model = torch.nn.DataParallel(model, device_ids=device_ids)
# torch.backends.cudnn.benchmark = True
lossFn = torch.nn.CrossEntropyLoss().cuda()
eval_loader_kwargs = {
'questions_h5': getattr(args, args.eval_split + '_h5'),
'data_json': args.data_json,
'vocab': args.vocab_json,
'batch_size': args.batch_size,
'input_type': args.input_type,
'num_frames': args.num_frames,
'split': args.eval_split,
'max_threads_per_gpu': args.max_threads_per_gpu,
'gpu_id': args.gpus[rank % len(args.gpus)],
'to_cache': args.cache
}
eval_loader = EqaDataLoader(**eval_loader_kwargs)
eval_loader.dataset._load_envs(start_idx=number)
print('eval_loader has %d samples' % len(eval_loader.dataset))
args.output_log_path = os.path.join(args.log_dir,
'eval_' + str(rank) + '.json')
model.load_state_dict(handle_load(shared_model.state_dict()))
model.eval()
metrics = VqaMetric(
info={'split': args.eval_split},
metric_names=['loss', 'accuracy', 'mean_rank', 'mean_reciprocal_rank'],
log_json=args.output_log_path)
all_envs_loaded = eval_loader.dataset._check_if_all_envs_loaded()
done = False
while done == False:
for batch in eval_loader:
# model.cuda()
idx, questions, answers, house, v, f, vt, pos, _, _, _ = batch
questions_var = Variable(questions.cuda())
answers_var = Variable(answers.cuda())
v_var = Variable(v.cuda(), requires_grad=True)
f_var = Variable(f.cuda())
vt_var = Variable(vt.cuda(), requires_grad=True)
attack_iter = 10
attack_momentum = 1
alpha = 2.0 / 255
epsilon = 16.0 / 255
vt_grad = torch.zeros(vt_var.size()).cuda()
for j in range(attack_iter):
scores, att_probs = model(v_var, f_var, vt_var, pos,
questions_var, j, str(int(idx[0])))
loss = lossFn(scores, answers_var)
loss.backward()
vg = vt_var.grad
begin, end, _ = get_info(idx[0], house[0])
if (begin == 1000 and end == 2000):
print(str(int(idx[0])), 'error')
vg[0][0][:begin] = 0
vg[0][0][end:] = 0
noise = attack_momentum * vt_grad + vg
vt_grad = noise
vt_var = vt_var.detach() + alpha * torch.sign(noise)
vt_var = torch.where(vt_var > vt + epsilon, vt + epsilon,
vt_var)
vt_var = torch.clamp(vt_var, 0, 1)
vt_var = torch.where(vt_var < vt - epsilon, vt - epsilon,
vt_var)
vt_var = torch.clamp(vt_var, 0, 1)
vt_var = Variable(vt_var.data, requires_grad=True).cuda()
with torch.no_grad():
model.eval()
scores, att_probs = model(v_var, f_var, vt_var, pos,
questions_var, 100, str(int(idx[0])))
accuracy, ranks = metrics.compute_ranks(
scores.data.cpu(), answers)
begin, end, oid = get_info(idx[0], house[0])
v_m = v_var[0][0]
f_m = f_var[0][0][begin:end]
vt_m = vt_var[0][0][begin:end]
nr.save_obj(
'/path/to/data/house/' + house[0] + '/attack_' +
str(int(idx[0])) + '_' + str(oid) + '.obj', v_m, f_m, vt_m)
if all_envs_loaded == False:
eval_loader.dataset._load_envs()
if len(eval_loader.dataset.pruned_env_set) == 0:
done = True
else:
done = True
ploter.plot('Adv_loss', 'train', 'Adv-loss', batches_done,
Adv_loss.item())
if batches_done % opt.sample_step == 0 or batches_done == last_iter:
save_image(gen_imgs.data[:min(25, gen_imgs.shape[0])],
os.path.join(opt.sample_dir,
'%05d-gen.png' % batches_done),
nrow=5,
normalize=True)
save_image(gt_imgs.data[:min(25, gt_imgs.shape[0])],
os.path.join(opt.sample_dir,
'%05d-gt.png' % batches_done),
nrow=5,
normalize=True)
nr.save_obj(
os.path.join(opt.sample_dir, '%05d.obj' % batches_done),
vertices[0, :, :], faces[0, :, :])
print('Saved sample image to {}...'.format(opt.sample_dir))
# validation on val dataset
if batches_done % opt.ckpt_step == 0 or batches_done == last_iter:
iou_val = eval_IoU(encoder, mesh_generator, dataset_val)
if opt.dataset == 'CVPR18':
MMD_val = eval_MMD(encoder, dataset_val)
f_log.write('batches_done: %d, validation iou: %f\r\n' %
(batches_done, iou_val))
ploter.plot('IoU_loss', 'voxel_IoU_val', 'IoU-loss',
batches_done, iou_val)
if opt.dataset == 'CVPR18':
ploter.plot('IoU_loss', 'MMD_val', 'IoU-loss',
batches_done, MMD_val)
