def test_save_obj(self): vertices, faces = neural_renderer.load_obj('./tests/data/teapot.obj') neural_renderer.save_obj('./tests/data/teapot2.obj', vertices, faces) vertices2, faces2 = neural_renderer.load_obj('./tests/data/teapot.obj') os.remove('./tests/data/teapot2.obj') assert np.allclose(vertices, vertices2) assert np.allclose(faces, faces2)
def test_save_obj(self): teapot = os.path.join(data_dir, 'teapot.obj') teapot2 = os.path.join(data_dir, 'teapot2.obj') vertices, faces = nr.load_obj(teapot) nr.save_obj(teapot2, vertices, faces) vertices2, faces2 = nr.load_obj(teapot2) os.remove(teapot2) assert torch.allclose(vertices, vertices2) assert torch.allclose(faces, faces2)
def run(): # arguments parser = argparse.ArgumentParser() parser.add_argument('-eid', '--experiment_id', type=str) parser.add_argument('-d', '--directory', type=str, default=DIRECTORY) parser.add_argument('-i', '--input_image', type=str) parser.add_argument('-oi', '--output_image', type=str) parser.add_argument('-oo', '--output_obj', type=str) parser.add_argument('-s', '--seed', type=int, default=RANDOM_SEED) parser.add_argument('-g', '--gpu', type=int, default=GPU) args = parser.parse_args() directory_output = os.path.join(args.directory, args.experiment_id) # set random seed, gpu random.seed(args.seed) np.random.seed(args.seed) cp.random.seed(args.seed) chainer.cuda.get_device(args.gpu).use() # load dataset image_in = skimage.io.imread(args.input_image).astype('float32') / 255 if image_in.ndim != 3 or image_in.shape[-1] != 4: raise Exception('Input must be a RGBA image.') images_in = image_in.transpose((2, 0, 1))[None, :, :, :] images_in = chainer.cuda.to_gpu(images_in) # setup model & optimizer model = model_nview.Model(img_size=64) model.to_gpu() chainer.serializers.load_npz(os.path.join(directory_output, 'model.npz'), model) # reconstruct .obj with chainer.configuration.using_config('train', False): vertices, faces = model.reconstruct(images_in) neural_renderer.save_obj(args.output_obj, vertices.data.get()[0], faces.get()[0]) # render reconstructed shape ones = chainer.cuda.to_gpu(np.ones((16, ), 'float32')) distances = 2.732 * ones elevations = 30. * ones azimuths = chainer.cuda.to_gpu( np.arange(0, 360, 360. / 16.).astype('float32')) * ones viewpoints = neural_renderer.get_points_from_angles( distances, elevations, azimuths) images_out = model.reconstruct_and_render( chainer.functions.tile(images_in, (16, 1, 1, 1)), viewpoints) image_out = tile_images(images_out.data.get()) image_out = (image_out * 255).clip(0, 255).astype('uint8') skimage.io.imsave(args.output_image, image_out) for i in range(4): for j in range(4): img_tmp = image_out[i * 64:(i + 1) * 64, j * 64:(j + 1) * 64, :] skimage.io.imsave(args.output_image + '%d%d.png' % (i, j), img_tmp)
def run(): parser = argparse.ArgumentParser() parser.add_argument('-io', '--filename_obj', type=str, default='./examples/data/teapot.obj') parser.add_argument('-ir', '--filename_ref', type=str, default='./examples/data/example2_ref.png') parser.add_argument('-oo', '--filename_output_optimization', type=str, default='./examples/data/example2_optimization.gif') parser.add_argument('-or', '--filename_output_result', type=str, default='./examples/data/example2_result.gif') parser.add_argument('-g', '--gpu', type=int, default=0) parser.add_argument('-ni', '--num_iters', type=int, default=300) args = parser.parse_args() working_directory = os.path.dirname(args.filename_output_result) if not os.path.exists(working_directory): os.makedirs(working_directory) model = Model(args.filename_obj, args.filename_ref) model.to_gpu() optimizer = chainer.optimizers.Adam() optimizer.setup(model) loop = tqdm.tqdm(range(args.num_iters)) for i in loop: loop.set_description('Optimizing') optimizer.target.cleargrads() loss = model() loss.backward() optimizer.update() images = model.renderer.render_silhouettes(model.vertices, model.faces) #images = model.renderer.render_silhouettes(model.vertices, model.faces, model.image_size) image = images.data.get()[0] scipy.misc.toimage(image, cmin=0, cmax=1).save('%s/_tmp_%04d.png' % (working_directory, i)) make_gif(working_directory, args.filename_output_optimization) # save obj file #vertices,faces,textures = model.mesh.get_batch(args.batch_size) ## fill back #textures_1 = chainer.functions.concat((textures, textures.transpose((0, 1, 4, 3, 2, 5))), axis=1) faces_1 = chainer.functions.concat((model.faces, model.faces[:, :, ::-1]), axis=1).data obj_fn = args.filename_output_result.split('/')[-1].split('.')[0] #output_directory = os.path.split(args.filename_output_result)[0]#'/'.join(args.filename_output.split('/')[-3:-1]) #import pdb #pdb.set_trace() neural_renderer.save_obj('%s/%s.obj'% (working_directory,obj_fn), model.vertices[0], faces_1[0]) #neural_renderer.save_obj('%s/%s.obj'% (output_directory,obj_fn), vertices[0], faces[0], textures[0].array) # draw object loop = tqdm.tqdm(range(0, 360, 4)) for num, azimuth in enumerate(loop): loop.set_description('Drawing') model.renderer.eye = neural_renderer.get_points_from_angles(2.732, 0, azimuth) images = model.renderer.render(model.vertices, model.faces, model.textures) image = images.data.get()[0].transpose((1, 2, 0)) scipy.misc.toimage(image, cmin=0, cmax=1).save('%s/_tmp_%04d.png' % (working_directory, num)) make_gif(working_directory, args.filename_output_result)
def run(): # set random seed, gpu seed = RANDOM_SEED random.seed(seed) np.random.seed(seed) cp.random.seed(seed) chainer.cuda.get_device(GPU).use() # load dataset image_in = skimage.io.imread('data/3d_input.png').astype('float32') / 255 if image_in.ndim != 3 or image_in.shape[-1] != 4: raise Exception('Input must be a RGBA image.') images_in = image_in.transpose((2, 0, 1))[None, :, :, :] images_in = chainer.cuda.to_gpu(images_in) # setup model & optimizer model = models.Model() model.to_gpu() chainer.serializers.load_npz('reconstruction.npz', model) # reconstruct .obj vertices, faces = model.reconstruct(images_in) neural_renderer.save_obj('data/target.obj', vertices.data.get()[0], faces.get()[0]) # render reconstructed shape ones = chainer.cuda.to_gpu(np.ones((16, ), 'float32')) distances = 2.732 * ones elevations = 30. * ones azimuths = chainer.cuda.to_gpu( np.arange(0, 360, 360. / 16.).astype('float32')) * ones viewpoints = neural_renderer.get_points_from_angles( distances, elevations, azimuths) images_out = model.reconstruct_and_render( chainer.functions.tile(images_in, (16, 1, 1, 1)), viewpoints) image_out = tile_images(images_out.data.get()) image_out = (image_out * 255).clip(0, 255).astype('uint8') skimage.io.imsave('output_3d.jpg', image_out)
def reconstruct(self, images, viewpoints=None, save_path=None, fill_back=False): vertices, faces = self.decoder(self.encoder(images)) textures = self.encoder_color(images)[0].view(-1, faces.size(1), TEXTURE_SIZE, TEXTURE_SIZE, TEXTURE_SIZE, 3) textures = textures.permute((0, 1, 4, 3, 2, 5)) if save_path is not None: if not fill_back: nr.save_obj(save_path, vertices[0], faces[0]) else: faces = torch.cat( (faces, faces[:, :, list(reversed(list(range(faces.shape[-1]))))]), dim=1) textures = torch.cat( (textures, textures.permute((0, 1, 4, 3, 2, 5))), dim=1) nr.save_obj(save_path, vertices[0], faces[0]) if viewpoints is None: if fill_back: faces = torch.cat( (faces, faces[:, :, list(reversed(list(range(faces.shape[-1]))))]), dim=1) textures = torch.cat( (textures, textures.permute((0, 1, 4, 3, 2, 5))), dim=1) return vertices, faces, textures else: self.renderer.eye = viewpoints textures = torch.ones(viewpoints.size(0), faces.size(1), 2, 2, 2, 3).float().cuda() images_out = self.renderer.render(vertices, faces, textures) return images_out, textures
def test_case1(self): data = [[ './tests/data/4e49873292196f02574b5684eaec43e9/model.obj', neural_renderer.get_points_from_angles(2.5, 10, -90), './tests/data/4e49873292196f02574b5684eaec43e9.png', ], [ './tests/data/1cde62b063e14777c9152a706245d48/model.obj', neural_renderer.get_points_from_angles(2.5, 10, 60), './tests/data/1cde62b063e14777c9152a706245d48.png', ]] filename_tmp = './tests/data/tmp.obj' renderer = neural_renderer.Renderer() renderer.draw_backside = False for i, (filename, viewpoint, reference) in enumerate(data): renderer.viewpoints = viewpoint ref = neural_renderer.imread(reference) vertices, faces, vertices_t, faces_t, textures = neural_renderer.load_obj( filename, load_textures=True) neural_renderer.save_obj(filename_tmp, vertices, faces, vertices_t, faces_t, textures) vertices, faces, vertices_t, faces_t, textures = neural_renderer.load_obj( filename_tmp, load_textures=True) vertices, faces, vertices_t, faces_t, textures = neural_renderer.to_gpu( (vertices[None, :, :], faces, vertices_t[None, :, :], faces_t, textures[None, :, :, :])) images = renderer.render(vertices, faces, vertices_t, faces_t, textures).data image = images[0].transpose((1, 2, 0)) chainer.testing.assert_allclose(ref, image, atol=1e-2, rtol=1e-2) for f in glob.glob('./tests/data/tmp*'): os.remove(f)
def run(): # load settings parser = argparse.ArgumentParser() parser.add_argument('-im', '--filename_mesh', type=str) parser.add_argument('-is', '--filename_style', type=str) parser.add_argument('-o', '--filename_output', type=str) parser.add_argument('-ls', '--lambda_style', type=float, default=1.) parser.add_argument('-lc', '--lambda_content', type=float, default=2e9) parser.add_argument('-ltv', '--lambda_tv', type=float, default=1e7) parser.add_argument('-emax', '--elevation_max', type=float, default=40.) parser.add_argument('-emin', '--elevation_min', type=float, default=20.) parser.add_argument('-lrv', '--lr_vertices', type=float, default=0.01) parser.add_argument('-lrt', '--lr_textures', type=float, default=1.0) parser.add_argument('-cd', '--camera_distance', type=float, default=2.732) parser.add_argument('-cdn', '--camera_distance_noise', type=float, default=0.1) parser.add_argument('-ts', '--texture_size', type=int, default=4) parser.add_argument('-lr', '--adam_lr', type=float, default=0.05) parser.add_argument('-ab1', '--adam_beta1', type=float, default=0.9) parser.add_argument('-ab2', '--adam_beta2', type=float, default=0.999) parser.add_argument('-bs', '--batch_size', type=int, default=4) parser.add_argument('-im_s', '--image_size', type=int, default=400) parser.add_argument('-ni', '--num_iteration', type=int, default=1000) parser.add_argument('-g', '--gpu', type=int, default=0) args = parser.parse_args() # create output directory directory_output = os.path.dirname(args.filename_output) if not os.path.exists(directory_output): os.makedirs(directory_output) # setup chainer #os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" # see issue #152 #os.environ["CUDA_VISIBLE_DEVICES"]=str(args.gpu) chainer.cuda.get_device_from_id(args.gpu).use() cp.random.seed(0) np.random.seed(0) # setup scene model = style_transfer_3d.StyleTransferModel( filename_mesh=args.filename_mesh, filename_style=args.filename_style, lambda_style=args.lambda_style, lambda_content=args.lambda_content, lambda_tv=args.lambda_tv, elevation_max=args.elevation_max, elevation_min=args.elevation_min, lr_vertices=args.lr_vertices, lr_textures=args.lr_textures, camera_distance=args.camera_distance, camera_distance_noise=args.camera_distance_noise, texture_size=args.texture_size, image_size=args.image_size) model.to_gpu() optimizer = neural_renderer.Adam(alpha=args.adam_lr, beta1=args.adam_beta1) optimizer.setup(model) # optimization #import pdb #pdb.set_trace() loop = tqdm.tqdm(range(args.num_iteration)) for _ in loop: optimizer.target.cleargrads() loss = model(args.batch_size) loss.backward() optimizer.update() loop.set_description('Optimizing. Loss %.4f' % loss.data) # save obj ##pdb.set_trace() #model.textures_1 = chainer.functions.concat((model.mesh.textures, model.mesh.textures.transpose((0, 1, 4, 3, 2, 5))), axis=1) ##model.textures_1 = chainer.functions.concat((model.mesh.textures, model.mesh.textures), axis=1) #obj_fn = args.filename_output.split('/')[-1].split('.')[0] #output_directory = os.path.split(args.filename_output)[0]#'/'.join(args.filename_output.split('/')[-3:-1]) ##neural_renderer.save_obj('%s/%s.obj'% (output_directory,obj_fn), model.mesh.vertices, model.mesh.faces, chainer.functions.tanh(model.textures_1).array) #neural_renderer.save_obj('%s/%s.obj'% (output_directory,obj_fn), model.mesh.vertices[0], model.mesh.faces[0], chainer.functions.tanh(model.textures_1[0]).array) # vertices, faces, textures = model.mesh.get_batch(args.batch_size) ## fill back textures_1 = chainer.functions.concat( (textures, textures.transpose((0, 1, 4, 3, 2, 5))), axis=1) faces_1 = chainer.functions.concat((faces, faces[:, :, ::-1]), axis=1).data # obj_fn = args.filename_output.split('/')[-1].split('.')[0] output_directory = os.path.split(args.filename_output)[ 0] #'/'.join(args.filename_output.split('/')[-3:-1]) #neural_renderer.save_obj('%s/%s.obj'% (output_directory,obj_fn), model.mesh.vertices.array, model.mesh.faces, model.mesh.textures.array) neural_renderer.save_obj('%s/%s.obj' % (output_directory, obj_fn), vertices[0], faces[0], textures[0].array) #neural_renderer.save_obj('%s/%s.obj'% (output_directory,obj_fn), model.mesh.vertices[0], model.mesh.faces[0], chainer.functions.tanh(model.textures_1[0]).array) # draw object model.renderer.background_color = (1, 1, 1) #model.renderer.background_color = (0, 0, 0) loop = tqdm.tqdm(range(0, 360, 4)) for num, azimuth in enumerate(loop): loop.set_description('Drawing') model.renderer.eye = neural_renderer.get_points_from_angles( 2.732, 30, azimuth) images = model.renderer.render(*model.mesh.get_batch(1)) image = images.data.get()[0].transpose((1, 2, 0)) scipy.misc.toimage(image, cmin=0, cmax=1).save('%s/_tmp_%04d.png' % (directory_output, num)) make_gif(directory_output, args.filename_output)
def main(): args = parse_args() cfg = Config.fromfile(args.config) # set cudnn_benchmark if cfg.get('cudnn_benchmark', False): torch.backends.cudnn.benchmark = True # update configs according to CLI args if args.work_dir is not None: cfg.work_dir = args.work_dir if args.resume_from is not None: cfg.resume_from = args.resume_from cfg.gpus = args.gpus cfg.train_cfg.rcnn.sampler.add_gt_as_proposals = False # Actually it doesn't matter. if args.ckpt: cfg.resume_from = args.ckpt if args.imgs_per_gpu > 0: cfg.data.imgs_per_gpu = args.imgs_per_gpu if args.nms_thr: cfg.test_cfg.rcnn.nms.iou_thr = args.nms_thr FOCAL_LENGTH = cfg.get('FOCAL_LENGTH', 1000) # init distributed env first, since logger depends on the dist info. if args.launcher == 'none': distributed = False else: distributed = True init_dist(args.launcher, **cfg.dist_params) # init logger before other steps logger = get_root_logger(cfg.log_level) logger.info('Distributed training: {}'.format(distributed)) # set random seeds if args.seed is not None: logger.info('Set random seed to {}'.format(args.seed)) set_random_seed(args.seed) model = build_detector(cfg.model, train_cfg=cfg.train_cfg, test_cfg=cfg.test_cfg) # train_dataset = get_dataset(cfg.datasets[0].train) train_dataset = get_dataset(cfg.datasets[1].train) if cfg.checkpoint_config is not None: # save mmdet version, config file content and class names in # checkpoints as meta data cfg.checkpoint_config.meta = dict(mmdet_version=__version__, config=cfg.text, CLASSES=train_dataset.CLASSES) # add an attribute for visualization convenience model.CLASSES = train_dataset.CLASSES model = MMDataParallel(model, device_ids=range(cfg.gpus)).cuda() # build runner optimizer = build_optimizer(model, cfg.optimizer) runner = Runner(model, lambda x: x, optimizer, cfg.work_dir, cfg.log_level) runner.resume(cfg.resume_from) model = runner.model # ONLY FOR DEBUG # print('remove DDP for debug!') # model = model._modules['module'] model.eval() dataset_cfg = eval_dataset_mapper[args.dataset] dataset_cfg.update(cfg.common_val_cfg) dataset_cfg.pop('max_samples') dataset = get_dataset(dataset_cfg) # dataset.debugging = True shuffle = False if args.dataset in stable_list else True data_loader = build_dataloader_fuse( dataset, 1, 0, cfg.gpus, dist=False, shuffle=shuffle, drop_last=False, ) dump_dir = os.path.join(cfg.work_dir, f'eval_{args.dataset}') os.makedirs(dump_dir, exist_ok=True) if args.viz_dir: os.makedirs(args.viz_dir, exist_ok=True) eval_handler = eval_handler_mapper[args.dataset]( writer=tqdm.write, viz_dir=args.viz_dir, FOCAL_LENGTH=FOCAL_LENGTH, work_dir=cfg.work_dir) # type: EvalHandler with torch.no_grad(): for i, data_batch in enumerate(tqdm(data_loader)): file_name = data_batch['img_meta'].data[0][0]['file_name'] try: bbox_results, pred_results = model(**data_batch, return_loss=False, use_gt_bboxes=args.use_gt) pred_results['bboxes'] = bbox_results if args.paper_dir: os.makedirs(args.paper_dir, exist_ok=True) img = denormalize(data_batch['img'].data[0][0].numpy()) verts = pred_results['pred_vertices'] + pred_results[ 'pred_translation'] dump_folder = osp.join(args.paper_dir, file_name) os.makedirs(dump_folder, exist_ok=True) plt.imsave(osp.join(dump_folder, 'img.png'), img) for obj_i, vert in enumerate(verts): nr.save_obj(osp.join(dump_folder, f'{obj_i}.obj'), vert, torch.tensor(smpl.faces.astype(np.int64))) save_pack = eval_handler(data_batch, pred_results, use_gt=args.use_gt) save_pack.update({'bbox_results': pred_results['bboxes']}) if args.dump_pkl: with open( osp.join(dump_dir, f"{save_pack['file_name']}.pkl"), 'wb') as f: pickle.dump(save_pack, f) except Exception as e: tqdm.write(f"Fail on {file_name}") tqdm.write(str(e)) eval_handler.finalize()
def fgsm(rank, args, shared_model, number): torch.cuda.set_device(args.gpus.index(args.gpus[rank % len(args.gpus)])) all_n = 0 # torch.cuda.set_device(args.gpus.index(args.gpus[rank % len(args.gpus)])) model_kwargs = {'vocab': load_vocab(args.vocab_json)} model = VqaLstmCnnAttentionModel(**model_kwargs) device_ids = [0,1] model = model.cuda(device_ids[0]) model = torch.nn.DataParallel(model, device_ids=device_ids) # torch.backends.cudnn.benchmark = True lossFn = torch.nn.CrossEntropyLoss().cuda() eval_loader_kwargs = { 'questions_h5': getattr(args, args.eval_split + '_h5'), 'data_json': args.data_json, 'vocab': args.vocab_json, 'target_obj_conn_map_dir': args.target_obj_conn_map_dir, 'batch_size': args.batch_size, 'input_type': args.input_type, 'num_frames': args.num_frames, 'split': args.eval_split, 'max_threads_per_gpu': args.max_threads_per_gpu, 'gpu_id': args.gpus[rank%len(args.gpus)], 'to_cache': args.cache } eval_loader = EqaDataLoader(**eval_loader_kwargs) # for ijcai in range(number): # eval_loader.dataset._load_envs() eval_loader.dataset._load_envs(start_idx=number) print('eval_loader has %d samples' % len(eval_loader.dataset)) args.output_log_path = os.path.join(args.log_dir, 'eval_' + str(rank) + '.json') model.load_state_dict(handle_load(shared_model.state_dict())) model.eval() metrics = VqaMetric( info={'split': args.eval_split}, metric_names=[ 'loss', 'accuracy', 'mean_rank', 'mean_reciprocal_rank' ], log_json=args.output_log_path) all_envs_loaded = eval_loader.dataset._check_if_all_envs_loaded() done = False print(number, ' begin') import copy import torch.nn as nn from PIL import Image softmax = nn.Softmax() allcor = [] while done == False: for ii,batch in enumerate(eval_loader): # model.cuda() if(ii>=0): idx, questions, answers, house, v, f, vt, pos, _, _, _ = batch print('all size:',v.size(),f.size(),vt.size()) #print(house) print(questions, answers) questions_var = Variable(questions.cuda()) print(questions_var.size()) answers_var = Variable(answers.cuda()) v_var = Variable(v.cuda(),requires_grad=True) f_var = Variable(f.cuda()) vt_var = Variable(vt.cuda(),requires_grad=True) begin, end = get_info(idx[0], house[0]) #print(vt_var[0][0][40010][0][0]) #print(vt_var[0][0][39506][0][0]) vt_test = copy.deepcopy(vt_var) epsilon = 32.0/256.0 vt_grad = torch.zeros(vt_var.size()).cuda() #vt_var.retain_grad() scores, att_probs,img_clean = model(v_var, f_var, vt_var, pos, questions_var) i1, i2 = torch.max(scores[0],0) mi = i2.cpu().numpy().tolist() ms = int(answers) print(mi) print(mi==int(ms)) if(mi==int(ms)): allcor.append(1.0) else: allcor.append(0.0) print(softmax(scores[0])) print(softmax(scores[0])[ms]) img_clean = img_clean[0] for iii in range(img_clean.size()[0]): imggg = img_clean[iii].detach().cpu().numpy() imggg = imggg * 255.0 imggg = imggg.transpose((1,2,0)) imggg = Image.fromarray(imggg.astype('uint8')) imggg.save('result_test/'+str(ii)+'_'+str(iii)+'_clean.jpg') loss = lossFn(scores, answers_var) loss.backward() #print(torch.max(vt_grad)) vt_grad = vt_var.grad v_grad = v_var.grad print('max grad',torch.max(vt_grad.data)) #print(vt_grad[0][0][40010][0][0]) #print(vt_grad[0][0][39506][0][0]) vt_var = vt_var.detach() + epsilon * torch.sign(vt_grad) #v_var = v_var.detach() + 1.0 * torch.sign(v_grad) vt_var = torch.clamp(vt_var, 0, 1) #vt_var = Variable(vt_var.data, requires_grad=True).cuda() with torch.no_grad(): model.eval() begin, end = get_info(idx[0], house[0]) #for iii in range(begin,end): #if(vt_test[0][0][iii][0][0][0][0] != vt_var[0][0][iii][0][0][0][0] or vt_test[0][0][iii][0][0][1][0] != vt_var[0][0][iii][0][0][1][0] or vt_test[0][0][iii][0][1][1][0] != vt_var[0][0][iii][0][1][1][0]): #print(iii) vt_test[0][0][begin:end] = vt_var[0][0][begin:end] #print(vt_test[0][0][40010][0][0]) #print(vt_test[0][0][39506][0][0]) #print((vt_test[0][0] == vt_var[0][0]).sum()) #print((vt_test[0][0].size()),(vt_var[0][0].size())) scores, att_probs,imgg = model(v_var, f_var, vt_test, pos, questions_var) imgg = imgg[0] #print(imgg.size()) for iii in range(imgg.size()[0]): imggg = imgg[iii].detach().cpu().numpy() imggg = imggg * 255.0 imggg = imggg.transpose((1,2,0)) imggg = Image.fromarray(imggg.astype('uint8')) imggg.save('result_test/'+str(ii)+'_'+str(iii)+'_adv.jpg') i1, i2 = torch.max(scores[0],0) mi = i2.cpu().numpy().tolist() ms = int(answers) print(mi) print(mi==int(ms)) print(softmax(scores[0])) print(softmax(scores[0])[ms]) for k in range(idx.shape[0]): begin, end = get_info(idx[k], house[k]) #print(begin, end) #begin, end = find_index('167', house[k]) v_m = v_var[k][0] f_m = f_var[k][0][begin:end] vt_m = vt_var[k][0][begin:end] nr.save_obj('/media/trs1/dataset/suncg_data/house/' + house[k] + '/attack_' + str(int(idx[k])) + '.obj', v_m, f_m, vt_m) idx = idx.cpu() questions = questions.cpu() answers = answers.cpu() questions_var = questions_var.cpu() answers_var = answers_var.cpu() v = v.cpu() f = f.cpu() vt = vt.cpu() v_m = v_m.cpu() f_m = f_m.cpu() vt_m = vt_m.cpu() v_var = v_var.detach().cpu() f_var = f_var.cpu() vt_var = vt_var.detach().cpu() vt_grad = vt_grad.cpu() print(house[k] + ' ' + str(int(idx[k])) + ' ok') all_n += 1 # handle_file(path) if all_envs_loaded == False: eval_loader.dataset._load_envs() if len(eval_loader.dataset.pruned_env_set) == 0: done = True else: done = True print(allcor) print(number, ' over')
def run(): # settings parser = argparse.ArgumentParser() parser.add_argument('-i', '--filename_obj', type=str) parser.add_argument('-o', '--filename_output', type=str) parser.add_argument('-d', '--output_directory', type=str) parser.add_argument('-al', '--adam_lr', type=float, default=0.01) parser.add_argument('-ab1', '--adam_beta1', type=float, default=0.9) parser.add_argument('-bs', '--batch_size', type=int, default=4) parser.add_argument('-ni', '--num_iteration', type=int, default=1000) parser.add_argument('-cd', '--camera_distance', type=float, default=2.5) parser.add_argument('-ib', '--init_bias', type=str, default='(0,0,0)') parser.add_argument('-g', '--gpu', type=int, default=0) args = parser.parse_args() args.init_bias = tuple([float(v) for v in args.init_bias[1:-1].split(',')]) # create output directory if not os.path.exists(args.output_directory): os.makedirs(args.output_directory) # setup chainer chainer.cuda.get_device_from_id(args.gpu).use() cp.random.seed(0) np.random.seed(0) # setup scene & optimizer model = deep_dream_3d.DeepDreamModel( args.filename_obj, camera_distance=args.camera_distance, init_bias=args.init_bias) model.to_gpu() optimizer = neural_renderer.Adam(alpha=args.adam_lr, beta1=args.adam_beta1) optimizer.setup(model) # optimization loop = tqdm.tqdm(range(args.num_iteration)) for _ in loop: optimizer.target.cleargrads() loss = model(args.batch_size) loss.backward() optimizer.update() loop.set_description('Optimizing. Loss %.4f' % loss.data) ## save obj #pdb.set_trace() vertices,faces,textures = model.mesh.get_batch(args.batch_size) ## fill back textures_1 = chainer.functions.concat((textures, textures.transpose((0, 1, 4, 3, 2, 5))), axis=1) faces_1 = chainer.functions.concat((faces, faces[:, :, ::-1]), axis=1).data # obj_fn = args.filename_output.split('/')[-1].split('.')[0] output_directory = os.path.split(args.filename_output)[0]#'/'.join(args.filename_output.split('/')[-3:-1]) #neural_renderer.save_obj('%s/%s.obj'% (output_directory,obj_fn), model.mesh.vertices.array, model.mesh.faces, model.mesh.textures.array) neural_renderer.save_obj('%s/%s.obj'% (output_directory,obj_fn), vertices[0], faces[0], textures[0].array) #neural_renderer.save_obj('%s/%s.obj'% (output_directory,obj_fn), model.mesh.vertices[0], model.mesh.faces[0], chainer.functions.tanh(model.textures_1[0]).array) # draw object model.renderer.background_color = (1, 1, 1) loop = tqdm.tqdm(range(0, 360, 4)) for num, azimuth in enumerate(loop): loop.set_description('Drawing') model.renderer.eye = neural_renderer.get_points_from_angles(2.732, 30, azimuth) images = model.renderer.render(*model.mesh.get_batch(1)) image = images.data.get()[0].transpose((1, 2, 0)) scipy.misc.toimage(image, cmin=0, cmax=1).save('%s/_tmp_%04d.png' % (args.output_directory, num)) make_gif(args.output_directory, args.filename_output)
def attack_fgsm(rank, args, shared_model, number): #? torch.cuda.set_device(args.gpus.index(args.gpus[rank % len(args.gpus)])) all_n = 0 # torch.cuda.set_device(args.gpus.index(args.gpus[rank % len(args.gpus)])) model_kwargs = {'vocab': load_vocab(args.vocab_json)} model = VqaLstmCnnAttentionModel(**model_kwargs) device_ids = [0, 1] model = model.cuda(device_ids[0]) model = torch.nn.DataParallel(model, device_ids=device_ids) # torch.backends.cudnn.benchmark = True lossFn = torch.nn.CrossEntropyLoss().cuda() eval_loader_kwargs = { 'questions_h5': getattr(args, args.eval_split + '_h5'), 'data_json': args.data_json, 'vocab': args.vocab_json, 'batch_size': args.batch_size, 'input_type': args.input_type, 'num_frames': args.num_frames, 'split': args.eval_split, 'max_threads_per_gpu': args.max_threads_per_gpu, 'gpu_id': args.gpus[rank % len(args.gpus)], 'to_cache': args.cache } eval_loader = EqaDataLoader(**eval_loader_kwargs) # for ijcai in range(number): # eval_loader.dataset._load_envs() eval_loader.dataset._load_envs(start_idx=number) print('eval_loader has %d samples' % len(eval_loader.dataset)) args.output_log_path = os.path.join(args.log_dir, 'eval_' + str(rank) + '.json') model.load_state_dict(handle_load(shared_model.state_dict())) model.eval() metrics = VqaMetric( info={'split': args.eval_split}, metric_names=['loss', 'accuracy', 'mean_rank', 'mean_reciprocal_rank'], log_json=args.output_log_path) all_envs_loaded = eval_loader.dataset._check_if_all_envs_loaded() done = False while done == False: for batch in eval_loader: idx, questions, answers, house, v, f, vt, pos, _, _, _ = batch questions_var = Variable(questions.cuda()) answers_var = Variable(answers.cuda()) v_var = Variable(v.cuda(), requires_grad=True) f_var = Variable(f.cuda()) vt_var = Variable(vt.cuda(), requires_grad=True) # noise level epsilon = 12.0 / 255.0 scores, att_probs = model(v_var, f_var, vt_var, pos, questions_var, 0, '0') loss = lossFn(scores, answers_var) loss.backward() # get grad for attack vt_grad = vt_var.grad vt_detach = vt_var.detach() begin, end, oid = get_info(idx[0], house[0]) if (begin == 1000 and end == 2000): print(str(int(idx[0])), 'error') vt_grad[0][0][:begin] = 0 vt_grad[0][0][end:] = 0 vt_var = vt_detach + epsilon * torch.sign(vt_grad) for k in range(idx.shape[0]): begin, end, oid = get_info(idx[k], house[k]) v_m = v_var[k][0] f_m = f_var[k][0][begin:end] vt_m = vt_var[k][0][begin:end] # save changed object to .obj file nr.save_obj( '/path/to/data/house/' + house[k] + '/attack_' + str(int(idx[k])) + '_' + str(oid) + '.obj', v_m, f_m, vt_m) with torch.no_grad(): model.eval() scores, att_probs = model(v_var, f_var, vt_var, pos, questions_var, 0, '0') accuracy, ranks = metrics.compute_ranks( scores.data.cpu(), answers) if all_envs_loaded == False: eval_loader.dataset._load_envs() if len(eval_loader.dataset.pruned_env_set) == 0: done = True else: done = True
def attack_pgd(rank, args, shared_model, number): torch.cuda.set_device(args.gpus.index(args.gpus[rank % len(args.gpus)])) all_n = 0 model_kwargs = {'vocab': load_vocab(args.vocab_json)} model = VqaLstmCnnAttentionModel(**model_kwargs) device_ids = [0, 1] model = model.cuda(device_ids[0]) model = torch.nn.DataParallel(model, device_ids=device_ids) # torch.backends.cudnn.benchmark = True lossFn = torch.nn.CrossEntropyLoss().cuda() eval_loader_kwargs = { 'questions_h5': getattr(args, args.eval_split + '_h5'), 'data_json': args.data_json, 'vocab': args.vocab_json, 'batch_size': args.batch_size, 'input_type': args.input_type, 'num_frames': args.num_frames, 'split': args.eval_split, 'max_threads_per_gpu': args.max_threads_per_gpu, 'gpu_id': args.gpus[rank % len(args.gpus)], 'to_cache': args.cache } eval_loader = EqaDataLoader(**eval_loader_kwargs) eval_loader.dataset._load_envs(start_idx=number) print('eval_loader has %d samples' % len(eval_loader.dataset)) args.output_log_path = os.path.join(args.log_dir, 'eval_' + str(rank) + '.json') model.load_state_dict(handle_load(shared_model.state_dict())) model.eval() metrics = VqaMetric( info={'split': args.eval_split}, metric_names=['loss', 'accuracy', 'mean_rank', 'mean_reciprocal_rank'], log_json=args.output_log_path) all_envs_loaded = eval_loader.dataset._check_if_all_envs_loaded() done = False while done == False: for batch in eval_loader: # model.cuda() idx, questions, answers, house, v, f, vt, pos, _, _, _ = batch questions_var = Variable(questions.cuda()) answers_var = Variable(answers.cuda()) v_var = Variable(v.cuda(), requires_grad=True) f_var = Variable(f.cuda()) vt_var = Variable(vt.cuda(), requires_grad=True) attack_iter = 10 attack_momentum = 1 alpha = 2.0 / 255 epsilon = 16.0 / 255 vt_grad = torch.zeros(vt_var.size()).cuda() for j in range(attack_iter): scores, att_probs = model(v_var, f_var, vt_var, pos, questions_var, j, str(int(idx[0]))) loss = lossFn(scores, answers_var) loss.backward() vg = vt_var.grad begin, end, _ = get_info(idx[0], house[0]) if (begin == 1000 and end == 2000): print(str(int(idx[0])), 'error') vg[0][0][:begin] = 0 vg[0][0][end:] = 0 noise = attack_momentum * vt_grad + vg vt_grad = noise vt_var = vt_var.detach() + alpha * torch.sign(noise) vt_var = torch.where(vt_var > vt + epsilon, vt + epsilon, vt_var) vt_var = torch.clamp(vt_var, 0, 1) vt_var = torch.where(vt_var < vt - epsilon, vt - epsilon, vt_var) vt_var = torch.clamp(vt_var, 0, 1) vt_var = Variable(vt_var.data, requires_grad=True).cuda() with torch.no_grad(): model.eval() scores, att_probs = model(v_var, f_var, vt_var, pos, questions_var, 100, str(int(idx[0]))) accuracy, ranks = metrics.compute_ranks( scores.data.cpu(), answers) begin, end, oid = get_info(idx[0], house[0]) v_m = v_var[0][0] f_m = f_var[0][0][begin:end] vt_m = vt_var[0][0][begin:end] nr.save_obj( '/path/to/data/house/' + house[0] + '/attack_' + str(int(idx[0])) + '_' + str(oid) + '.obj', v_m, f_m, vt_m) if all_envs_loaded == False: eval_loader.dataset._load_envs() if len(eval_loader.dataset.pruned_env_set) == 0: done = True else: done = True
ploter.plot('Adv_loss', 'train', 'Adv-loss', batches_done, Adv_loss.item()) if batches_done % opt.sample_step == 0 or batches_done == last_iter: save_image(gen_imgs.data[:min(25, gen_imgs.shape[0])], os.path.join(opt.sample_dir, '%05d-gen.png' % batches_done), nrow=5, normalize=True) save_image(gt_imgs.data[:min(25, gt_imgs.shape[0])], os.path.join(opt.sample_dir, '%05d-gt.png' % batches_done), nrow=5, normalize=True) nr.save_obj( os.path.join(opt.sample_dir, '%05d.obj' % batches_done), vertices[0, :, :], faces[0, :, :]) print('Saved sample image to {}...'.format(opt.sample_dir)) # validation on val dataset if batches_done % opt.ckpt_step == 0 or batches_done == last_iter: iou_val = eval_IoU(encoder, mesh_generator, dataset_val) if opt.dataset == 'CVPR18': MMD_val = eval_MMD(encoder, dataset_val) f_log.write('batches_done: %d, validation iou: %f\r\n' % (batches_done, iou_val)) ploter.plot('IoU_loss', 'voxel_IoU_val', 'IoU-loss', batches_done, iou_val) if opt.dataset == 'CVPR18': ploter.plot('IoU_loss', 'MMD_val', 'IoU-loss', batches_done, MMD_val)