def compileShader(self): create_opengl_context((self.width, self.height)) glClear(GL_COLOR_BUFFER_BIT) self.shader = shaders.compileProgram(self.compileVertexShader(), self.compileFragmentShader()) self.xpos = glGetUniformLocation(self.shader, 'xpos') self.ypos = glGetUniformLocation(self.shader, 'ypos') self.vdir_x = glGetUniformLocation(self.shader, 'vdir_x') self.vdir_y = glGetUniformLocation(self.shader, 'vdir_y') self.vdir_z = glGetUniformLocation(self.shader, 'vdir_z') self.arrow_size = glGetUniformLocation(self.shader, 'size') self.res_loc = glGetUniformLocation(self.shader, 'iResolution')
def main(): # Parse args args = parse_args(object_yaw_min=0, object_yaw_max=360, object_pitch_min=-10, object_pitch_max=30, object_z_min=15, object_z_max=60) # Setup logging logger = logging.getLogger() logger.setLevel(logging.ERROR) tf.logging.set_verbosity(tf.logging.ERROR) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' log = logging.getLogger('shapeshifter') log.setLevel(logging.DEBUG if args.verbose else logging.INFO) formatter = logging.Formatter( '%(asctime)s: %(levelname)s %(name)s: %(message)s') handler = logging.StreamHandler() handler.setFormatter(formatter) log.addHandler(handler) log.propagate = False # Set seeds from the start if args.seed: log.debug("Setting seed") np.random.seed(args.seed) tf.set_random_seed(args.seed) # Load textures, textures_masks, backgrounds, and mesh log.debug("Loading textures, textures masks, backgrounds, and mesh") backgrounds = load_and_tile_images(args.backgrounds) textures = load_and_tile_images(args.textures) textures_masks = (load_and_tile_images(args.textures_masks)[:, :, :, :1] >= 0.5).astype(np.float32) assert (textures.shape[:3] == textures_masks.shape[:3]) objects = [ meshutil.normalize_mesh(meshutil.load_obj(obj)) for obj in args.objects ] objects_masks = None # Create OpenGL context and mesh renderer log.debug("Creating renderer") create_opengl_context((backgrounds.shape[2], backgrounds.shape[1])) renderer = glrenderer.MeshRenderer( (backgrounds.shape[2], backgrounds.shape[1])) # Create test data generate_data_partial = partial(generate_data, renderer=renderer, backgrounds=backgrounds, objects=objects, objects_masks=objects_masks, objects_class=args.target_class, objects_transforms={ 'yaw_range': args.object_yaw_range, 'yaw_bins': args.object_yaw_bins, 'yaw_fn': args.object_yaw_fn, 'pitch_range': args.object_pitch_range, 'pitch_bins': args.object_pitch_bins, 'pitch_fn': args.object_pitch_fn, 'roll_range': args.object_roll_range, 'roll_bins': args.object_roll_bins, 'roll_fn': args.object_roll_fn, 'x_range': args.object_x_range, 'x_bins': args.object_x_bins, 'x_fn': args.object_x_fn, 'y_range': args.object_y_range, 'y_bins': args.object_y_bins, 'y_fn': args.object_y_fn, 'z_range': args.object_z_range, 'z_bins': args.object_z_bins, 'z_fn': args.object_z_fn }, textures_transforms={ 'yaw_range': args.texture_yaw_range, 'yaw_bins': args.texture_yaw_bins, 'yaw_fn': args.texture_yaw_fn, 'pitch_range': args.texture_pitch_range, 'pitch_bins': args.texture_pitch_bins, 'pitch_fn': args.texture_pitch_fn, 'roll_range': args.texture_roll_range, 'roll_bins': args.texture_roll_bins, 'roll_fn': args.texture_roll_fn, 'x_range': args.texture_x_range, 'x_bins': args.texture_x_bins, 'x_fn': args.texture_x_fn, 'y_range': args.texture_y_range, 'y_bins': args.texture_y_bins, 'y_fn': args.texture_y_fn, 'z_range': args.texture_z_range, 'z_bins': args.texture_z_bins, 'z_fn': args.texture_z_fn }, seed=args.seed) # Create adversarial textures, render mesh using them, and pass rendered images into model. Finally, create summary statistics. log.debug("Creating perturbable texture") textures_var_, textures_ = create_textures( textures, textures_masks, use_spectral=args.spectral, soft_clipping=args.soft_clipping) log.debug("Creating rendered input images") input_images_ = create_rendered_images(args.batch_size, textures_) log.debug("Creating object detection model") predictions, detections, losses = create_model(input_images_, args.model_config, args.model_checkpoint, is_training=True) log.debug("Creating attack losses") victim_class_, target_class_, losses_summary_ = create_attack( textures_, textures_var_, predictions, losses, optimizer_name=args.optimizer, clip=args.sign_gradients) log.debug("Creating evaluation metrics") metrics_summary_, texture_summary_ = create_evaluation( victim_class_, target_class_, textures_, textures_masks, input_images_, detections) summaries_ = tf.summary.merge([losses_summary_, metrics_summary_]) global_init_op_ = tf.global_variables_initializer() local_init_op_ = tf.local_variables_initializer() # Create tensorboard file writer for train and test evaluations saver = tf.train.Saver([textures_var_, tf.train.get_global_step()]) train_writer = None test_writer = None if args.logdir is not None: log.debug(f"Tensorboard logging: {args.logdir}") os.makedirs(args.logdir, exist_ok=True) arguments_summary_ = tf.summary.text( 'Arguments', tf.constant('```' + ' '.join(sys.argv[1:]) + '```')) # TODO: Save argparse graph = None if args.save_graph: log.debug("Graph will be saved to tensorboard") graph = tf.get_default_graph() train_writer = tf.summary.FileWriter(args.logdir + '/train', graph=graph) test_writer = tf.summary.FileWriter(args.logdir + '/test') # Find existing checkpoint os.makedirs(args.logdir + '/checkpoints', exist_ok=True) checkpoint_path = tf.train.latest_checkpoint(args.logdir + '/checkpoints') args.checkpoint = checkpoint_path # Create session log.debug("Creating session") config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) sess.run(global_init_op_) sess.run(local_init_op_) # Set initial texture if args.checkpoint is not None: log.debug(f"Restoring from checkpoint: {args.checkpoint}") saver.restore(sess, args.checkpoint) else: if args.gray_start: log.debug("Setting texture to gray") textures = np.zeros_like(textures) + 128 / 255 if args.random_start > 0: log.debug( f"Adding uniform random perturbation texture with at most {args.random_start}/255 per pixel" ) textures = textures + np.random.randint( size=textures.shape, low=-args.random_start, high=args.random_start) / 255 sess.run('project_op', {'textures:0': textures}) # Get global step step = sess.run('global_step:0') if train_writer is not None: log.debug("Running arguments summary") summary = sess.run(arguments_summary_) train_writer.add_summary(summary, step) test_writer.add_summary(summary, step) loss_tensors = [ 'total_loss:0', 'total_rpn_cls_loss:0', 'total_rpn_loc_loss:0', 'total_rpn_foreground_loss:0', 'total_rpn_background_loss:0', 'total_rpn_cw_loss:0', 'total_box_cls_loss:0', 'total_box_loc_loss:0', 'total_box_target_loss:0', 'total_box_victim_loss:0', 'total_box_target_cw_loss:0', 'total_box_victim_cw_loss:0', 'total_sim_loss:0', 'grad_l2:0', 'grad_linf:0' ] metric_tensors = [ 'proposal_average_precision:0', 'victim_average_precision:0', 'target_average_precision:0' ] output_tensors = loss_tensors + metric_tensors log.info( 'global_step [%s]', ', '.join([ tensor.replace(':0', '').replace('total_', '') for tensor in output_tensors ])) test_feed_dict = { 'learning_rate:0': args.learning_rate, 'momentum:0': args.momentum, 'decay:0': args.decay, 'rpn_iou_thresh:0': args.rpn_iou_threshold, 'rpn_cls_weight:0': args.rpn_cls_weight, 'rpn_loc_weight:0': args.rpn_loc_weight, 'rpn_foreground_weight:0': args.rpn_foreground_weight, 'rpn_background_weight:0': args.rpn_background_weight, 'rpn_cw_weight:0': args.rpn_cw_weight, 'rpn_cw_conf:0': args.rpn_cw_conf, 'box_iou_thresh:0': args.box_iou_threshold, 'box_cls_weight:0': args.box_cls_weight, 'box_loc_weight:0': args.box_loc_weight, 'box_target_weight:0': args.box_target_weight, 'box_victim_weight:0': args.box_victim_weight, 'box_target_cw_weight:0': args.box_target_cw_weight, 'box_target_cw_conf:0': args.box_target_cw_conf, 'box_victim_cw_weight:0': args.box_victim_cw_weight, 'box_victim_cw_conf:0': args.box_victim_cw_conf, 'sim_weight:0': args.sim_weight, 'victim_class:0': args.victim_class, 'target_class:0': args.target_class } # Keep attacking until CTRL+C. The only issue is that we may be in the middle of some operation. try: log.debug("Entering attacking loop (use ctrl+c to exit)") while True: # Run summaries as necessary if args.logdir and step % args.save_checkpoint_every == 0: log.debug("Saving checkpoint") saver.save(sess, args.logdir + '/checkpoints/texture', global_step=step, write_meta_graph=False, write_state=True) if step % args.save_texture_every == 0 and test_writer is not None: log.debug("Writing texture summary") test_texture = sess.run(texture_summary_) test_writer.add_summary(test_texture, step) if step % args.save_test_every == 0: log.debug("Runnning test summaries") start_time = time.time() sess.run(local_init_op_) batch_accumulate(sess, test_feed_dict, args.test_batch_size, args.batch_size, generate_data_partial, detections, predictions, args.categories) end_time = time.time() log.debug( f"Loss accumulation took {end_time - start_time} seconds") test_output = sess.run(output_tensors, test_feed_dict) log.info('test %d %s', step, test_output) if test_writer is not None: log.debug("Writing test summaries") test_summaries = sess.run(summaries_, test_feed_dict) test_writer.add_summary(test_summaries, step) # Create train feed_dict train_feed_dict = test_feed_dict.copy() train_feed_dict[ 'image_channel_multiplicative_noise:0'] = args.image_multiplicative_channel_noise_range train_feed_dict[ 'image_channel_additive_noise:0'] = args.image_additive_channel_noise_range train_feed_dict[ 'image_pixel_multiplicative_noise:0'] = args.image_multiplicative_pixel_noise_range train_feed_dict[ 'image_pixel_additive_noise:0'] = args.image_additive_pixel_noise_range train_feed_dict[ 'image_gaussian_noise_stddev:0'] = args.image_gaussian_noise_stddev_range train_feed_dict[ 'texture_channel_multiplicative_noise:0'] = args.texture_multiplicative_channel_noise_range train_feed_dict[ 'texture_channel_additive_noise:0'] = args.texture_additive_channel_noise_range train_feed_dict[ 'texture_pixel_multiplicative_noise:0'] = args.texture_multiplicative_pixel_noise_range train_feed_dict[ 'texture_pixel_additive_noise:0'] = args.texture_additive_pixel_noise_range train_feed_dict[ 'texture_gaussian_noise_stddev:0'] = args.texture_gaussian_noise_stddev_range # Zero out gradient accumulation, losses, and metrics, then accumulate batches log.debug("Starting gradient accumulation...") start_time = time.time() sess.run(local_init_op_) batch_accumulate(sess, train_feed_dict, args.train_batch_size, args.batch_size, generate_data_partial, detections, predictions, args.categories) end_time = time.time() log.debug( f"Gradient accumulation took {end_time - start_time} seconds") train_output = sess.run(output_tensors, train_feed_dict) log.info('train %d %s', step, train_output) if step % args.save_train_every == 0 and train_writer is not None: log.debug("Writing train summaries") train_summaries = sess.run(summaries_, test_feed_dict) train_writer.add_summary(train_summaries, step) # Update textures and project texture to feasible set # TODO: We can probably run these together but probably need some control dependency log.debug("Projecting attack") sess.run('attack_op', train_feed_dict) sess.run('project_op') step = sess.run('global_step:0') except KeyboardInterrupt: log.warn('Interrupted') finally: if test_writer is not None: test_writer.close() if train_writer is not None: train_writer.close() if sess is not None: sess.close()
from lucid.misc.gl import meshutil from lucid.misc.gl import glrenderer import lucid.misc.io.showing as show import lucid.misc.io as lucid_io from lucid.misc.tfutil import create_session from lucid.modelzoo import vision_models from lucid.optvis import objectives from lucid.optvis import param from lucid.optvis.style import StyleLoss, mean_l1_loss from lucid.optvis.param.spatial import sample_bilinear """You can check the installed version of OpenGL:""" create_opengl_context() gl.glGetString(gl.GL_VERSION) model = vision_models.InceptionV1() model.load_graphdef() """## Loading 3D model Let's download some 3D models first. This is similar to the steps in the [3D Feature Visualization notebook](https://colab.research.google.com/github/tensorflow/lucid/blob/master/notebooks/differentiable-parameterizations/texture_synth_3d.ipynb) if you're keen on the details and haven't completed that notebook yet. """ TEXTURE_SIZE = 1024 !gsutil cp gs://deepdream/article_models.zip . && \ unzip -qo article_models.zip && \ ls -al article_models && \
import pytest import os import numpy as np HAVE_COLAB_NVIDIA = (os.path.exists('/usr/lib64-nvidia/') and os.path.exists('/opt/bin/nvidia-smi')) WIDTH, HEIGHT = 200, 100 if HAVE_COLAB_NVIDIA: from lucid.misc.gl import glcontext # must be imported before OpenGL.GL import OpenGL.GL as gl from lucid.misc.gl import glrenderer glcontext.create_opengl_context((WIDTH, HEIGHT)) @pytest.mark.skipif(not HAVE_COLAB_NVIDIA, reason="GPU Colab kernel only") def test_gl_context(): # Render triangle gl.glClear(gl.GL_COLOR_BUFFER_BIT) gl.glBegin(gl.GL_TRIANGLES) gl.glColor3f(1, 0, 0) gl.glVertex2f(0, 1) gl.glColor3f(0, 1, 0) gl.glVertex2f(-1, -1) gl.glColor3f(0, 0, 1) gl.glVertex2f(1, -1)