def get_transforms(complexity=0): if complexity == 4: return [ transform.pad(32), transform.jitter(64), transform.random_scale([n / 100. for n in range(80, 120)]), transform.random_rotate(list(range(-10, 11)) + 5 * [0]), #transform.random_rotate( list( range( -10, 10 ) ) + list( range( -5, 5 ) ) + 10 * list( range( -2, 2 ) ) ), transform.jitter(8) ] if complexity == 3: return [ transform.pad(16), transform.jitter(16), transform.random_scale([1 + (i - 5) / 50. for i in range(11)]), transform.random_rotate(list(range(-10, 11)) + 5 * [0]), transform.jitter(8) ] if complexity == 2: return transform.standard_transforms if complexity == 1: # no rotations return [ transform.pad(16), transform.jitter(32), transform.random_scale([n / 100. for n in range(80, 120)]), transform.jitter(2) ] else: # no transforms return []
def render_atlas_tile(model,op_name,directions,icon_size=45,n_steps=127,transforms_amount=1,cossim_pow=0,L2_amount=2): transforms_options = [ [ transform.jitter(2) ], [ transform.pad(12, mode="constant", constant_value=.5), transform.jitter(8), transform.random_scale([1 + (i - 5) / 50. for i in range(11)]), transform.random_rotate(list(range(-10, 11)) + 5 * [0]), transform.jitter(4), ], [ transform.pad(2, mode='constant', constant_value=.5), transform.jitter(4), transform.jitter(4), transform.jitter(8), transform.jitter(8), transform.jitter(8), transform.random_scale([0.995**n for n in range(-5,80)] + [0.998**n for n in 2*list(range(20,40))]), transform.random_rotate(list(range(-20,20))+list(range(-10,10))+list(range(-5,5))+5*[0]), transform.jitter(2), ], ] param_f = lambda: param.image(icon_size, batch=directions.shape[0]) obj = objectives.Objective.sum( [objectives.direction_neuron(op_name, v, batch=n, cossim_pow=cossim_pow) for n,v in enumerate(directions) ]) - L2_amount * objectives.L2("input", 0.5) * objectives.L2("input", 0.5) thresholds=(n_steps//2, n_steps) vis_imgs = render.render_vis(model, obj, param_f, transforms=transforms_options[transforms_amount], thresholds=thresholds, verbose=False)[-1] return vis_imgs
def feature_inversion(img, model, layer=None, n_steps=512, cossim_pow=0.0): with tf.Graph().as_default(), tf.Session() as sess: img = imgToModelSize(img, model) objective = objectives.Objective.sum([ 1.0 * dot_compare(layer, cossim_pow=cossim_pow), objectives.blur_input_each_step(), ]) t_input = tf.placeholder(tf.float32, img.shape) param_f = param.image(img.shape[0], decorrelate=True, fft=True, alpha=False) param_f = tf.stack([param_f[0], t_input]) transforms = [ transform.pad(8, mode='constant', constant_value=.5), transform.jitter(8), transform.random_scale([0.9, 0.95, 1.05, 1.1] + [1]*4), transform.random_rotate(list(range(-5, 5)) + [0]*5), transform.jitter(2), ] T = render.make_vis_T(model, objective, param_f, transforms=transforms) loss, vis_op, t_image = T("loss"), T("vis_op"), T("input") tf.global_variables_initializer().run() for i in range(n_steps): _ = sess.run([vis_op], {t_input: img}) result = t_image.eval(feed_dict={t_input: img}) show(result[0])
def generate(model, args): print('[GENERATE] Ran with layer {} and neuron {}'.format( args['layer'], args['neuron'])) layer_id = args['layer'].split(' ')[0] layer_neuron = '{}:{}'.format(layer_id, args['neuron']) s = int(args['size']) min_scale = args['transform_min'] max_scale = args['transform_max'] scale_offset = (max_scale - min_scale) * 10 # https://github.com/tensorflow/lucid/issues/148 with tf.Graph().as_default() as graph, tf.Session() as sess: t_img = param.image(s) crop_W = int(s / 2) t_offset = tf.random.uniform((2, ), 0, s - crop_W, dtype="int32") t_img_crop = t_img[:, t_offset[0]:t_offset[0] + crop_W, t_offset[1]:t_offset[1] + crop_W] if (args['transforms']): transforms = [ transform.jitter(2), transform.random_scale( [min_scale + n / 10. for n in range(20)]), transform.random_rotate(range(-10, 11)), transform.jitter(2) ] T = render.make_vis_T(model, layer_neuron, t_img_crop, transforms=transforms) else: T = render.make_vis_T(model, layer_neuron, t_img_crop) tf.initialize_all_variables().run() for i in range(1024): T("vis_op").run() img = t_img.eval()[0] # https://github.com/tensorflow/lucid/issues/108 # img = render.render_vis(model, layer_neuron)[-1][0] img = Image.fromarray(np.uint8(img * 255)) return {'image': img}
def __init__(self, content, overlap_px=10, repeat=2, num_strokes=5, painter_type="GAN", connected=True, alternate=True, bw=False, learning_rate=0.1, gpu_mode=True, graph=None): self.overlap_px = overlap_px self.repeat = repeat self.full_size = 64 * repeat - overlap_px * (repeat - 1) self.unrepeated_num_strokes = num_strokes self.num_strokes = num_strokes * self.repeat**2 self.painter_type = painter_type self.connected = connected self.alternate = alternate self.bw = bw print('full_size', self.full_size, 'max_seq_len', self.num_strokes) self.content = content self.inception_v1 = models.InceptionV1() self.inception_v1.load_graphdef() transforms = [ transform.pad(12, mode='constant', constant_value=.5), transform.jitter(8), transform.random_scale([1 + (i - 5) / 50. for i in range(11)]), transform.random_rotate(list(range(-5, 5)) + 5 * [0]), transform.jitter(4), ] self.transform_f = render.make_transform_f(transforms) self.optim = render.make_optimizer( tf.train.AdamOptimizer(learning_rate), []) self.gpu_mode = gpu_mode if not gpu_mode: with tf.device('/cpu:0'): tf.logging.info('Model using cpu.') self._build_graph(graph) else: #tf.logging.info('Model using gpu.') self._build_graph(graph) self._init_session()
def feature_inversion(img, model, layer, n_steps=512, cossim_pow=0.0, verbose=True): if isinstance(layer, str): layers = [layer] elif isinstance(layer, (tuple, list)): layers = layer else: raise TypeError("layer must be str, tuple or list") with tf.Graph().as_default(), tf.Session() as sess: img = imgToModelSize(img, model) objective = objectives.Objective.sum([ 1.0 * dot_compare(layer, cossim_pow=cossim_pow, batch=i + 1) for i, layer in enumerate(layers) ]) t_input = tf.placeholder(tf.float32, img.shape) param_f = param.image(img.shape[0], decorrelate=True, fft=True, alpha=False, batch=len(layers)) param_f = tf.concat([t_input[None], param_f], 0) transforms = [ transform.pad(8, mode='constant', constant_value=.5), transform.jitter(8), transform.random_scale([0.9, 0.95, 1.05, 1.1] + [1] * 4), transform.random_rotate(list(range(-5, 5)) + [0] * 5), transform.jitter(2), ] T = render.make_vis_T(model, objective, param_f, transforms=transforms) loss, vis_op, t_image = T("loss"), T("vis_op"), T("input") tf.global_variables_initializer().run() for i in range(n_steps): _ = sess.run([vis_op], {t_input: img}) result = t_image.eval(feed_dict={t_input: img}) if verbose: lucid.misc.io.showing.images(result[1:], layers) return result
def get_channel_objective_stimuli(layer, channel): img_size = 224 padding_size = 16 param_f = lambda: param.image(img_size + 2 * padding_size, batch=number_images) objective_per_image = objectives.channel(layer, channel) diversity_loss = -1e2 * objectives.diversity(layer) # transformations as described in Feature Visualization blog post kwargs = dict( thresholds=(2560, ), optimizer=tf.train.AdamOptimizer(learning_rate=0.05), transforms=[ transform.jitter(16), transform.random_scale((1.0, 0.975, 1.025, 0.95, 1.05)), transform.random_rotate((-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5)), transform.jitter(8), ], ) # generate min stimuli _, min_stimuli, min_loss, loss_additional_global_list = render.render_vis( model, -objective_per_image, diversity_loss, param_f, use_fixed_seed=True, **kwargs, ) # the optimization saves multiple states of the results # the last item is the final value min_stimuli = min_stimuli[-1] min_loss = min_loss[-1] min_loss_additional_global = loss_additional_global_list[-1] # undo/crop padding min_stimuli = min_stimuli[:, padding_size:-padding_size, padding_size:-padding_size] # invert loss again min_loss = -min_loss # generate max stimuli _, max_stimuli, max_loss, loss_additional_global_list = render.render_vis( model, objective_per_image, diversity_loss, param_f, use_fixed_seed=True, **kwargs, ) # see above max_stimuli = max_stimuli[-1] max_loss = max_loss[-1] max_loss_additional_global = loss_additional_global_list[-1] # undo/crop padding max_stimuli = max_stimuli[:, padding_size:-padding_size, padding_size:-padding_size] return ( min_stimuli, min_loss, min_loss_additional_global, max_stimuli, max_loss, max_loss_additional_global, )
objective, optimizer=optimizer, transforms=[], param_f=lambda: param.image(64, fft=False, decorrelate=False), thresholds=thresholds, verbose=True) fig([imgs]) JITTER = 1 ROTATE = 5 SCALE = 1.1 transforms = [ transform.pad(2 * JITTER), transform.jitter(JITTER), transform.random_scale([SCALE**(n / 10.) for n in range(-10, 11)]), transform.random_rotate(range(-ROTATE, ROTATE + 1)) ] imgs2 = render.render_vis(model, objective, transforms=transforms, param_f=lambda: param.image(64), thresholds=thresholds, verbose=True) fig([imgs, imgs2]) print('DONE')
def neuron_groups(img_name, layers, model, attr_classes, factorization_methods, flag1, flag_read_attr=False, iter_num=100, SG_path=False, labels=None, pos_flag=1, thres_explained_var=0.7, vis_random_seed=0, image_size=0, debug_flag=0): img = load(img_name) # img = load("./data/doghead224.jpeg") # img = load("./data/cathead224.jpeg") # img = resize(img, (224, 224, 3), order=1, mode='constant', anti_aliasing=False).astype(np.float32) for attr_class in attr_classes: root_directory = create_root_dir(img_name, attr_class, flag1) if flag1 == "Shap": AM_list_L, logit_list, channel_attr_list, kept_channel_list_L \ = compute_shap(img, model, attr_class, layers, flag1, flag_read_attr=flag_read_attr, iter_num=iter_num, labels=labels, save_directory=root_directory) elif flag1 == "IGSG": AM_list_L, logit_list, channel_attr_list, kept_channel_list_L \ = compute_igsg(img, model, attr_class, layers, flag1, flag_read_attr=flag_read_attr, iter_num=iter_num, SG_path=SG_path, labels=labels, save_directory=root_directory) else: continue print_result_from_logit(logit_list, labels) for i_pos_neg in range(pos_flag): AM_list = AM_list_L[i_pos_neg] kept_channel_list = kept_channel_list_L[i_pos_neg] if debug_flag == 1: debug_show_AM_plus_img(AM_list, img, model) if i_pos_neg < 1: reverse_suffix = '_pos' else: reverse_suffix = '_neg' for layer_index, AM in enumerate(AM_list): layer = layers[layer_index] channel_attr = channel_attr_list[layer_index] kept_channel_index = kept_channel_list[layer_index] for factorization_method in factorization_methods: spatial_factors, channel_factors, n_groups = \ decompose_AM_get_group_num(factorization_method, AM, thres_explained_var) # if debug_flag == 1: # decompose_AM_with_UMAP(AM, n_groups) channel_factors_max_index, channel_shap, short_index, long_index, \ n_groups = map_shap_attr_to_long(channel_factors, channel_attr, kept_channel_index) # AM = np.squeeze(AM) spatial_factors = weight_AM2spatial_factor( AM, spatial_factors, n_groups, short_index, kept_channel_index, channel_attr, i_pos_neg) # If the attribution is negative, channel_shap should be multiply -1 if i_pos_neg > 0: channel_shap = channel_shap * -1 # Sorting based on sum of Shapley values ns_sorted = get_sort_groups_with_shap_scores(channel_shap) every_group_attr_sorted, spatial_factors, channel_shap, n_groups =\ sort_groups_features(ns_sorted, spatial_factors, channel_shap, n_groups) no_slash_layer_name = ''.join(layer.split('/')) save_directory = create_factorization_dir( root_directory, factorization_method, no_slash_layer_name, reverse_suffix, n_groups) gen_spatial_heat_maps(85, n_groups, spatial_factors, save_directory, attr_class, factorization_method, no_slash_layer_name, img, AM, model) gen_info_txt(channel_shap, n_groups, save_directory, factorization_method, attr_class, every_group_attr_sorted) # Using feature attributions times activation maps as loss function to update visualization image channel_shap = channel_shap.astype("float32") obj = sum( utils.dot_attr_actmaps(layer, channel_shap[i], batch=i) for i in range(n_groups)) ''' For feature visualization, the library "lucid" will be useful because it has implements many loss functions of different literatures, image processing operators, and collected several pretrained tensorflow network. ''' transforms = [ transform.pad(12), transform.jitter(8), transform.random_scale( [n / 100. for n in range(80, 120)]), transform.random_rotate( list(range(-10, 10)) + list(range(-5, 5)) + 10 * list(range(-2, 2))), transform.jitter(2) ] # image parameterization with shared params for aligned optimizing images def interpolate_f(): unique = fft_image( (n_groups, image_size, image_size, 3), random_seed=vis_random_seed) shared = [ lowres_tensor( (n_groups, image_size, image_size, 3), (1, image_size // 2, image_size // 2, 3), random_seed=vis_random_seed), lowres_tensor( (n_groups, image_size, image_size, 3), (1, image_size // 4, image_size // 4, 3), random_seed=vis_random_seed), lowres_tensor( (n_groups, image_size, image_size, 3), (1, image_size // 8, image_size // 8, 3), random_seed=vis_random_seed), lowres_tensor( (n_groups, image_size, image_size, 3), (2, image_size // 8, image_size // 8, 3), random_seed=vis_random_seed), lowres_tensor( (n_groups, image_size, image_size, 3), (1, image_size // 16, image_size // 16, 3), random_seed=vis_random_seed), lowres_tensor( (n_groups, image_size, image_size, 3), (2, image_size // 16, image_size // 16, 3), random_seed=vis_random_seed) ] return utils.to_valid_rgb(unique + sum(shared), decorrelate=True) group_icons = render.render_vis( model, objective_f=obj, param_f=interpolate_f, transforms=transforms, verbose=False, relu_gradient_override=False)[-1] save_imgs(group_icons, save_directory, attr_class, factorization_method, no_slash_layer_name) print("Layer {} and class {} -> finished".format( layer, attr_class))
def test_render_Inception_v1_slim(): K.set_learning_phase(0) model_path = 'model/tf_inception_v1_slim.pb' if not (os.path.exists(model_path)): with K.get_session().as_default(): model = InceptionV1_slim(include_top=True, weights='imagenet') os.makedirs('./model', exist_ok=True) #model.save('./model/inception_v1_keras_model.h5') frozen_graph = freeze_session( K.get_session(), output_names=[out.op.name for out in model.outputs], clear_devices=True) # Save the pb model tf.io.write_graph(frozen_graph, logdir="model", name="tf_inception_v1_slim.pb", as_text=False) with tf.Graph().as_default() as graph, tf.Session() as sess: # f = gfile.FastGFile("/model/tf_inception_v1.pb", 'rb') # graph_def = tf.GraphDef() # # Parses a serialized binary message into the current message. # graph_def.ParseFromString(f.read()) # f.close() # sess.graph.as_default() # # Import a serialized TensorFlow `GraphDef` protocol buffer # # and place into the current default `Graph`. # tf.import_graph_def(graph_def) # nodes_tab = [n.name for n in tf.get_default_graph().as_graph_def().node] #print(nodes_tab) with gradient_override_map({ 'Relu': redirected_relu_grad, 'ReLU': redirected_relu_grad }): # cela ne semble pas apporter quoique ce soit de particulier lucid_inception_v1 = Lucid_Inception_v1_slim() lucid_inception_v1.load_graphdef() neuron1 = ('mixed4b_pre_relu', 111) # large fluffy C = lambda neuron: objectives.channel(*neuron) out = render.render_vis(lucid_inception_v1, 'Mixed_4b_Concatenated/concat:452',\ relu_gradient_override=True,use_fixed_seed=True) plt.imshow(out[0][0]) JITTER = 1 ROTATE = 5 SCALE = 1.1 transforms = [ transform.pad(2 * JITTER), transform.jitter(JITTER), transform.random_scale([SCALE**(n / 10.) for n in range(-10, 11)]), transform.random_rotate(range(-ROTATE, ROTATE + 1)) ] # https://github.com/tensorflow/lucid/issues/82 with gradient_override_map({ 'Relu': redirected_relu_grad, 'ReLU': redirected_relu_grad }): out = render.render_vis(lucid_inception_v1, "Mixed_4b_Concatenated/concat:452", transforms=transforms, param_f=lambda: param.image(64), thresholds=[2048], verbose=False,\ relu_gradient_override=True,use_fixed_seed=True) # out = render.render_vis(lucid_inception_v1, "Mixed_4d_Branch_2_b_3x3_act/Relu:452", transforms=transforms, # param_f=lambda: param.image(64), # thresholds=[2048], verbose=False) # Cela ne marche pas ! plt.imshow(out[0][0]) out = render.render_vis(lucid_inception_v1, "Mixed_3c_Concatenated/concat:479", transforms=transforms, param_f=lambda: param.image(64), thresholds=[2048], verbose=False,\ relu_gradient_override=True,use_fixed_seed=True) plt.imshow(out[0][0])
def run(self, layer, class_, channel=None, style_template=None, transforms=False, opt_steps=1000): """ layer : layer_name to visualize class_ : class to consider style_template: template for comparision of generated activation maximization map transforms : transforms required opt_steps : number of optimization steps """ self.layer = layer self.channel = channel if channel is not None else 0 with tf.Graph().as_default() as graph, tf.Session() as sess: if style_template is not None: gram_template = tf.constant( np.load(style_template), #[1:-1,:,:], dtype=tf.float32) print('Gram Shape = {}'.format(gram_template.shape)) obj = self._channel(self.layer + "/convolution", self.channel, gram=style_template) obj += -self.L1 * objectives.L1(constant=.5) obj += self.TV * objectives.total_variation() obj += self.blur * objectives.blur_input_each_step() if transforms == True: transforms = [ transform.pad(2 * self.jitter), transform.jitter(self.jitter), transform.random_scale( [self.scale**(n / 10.) for n in range(-10, 11)]), transform.random_rotate( range(-self.rotate, self.rotate + 1)) ] else: transforms = [] T = render.make_vis_T( self.model, obj, param_f=lambda: self.image(240, channels=self.n_channels, fft=self.decorrelate, decorrelate=self.decorrelate), optimizer=None, transforms=transforms, relu_gradient_override=False) tf.initialize_all_variables().run() for i in range(opt_steps): T("vis_op").run() plt.figure(figsize=(10, 10)) texture_images = [] for i in range(1, self.n_channels + 1): plt.subplot(1, self.n_channels, i) image = T("input").eval()[:, :, :, i - 1].reshape((240, 240)) print("channel: ", i, image.min(), image.max()) plt.imshow(image, cmap='gray', interpolation='bilinear', vmin=0., vmax=1.) plt.xticks([]) plt.yticks([]) texture_images.append(image) # show(np.hstack(T("input").eval())) if class_ is not None: os.makedirs(os.path.join(self.savepath, class_), exist_ok=True) plt.savefig(os.path.join( self.savepath, class_, self.layer + '_' + str(self.channel) + '.png'), bbox_inches='tight') else: plt.savefig(os.path.join( self.savepath, self.layer + '_' + str(self.channel) + '.png'), bbox_inches='tight') return np.array(texture_images).transpose(1, 2, 0)
def test_render_ResNet50(): tf.reset_default_graph() K.set_learning_phase(0) if not (os.path.isfile("model/tf_resnet50.pb")): with K.get_session().as_default(): model = tf.keras.applications.resnet50.ResNet50(include_top=True, weights='imagenet',\ input_shape= (224, 224, 3)) print(model.input) os.makedirs('./model', exist_ok=True) frozen_graph = freeze_session( K.get_session(), output_names=[out.op.name for out in model.outputs], clear_devices=True) # Save the pb model tf.io.write_graph(frozen_graph, logdir="model", name="tf_resnet50.pb", as_text=False) nodes_tab = [ n.name for n in tf.get_default_graph().as_graph_def().node ] print(nodes_tab) lucid_resnet50 = Lucid_ResNet() lucid_resnet50.load_graphdef() out = render.render_vis(lucid_resnet50, 'conv4_block6_2_conv/Conv2D:0',\ relu_gradient_override=True,use_fixed_seed=True) plt.figure() plt.imshow(out[0][0]) out = render.render_vis(lucid_resnet50, 'conv2_block1_2_conv/Conv2D:32',\ relu_gradient_override=True,use_fixed_seed=True) plt.figure() plt.imshow(out[0][0]) JITTER = 1 ROTATE = 5 SCALE = 1.1 transforms = [ transform.pad(2 * JITTER), transform.jitter(JITTER), transform.random_scale([SCALE**(n / 10.) for n in range(-10, 11)]), transform.random_rotate(range(-ROTATE, ROTATE + 1)) ] imgs = render.render_vis(lucid_resnet50, 'conv4_block4_2_conv/Conv2D:0', transforms=transforms, param_f=lambda: param.image(64), thresholds=[2048], verbose=False, relu_gradient_override=True, use_fixed_seed=True) plt.figure() plt.imshow(imgs[0][0]) input('wait') plt.close()
def test_autocorr_render_Inception_v1(): tf.reset_default_graph() K.set_learning_phase(0) if not (os.path.isfile("model/tf_inception_v1.pb")): with K.get_session().as_default(): model = inception_v1_oldTF( weights='imagenet', include_top=True) #include_top=True, weights='imagenet') print(model.input) os.makedirs('./model', exist_ok=True) frozen_graph = freeze_session( K.get_session(), output_names=[out.op.name for out in model.outputs], clear_devices=True) # Save the pb model tf.io.write_graph(frozen_graph, logdir="model", name="tf_inception_v1.pb", as_text=False) nodes_tab = [ n.name for n in tf.get_default_graph().as_graph_def().node ] print(nodes_tab) #with tf.Graph().as_default() as graph, tf.Session() as sess: with gradient_override_map({ 'Relu': redirected_relu_grad, 'ReLU': redirected_relu_grad }): lucid_inception_v1 = Lucid_InceptionV1() lucid_inception_v1.load_graphdef() out = render.render_vis(lucid_inception_v1, 'mixed4a_1x1_pre_relu/Conv2D:0',\ relu_gradient_override=True,use_fixed_seed=True) plt.figure() plt.imshow(out[0][0]) out = render.render_vis(lucid_inception_v1, 'mixed4b_pre_relu/concat:452',\ relu_gradient_override=True,use_fixed_seed=True) plt.figure() plt.imshow(out[0][0]) JITTER = 1 ROTATE = 5 SCALE = 1.1 transforms = [ transform.pad(2 * JITTER), transform.jitter(JITTER), transform.random_scale([SCALE**(n / 10.) for n in range(-10, 11)]), transform.random_rotate(range(-ROTATE, ROTATE + 1)) ] imgs = render.render_vis(lucid_inception_v1, 'mixed4b_pre_relu/concat:452', transforms=transforms, param_f=lambda: param.image(64), thresholds=[2048], verbose=False, relu_gradient_override=True, use_fixed_seed=True) plt.figure() plt.imshow(imgs[0][0])
def get_neuron_objective_stimuli(layer, channel): def get_crop_idxs(layer_str, channel): filtered_df = neuron_rf_sizes.loc[ (neuron_rf_sizes["layer_name"] + "_" + neuron_rf_sizes["pre_post_relu"] == layer_str) & (neuron_rf_sizes["feature_map_number"] == channel)] return int(filtered_df["min_idx"].iloc[0]), int( filtered_df["max_idx"].iloc[0]) img_size = 224 padding_size = 16 # avoid edge artifacts as described in Feature Visualization blog param_f = lambda: param.image(img_size + 2 * padding_size, batch=number_images) objective_per_image = objectives.neuron(layer, channel) diversity_loss = -1e2 * objectives.diversity(layer) # transformations as described in Feature Visualization blog post kwargs = dict( thresholds=(2560, ), optimizer=tf.train.AdamOptimizer(learning_rate=0.05), transforms=[ transform.jitter(16), transform.random_scale((1.0, 0.975, 1.025, 0.95, 1.05)), transform.random_rotate((-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5)), transform.jitter(8), ], ) # generate min stimuli _, min_stimuli, min_loss, loss_additional_global_list = render.render_vis( model, -objective_per_image, diversity_loss, param_f, use_fixed_seed=True, **kwargs, ) # the optimization saves multiple states of the results # the last item is the final value min_stimuli = min_stimuli[-1] min_loss = min_loss[-1] min_loss_additional_global = loss_additional_global_list[-1] # undo/crop padding as described in Feature Visualization blog min_stimuli = min_stimuli[:, padding_size:-padding_size, padding_size:-padding_size] # invert min_loss again min_loss = -min_loss # generate max stimuli _, max_stimuli, max_loss, loss_additional_global_list = render.render_vis( model, objective_per_image, diversity_loss, param_f, use_fixed_seed=True, **kwargs, ) # see above max_stimuli = max_stimuli[-1] max_loss = max_loss[-1] max_loss_additional_global = loss_additional_global_list[-1] # undo/crop padding max_stimuli = max_stimuli[:, padding_size:-padding_size, padding_size:-padding_size] # select crop for neuron objective min_idx, max_idx = get_crop_idxs(layer, channel) min_stimuli = min_stimuli[:, min_idx:max_idx + 1, min_idx:max_idx + 1] max_stimuli = max_stimuli[:, min_idx:max_idx + 1, min_idx:max_idx + 1] return ( min_stimuli, min_loss, min_loss_additional_global, max_stimuli, max_loss, max_loss_additional_global, )
def render_facet(model, neuron_obj, layers, style_attrs, strength=(0.1, 0.3), l2_weight=10.0, resolution=128, alpha=False): def mean_alpha(): def inner(T): input_t = T("input") return tf.sqrt(tf.reduce_mean(input_t[..., 3:]**2)) return objectives.Objective(inner) standard_transforms = [ transform.pad(2, mode='constant', constant_value=.5), transform.jitter(4), transform.jitter(4), transform.jitter(4), transform.jitter(4), transform.jitter(4), transform.jitter(4), transform.jitter(4), transform.jitter(4), transform.jitter(4), transform.jitter(4), transform.random_scale([0.995**n for n in range(-5, 80)] + [0.998**n for n in 2 * list(range(20, 40))]), transform.random_rotate( list(range(-20, 20)) + list(range(-10, 10)) + list(range(-5, 5)) + 5 * [0]), transform.jitter(2), transform.crop_or_pad_to(resolution, resolution) ] if alpha: standard_transforms.append(transform.collapse_alpha_random()) param_f = lambda: param.image(resolution, batch=9, alpha=True) else: param_f = lambda: param.image(resolution, batch=9) optimizer = tf.train.AdamOptimizer(0.02) ultimate_layer = [ n.name for n in model.graph_def.node if "image_block_4" in n.name ][-1] obj = vector(ultimate_layer, neuron_obj) facetsp = [(5 / len(layers)) * attr(obj, style, [layer], strength) for style, layer in list(zip(style_attrs, layers))] for facetp in facetsp: obj = obj + facetp obj = obj + l2_weight * l2() if alpha: obj -= mean_alpha() obj -= 1e2 * objectives.blur_alpha_each_step() data = render.render_vis(model, obj, param_f, transforms=standard_transforms, optimizer=optimizer, thresholds=(1024 * 4, )) return data
def render_icons( directions, model, layer, size=80, n_steps=128, verbose=False, S=None, num_attempts=3, cossim=True, alpha=False, ): model.load_graphdef() image_attempts = [] loss_attempts = [] depth = 4 if alpha else 3 batch = len(directions) input_shape = (batch, size, size, depth) # Render two attempts, and pull the one with the lowest loss score. for attempt in range(num_attempts): # Render an image for each activation vector param_f = lambda: param.image(size, batch=len(directions), fft=True, decorrelate=True, alpha=alpha) if cossim is True: obj_list = [ direction_neuron_cossim_S(layer, v, batch=n, S=S) for n, v in enumerate(directions) ] else: obj_list = [ direction_neuron_S(layer, v, batch=n, S=S) for n, v in enumerate(directions) ] obj_list += [objectives.penalize_boundary_complexity(input_shape, w=5)] obj = objectives.Objective.sum(obj_list) # holy mother of transforms transforms = [ transform.pad(16, mode='constant'), transform.jitter(4), transform.jitter(4), transform.jitter(8), transform.jitter(8), transform.jitter(8), transform.random_scale(0.998**n for n in range(20, 40)), transform.random_rotate( chain(range(-20, 20), range(-10, 10), range(-5, 5), 5 * [0])), transform.jitter(2), transform.crop_or_pad_to(size, size) ] if alpha: transforms.append(transform.collapse_alpha_random()) # This is the tensorflow optimization process # print("attempt: ", attempt) with tf.Graph().as_default(), tf.Session() as sess: learning_rate = 0.05 losses = [] trainer = tf.train.AdamOptimizer(learning_rate) T = render.make_vis_T(model, obj, param_f, trainer, transforms) vis_op, t_image = T("vis_op"), T("input") losses_ = [obj_part(T) for obj_part in obj_list] tf.global_variables_initializer().run() for i in range(n_steps): loss, _ = sess.run([losses_, vis_op]) losses.append(loss) # if i % 100 == 0: # print(i) img = t_image.eval() img_rgb = img[:, :, :, :3] if alpha: # print("alpha true") k = 0.8 bg_color = 0.0 img_a = img[:, :, :, 3:] img_merged = img_rgb * ( (1 - k) + k * img_a) + bg_color * k * (1 - img_a) image_attempts.append(img_merged) else: # print("alpha false") image_attempts.append(img_rgb) loss_attempts.append(losses[-1]) # Use only the icons with the lowest loss loss_attempts = np.asarray(loss_attempts) loss_final = [] image_final = [] # print("merging best scores from attempts...") for i, d in enumerate(directions): # note, this should be max, it is not a traditional loss mi = np.argmax(loss_attempts[:, i]) image_final.append(image_attempts[mi][i]) return (image_final, loss_final)
def test_render_VGG19(): #with tf.Graph().as_default() as graph, tf.Session() as sess: K.set_learning_phase(0) with K.get_session().as_default(): #images = tf.placeholder("float32", [None, 224, 224, 3], name="input") # <Code to construct & load your model inference graph goes here> model = tf.keras.applications.vgg19.VGG19(include_top=False, weights='imagenet', input_shape=(224, 224, 3)) # ! il va falloir ajouter des noeuds / node pre_relu ! os.makedirs('./model', exist_ok=True) #model.save('./model/keras_model.h5') frozen_graph = freeze_session( K.get_session(), output_names=[out.op.name for out in model.outputs]) # Show current session graph with TensorBoard in Jupyter Notebook. #show_graph(tf.get_default_graph().as_graph_def()) tf.io.write_graph(frozen_graph, logdir="model", name="tf_vgg19.pb", as_text=False) nodes_tab = [ n.name for n in tf.get_default_graph().as_graph_def().node ] # base_Model_instance = base_Model() # base_Model_instance.suggest_save_args() with tf.Graph().as_default() as graph, tf.Session() as sess: lucid_vgg = Lucid_VGGNet() lucid_vgg.load_graphdef() # for node in lucid_vgg.graph_def.node: # if 'conv' in node.op: # print(node.name) #Model.suggest_save_args() #lucid_model = Model.load_graphdef("tf_model.pb") LEARNING_RATE = 0.05 # Valeur par default optimizer = tf.train.AdamOptimizer(LEARNING_RATE) output_im = render.render_vis(lucid_vgg, "block1_conv1/Conv2D:0", use_fixed_seed=0) # Il semblerait que par default cela renvoit un crop de l image genere en 224*224 de taille 128*128 plt.imshow(output_im[0][0]) print(np.max(output_im), np.min(output_im)) output_im = render.render_vis(lucid_vgg, "block1_conv1/BiasAdd:1", use_fixed_seed=0) plt.imshow(output_im[0][0]) output_im = render.render_vis(lucid_vgg, "block1_conv1/Relu:0", use_fixed_seed=True) plt.imshow(output_im[0][0]) output_im = render.render_vis(lucid_vgg, "block1_conv1/Relu:1", use_fixed_seed=True) plt.imshow(output_im[0][0]) output_im = render.render_vis(lucid_vgg, "block1_conv1/Relu:2", use_fixed_seed=True) plt.imshow(output_im[0][0]) output_im = render.render_vis(lucid_vgg, "block1_conv1/Relu:3", use_fixed_seed=True) plt.imshow(output_im[0][0]) output_im = render.render_vis(lucid_vgg, "block5_conv4/Relu:0") plt.imshow(output_im[0][0]) # <IPython.core.display.HTML object> only plot in jupyter Notebook param_f = lambda: param.image(128) output_im = render.render_vis(lucid_vgg, "block5_conv4/BiasAdd:100", param_f, thresholds=[256]) plt.imshow(output_im[0][0]) output_im = render.render_vis(lucid_vgg, "block5_conv4/Relu:100", param_f, thresholds=[256]) plt.imshow(output_im[0][0]) # Using alternate parameterizations is one of the primary ingredients for # effective visualization param_f = lambda: param.image(224, fft=True, decorrelate=True) output_im = render.render_vis(lucid_vgg, "block1_conv1/Relu:0", param_f) plt.imshow(output_im[0][0]) output_im = render.render_vis(lucid_vgg, "block5_conv4/Relu:0", param_f) plt.imshow(output_im[0][0]) JITTER = 1 ROTATE = 5 SCALE = 1.1 transforms = [ transform.pad(2 * JITTER), transform.jitter(JITTER), transform.random_scale([SCALE**(n / 10.) for n in range(-10, 11)]), transform.random_rotate(range(-ROTATE, ROTATE + 1)) ] image_full_tricks = render.render_vis( lucid_vgg, "block5_conv4/Relu:0", transforms=transforms, param_f=lambda: param.image(64, fft=True, decorrelate=True), optimizer=optimizer, thresholds=[2048], verbose=False) plt.imshow(image_full_tricks[0][0]) image_full_tricks = render.render_vis( lucid_vgg, "block3_conv1/Relu:0", transforms=transforms, param_f=lambda: param.image(64, fft=True, decorrelate=True), optimizer=optimizer, thresholds=[2048], verbose=False) plt.imshow(image_full_tricks[0][0]) image_full_tricks = render.render_vis( lucid_vgg, "block3_conv1/Relu:1", transforms=transforms, param_f=lambda: param.image(64, fft=False, decorrelate=False), optimizer=optimizer, thresholds=[2048], verbose=False) plt.imshow(image_full_tricks[0][0])