def main(unused_args): model = vision_models.ResnetV1_50_slim() model.load_graphdef() print('------------------------------') print('Loaded Parameters') print('------------------------------') print('Chosen Class: {}'.format(FLAGS.chosen_class)) print('Hue : {}'.format(FLAGS.hue_hex)) print('No. of Steps: {}'.format(FLAGS.steps)) print('Output Size : {}'.format(FLAGS.output_size)) print('Output Path : {}'.format(FLAGS.output_path)) print('------------------------------') render_vis( model=model, objective_f=objectives.channel( 'resnet_v1_50/SpatialSqueeze', FLAGS.chosen_class, ), param_f=lambda: cppn(), optimizer=tf.train.AdamOptimizer(5e-3), transforms=[], steps=FLAGS.steps, output_size=FLAGS.output_size, output_path=FLAGS.output_path, ) classify(FLAGS.output_path)
def visualize_neuron(algo='apex', env='SeaquestNoFrameskip-v4', run_id=1, tag="final", param_f=lambda: image([1, 84, 84, 4]), do_render=False, transforms=[ transform.jitter(3), ], layer_no=0, neuron=0, regularization=0, **params): tf.reset_default_graph() m = MakeAtariModel(algo, env, run_id, tag, local=False)() m.load_graphdef() if (m.layers[layer_no]['type'] == 'dense'): obj = objectives.channel(m.layers[layer_no]['name'], neuron) else: obj = channel(m.layers[layer_no]['name'], neuron, ordering=m.channel_order) out = optimize_input(obj + regularization, m, param_f, transforms, do_render=do_render, **params) return out
def __call__(self, layer_name, channel_index): obj = objectives.channel(layer_name, channel_index) image = render.render_vis(self.model, obj, param_f=self.param_f, thresholds=[self.threshold], use_fixed_seed=True, verbose=False) return np.array(image[0][0])
def start(self): self.image = None self._doRun(True) obj = objectives.channel(self.layer_id, self.unit) self.image = render.render_vis(self.model, obj) #self.image = render.render_vis(self.model, self.unit_id) self._doRun(False)
def render_feature(cppn_f=lambda: image_cppn(84), optimizer=tf.train.AdamOptimizer(0.001), objective=objectives.channel('noname', 0), transforms=[]): vis = render.render_vis(m, objective, param_f=cppn_f, optimizer=optimizer, transforms=transforms, thresholds=[2**i for i in range(5, 10)], verbose=False) #show(vis) return vis
def get_lucid_orig_image_losses(args): #import lucid.modelzoo.vision_models as models #model = models.InceptionV1() #model.load_graphdef() #param_f = lambda: param.image(128, batch=4) #obj = objectives.channel("mixed5a", 9) - 1e2*objectives.diversity("mixed5a") #obj = objectives.channel("mixed5a", 9) #all_images = render.render_vis(model, obj, param_f) #image = all_images[0] global USE_ORG_IMPORT_MODEL USE_ORG_IMPORT_MODEL = True import lucid.modelzoo.vision_models as models model = models.InceptionV1() model.load_graphdef() layer = "mixed5a" num_units = 16 param_f = lambda: param.image(224, batch=num_units) obj = objectives.channel(layer, 0, 0) for idx in range(1, num_units): obj += objectives.channel(layer, idx, idx) image, all_losses = render_vis(model, obj, param_f) return layer, image, all_losses
def start_multi(self): self.image = None self._doRun(True) logger.info("!!! running all:") for unit in range(self.layer_units): self.unit = unit self.notify_observers(EngineChange(unit_changed=True)) logger.info(f"!!! running unit {unit}") obj = objectives.channel(self.layer_id, unit) self.image = render.render_vis(self.model, obj) if not self.running: break self._doRun(True) self._doRun(False)
def render_set(n, channel): print("Starting", channel, n) obj = objectives.channel(channel, n) # Add this to "sharpen" the image... too much and it gets crazy #obj += 0.001*objectives.total_variation() sess = create_session() t_size = tf.placeholder_with_default(size_n, []) f_model = os.path.join(save_model_dest, channel + f"_{n}.npy") T = render.make_vis_T( model, obj, param_f=lambda: cppn(t_size), transforms=[], optimizer=optimizer, ) tf.global_variables_initializer().run() train_vars = sess.graph.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) if not os.path.exists(f_model): for i in tqdm(range(training_steps)): _, loss = sess.run([ T("vis_op"), T("loss"), ]) # Save trained variables params = np.array(sess.run(train_vars), object) save(params, f_model) else: params = load(f_model) # Save final image feed_dict = dict(zip(train_vars, params)) feed_dict[t_size] = image_size images = T("input").eval(feed_dict) img = images[0] sess.close() f_image = os.path.join(save_image_dest, channel + f"_{n}.jpg") imageio.imwrite(f_image, img) print(f"Saved to {f_image}")
def render_set(n, channel, train_n): # Creates independent images param_f = lambda : create_network(batch_size) obj = sum( objectives.channel(channel, n, batch=i) for i in range(batch_size) ) # This gives some visual similarity to the models #obj += 10*objectives.input_diff(target_img) # This does as well but not as nice #obj += 0.01*objectives.alignment(channel, decay_ratio=3) # This gives some visual similarity to the models #obj += 10*objectives.input_diff(target_img) # See more here # https://colab.research.google.com/github/tensorflow/lucid/blob/master/notebooks/differentiable-parameterizations/aligned_interpolation.ipynb#scrollTo=jOCYDhRrnPjp T = render.make_vis_T( model, obj, param_f=param_f, transforms=[], optimizer=optimizer, ) saver = tf.train.Saver() tf.global_variables_initializer().run() for i in tqdm(range(train_n)): _, loss = sess.run([T("vis_op"), T("loss"), ]) # Save trained variables f_model = os.path.join(save_model_dest, channel + f"_{n}_batches_{batch_size}.ckpt") save_path = saver.save(sess, f_model) # Return image images = T("input").eval({t_size: 600}) return images
def test_integration_any_channels(): inceptionv1 = InceptionV1() objectives_f = [ objectives.deepdream("mixed4a_pre_relu"), objectives.channel("mixed4a_pre_relu", 360), objectives.neuron("mixed3a", 177) ] params_f = [ lambda: param.grayscale_image_rgb(128), lambda: arbitrary_channels_to_rgb(128, channels=10) ] for objective_f in objectives_f: for param_f in params_f: rendering = render.render_vis( inceptionv1, objective_f, param_f, verbose=False, thresholds=(0, 64), use_fixed_seed=True, ) start_image, end_image = rendering assert (start_image != end_image).any()
print('loading model') model = models.InceptionV1() model.load_graphdef() print('calculating') neuron = ("mixed4a_pre_relu", 476) version = 7 size = 64 #resulting image cube haa dimensions size X size X size X 3 def param_f3d(size): temp = imageCube(size) return tf.concat([ temp, tf.transpose(temp, [1, 0, 2, 3]), tf.transpose(temp, [2, 1, 0, 3]) ], 0) objective = objectives.channel(*neuron) image_cube = render.render_vis( model, objective, lambda: param_f3d(size), transforms=transform.standard_transforms, thresholds=(512, )) # threshold number of steps. #I used 4096 image_cube = np.array(image_cube)[:, :size] #image cube np.save(f"featureCube{size}_{version}.npy", image_cube)
# Save trained variables train_vars = sess.graph.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) params = np.array(sess.run(train_vars), object) save(params, f_model) # Save final image images = T("input").eval({t_size: 600}) img = images[0] sess.close() imsave(f_image, img) C = [objectives.channel(channel, cn) for cn in CN] #C0 = objectives.channel(channel, cn0) #C1 = objectives.channel(channel, cn1) # Render the fixed points for kn, obj in enumerate(C): render_set(channel, starting_training_steps, f'A{kn}', objective=obj) #render_set(channel, starting_training_steps, 'A0', objective=C0) #render_set(channel, starting_training_steps, 'A1', objective=C1) MODELS = [ load(os.path.join(save_model_dest, channel + f"_A{kn}.npy")) for kn in range(len(C))
def _get_objective(self, layer, index=0): return objectives.channel(layer, index)
import lucid.modelzoo.vision_models as models from lucid.misc.io import show import lucid.optvis.objectives as objectives import lucid.optvis.param as param import lucid.optvis.render as render import lucid.optvis.transform as transform #import lucid.optvis.optimizer as transfor import tensorflow as tf import matplotlib.pyplot as plt model = models.InceptionV1() model.load_graphdef() neuron1 = ('mixed4b_pre_relu', 452) C = lambda neuron: objectives.channel(*neuron) out = render.render_vis(model, C(neuron1)) plt.imshow(out[0][0]) JITTER = 1 ROTATE = 5 SCALE = 1.1 transforms = [ transform.pad(2 * JITTER), transform.jitter(JITTER), transform.random_scale([SCALE**(n / 10.) for n in range(-10, 11)]), transform.random_rotate(range(-ROTATE, ROTATE + 1)) ]
def test_render_Inception_v1_slim(): K.set_learning_phase(0) model_path = 'model/tf_inception_v1_slim.pb' if not (os.path.exists(model_path)): with K.get_session().as_default(): model = InceptionV1_slim(include_top=True, weights='imagenet') os.makedirs('./model', exist_ok=True) #model.save('./model/inception_v1_keras_model.h5') frozen_graph = freeze_session( K.get_session(), output_names=[out.op.name for out in model.outputs], clear_devices=True) # Save the pb model tf.io.write_graph(frozen_graph, logdir="model", name="tf_inception_v1_slim.pb", as_text=False) with tf.Graph().as_default() as graph, tf.Session() as sess: # f = gfile.FastGFile("/model/tf_inception_v1.pb", 'rb') # graph_def = tf.GraphDef() # # Parses a serialized binary message into the current message. # graph_def.ParseFromString(f.read()) # f.close() # sess.graph.as_default() # # Import a serialized TensorFlow `GraphDef` protocol buffer # # and place into the current default `Graph`. # tf.import_graph_def(graph_def) # nodes_tab = [n.name for n in tf.get_default_graph().as_graph_def().node] #print(nodes_tab) with gradient_override_map({ 'Relu': redirected_relu_grad, 'ReLU': redirected_relu_grad }): # cela ne semble pas apporter quoique ce soit de particulier lucid_inception_v1 = Lucid_Inception_v1_slim() lucid_inception_v1.load_graphdef() neuron1 = ('mixed4b_pre_relu', 111) # large fluffy C = lambda neuron: objectives.channel(*neuron) out = render.render_vis(lucid_inception_v1, 'Mixed_4b_Concatenated/concat:452',\ relu_gradient_override=True,use_fixed_seed=True) plt.imshow(out[0][0]) JITTER = 1 ROTATE = 5 SCALE = 1.1 transforms = [ transform.pad(2 * JITTER), transform.jitter(JITTER), transform.random_scale([SCALE**(n / 10.) for n in range(-10, 11)]), transform.random_rotate(range(-ROTATE, ROTATE + 1)) ] # https://github.com/tensorflow/lucid/issues/82 with gradient_override_map({ 'Relu': redirected_relu_grad, 'ReLU': redirected_relu_grad }): out = render.render_vis(lucid_inception_v1, "Mixed_4b_Concatenated/concat:452", transforms=transforms, param_f=lambda: param.image(64), thresholds=[2048], verbose=False,\ relu_gradient_override=True,use_fixed_seed=True) # out = render.render_vis(lucid_inception_v1, "Mixed_4d_Branch_2_b_3x3_act/Relu:452", transforms=transforms, # param_f=lambda: param.image(64), # thresholds=[2048], verbose=False) # Cela ne marche pas ! plt.imshow(out[0][0]) out = render.render_vis(lucid_inception_v1, "Mixed_3c_Concatenated/concat:479", transforms=transforms, param_f=lambda: param.image(64), thresholds=[2048], verbose=False,\ relu_gradient_override=True,use_fixed_seed=True) plt.imshow(out[0][0])
def print_PCA_images(model_path,layer_to_print,weights,index_features_withinLayer,\ path_output='',prexif_name='',\ input_name='block1_conv1_input',Net='VGG',sizeIm=256,\ DECORRELATE=True,ROBUSTNESS=True,\ inverseAndSave=True,cossim=False,dot_vector = False, num_features=None): # ,printOnlyRGB=True if not (os.path.isfile(os.path.join(model_path))): raise (ValueError(model_path + ' does not exist !')) if Net == 'VGG': lucid_net = Lucid_VGGNet(model_path=model_path, input_name=input_name) elif Net == 'InceptionV1': lucid_net = Lucid_InceptionV1(model_path=model_path, input_name=input_name) elif Net == 'InceptionV1_slim': lucid_net = Lucid_Inception_v1_slim(model_path=model_path, input_name=input_name) elif Net == 'ResNet': lucid_net = Lucid_ResNet(model_path=model_path, input_name=input_name) else: raise (ValueError(Net + 'is unkonwn')) lucid_net.load_graphdef() #nodes_tab = [n.name for n in tf.compat.v1.get_default_graph().as_graph_def().node] obj_str, kind_layer = get_obj_and_kind_layer(layer_to_print, Net) param_f = lambda: param.image( sizeIm, fft=DECORRELATE, decorrelate=DECORRELATE) if DECORRELATE: ext = '_Deco' else: ext = '' if ROBUSTNESS: transforms = transform.standard_transforms ext += '' else: transforms = [] ext += '_noRob' # LEARNING_RATE = 0.005 # Valeur par default # optimizer = tf.train.AdamOptimizer(LEARNING_RATE) if Net == 'VGG': name_base = layer_to_print + kind_layer + '_' + prexif_name + ext + '.png' elif Net == 'InceptionV1': name_base = layer_to_print + kind_layer + '_' + prexif_name + ext + '.png' elif Net == 'InceptionV1_slim': name_base = layer_to_print + kind_layer + '_' + prexif_name + ext + '.png' else: raise (NotImplementedError) # input = couple of Two arguments : first one name of the layer, # second one number of the features must be an integer if (cossim is True): raise (NotImplementedError) # Pas fini ici il faut le S # cette fonction objectif est le produit entre le produit scalaire de v et des couches ainsi que # multiplier par le cosinus a une certaine valeur obj_list = ([ direction_neuron_cossim_S(layer, v, batch=n, S=S, cossim_pow=4) for n, v in enumerate(directions) ]) total_obj = objectives.Objective.sum(obj_list) elif dot_vector: assert (not (num_features is None)) def inner(T): layer = T(obj_str) #print('num_features',num_features) if len(weights) == num_features: total_obj = tf.reduce_mean(layer * weights) else: weights_total = np.zeros((num_features, )) weights_total[index_features_withinLayer] = weights total_obj = tf.reduce_mean(layer * weights_total) return (total_obj) total_obj = inner else: C = lambda layer_i: objectives.channel(*layer_i) total_obj = None for i, weight_i in zip(index_features_withinLayer, weights): if total_obj is None: total_obj = weight_i * C((obj_str, i)) else: total_obj += weight_i * C((obj_str, i)) output_im = render.render_vis( lucid_net, total_obj, transforms=transforms, thresholds=[2048], param_f=param_f, # optimizer=optimizer, use_fixed_seed=True) image = np.array(output_im[0][0] * 255) # car une seule image dans le batch name_output = os.path.join(path_output, name_base) new_output_path = os.path.join(path_output, 'RGB') if inverseAndSave: name_output = name_output.replace('.png', '_toRGB.png') image = image[:, :, [2, 1, 0]] tf.keras.preprocessing.image.save_img(name_output, image) else: tf.keras.preprocessing.image.save_img(name_output, image) pathlib.Path(new_output_path).mkdir(parents=True, exist_ok=True) change_from_BRG_to_RGB(img_name_path=name_output, output_path=new_output_path, ext_name='toRGB')
input_name = 'input' def show_image(image): html = "" data_url = _image_url(image) html += '<img width=\"100\" style=\"margin: 10px\" src=\"' + data_url + '\">' with open("img.html", "w") as f: f.write(html) _display_html(html) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--crop_size', type=int, default=128) parser.add_argument('--model_file', type=str, default='lucid_model.pb') args = parser.parse_args() model = LucidModel() model.model_path = args.model_file model.image_shape = [args.crop_size, args.crop_size, 3] print("Nodes in graph:") for node in model.graph_def.node: print(node.name) print("=" * 30) obj = objectives.channel("prediction/Conv2D", 0) - objectives.channel("prediction/Conv2D", 0) res = render.render_vis(model, obj, transforms=[]) show_image(res)
import lucid.optvis.param as param from lucid.modelzoo.vision_base import Model class FrozenNetwork(Model): model_path = network_protobuf_path image_shape = [256, 256, 3] image_value_range = (0, 1) input_name = 'input_1' network = FrozenNetwork() network.load_graphdef() obj = objectives.channel(layer_name, neuron_index) param_f = lambda: param.image(512, fft=True, decorrelate=True) renders = render.render_vis(network, obj, param_f, thresholds=(2024, )) last_image_file = sorted(glob.glob("projection/out/*step*.png"))[-1] stylegan_render = imageio.imread(last_image_file) lucid_render = renders[0][0] lucid_render = (np.clip(lucid_render, 0, 1) * 255).astype(np.uint8) h, w = lucid_render.shape[:2] canvas = PIL.Image.new('RGB', (w * 2, h), 'white') canvas.paste(Image.fromarray(lucid_render), (0, 0)) canvas.paste( Image.fromarray(stylegan_render).resize((w, h), PIL.Image.LANCZOS), (w, 0)) canvas.save("projection/combined_%s_%03d.png" % (layer_name.split("/")[0], neuron_index))
def make_lucid_imagenet_dataset( model_tag, n_random=9, min_size=3, max_prop=0.8, display=True, infodir='/project/clusterability_in_neural_networks/results/', savedir='/project/clusterability_in_neural_networks/datasets/'): assert model_tag in VIS_NETS with open(infodir + model_tag + '_clustering_info.pkl', 'rb') as f: clustering_info = pickle.load(f) layer_names = clustering_info['layers'] labels_in_layers = [ np.array(lyr_labels) for lyr_labels in clustering_info['labels'] ] layer_sizes = [len(labels) for labels in labels_in_layers] n_clusters = max([max(labels) for labels in labels_in_layers]) + 1 if model_tag == 'vgg16': lucid_net = models.VGG16_caffe() elif model_tag == 'vgg19': lucid_net = models.VGG19_caffe() else: lucid_net = models.ResnetV1_50_slim() lucid_net.load_graphdef() layer_map = NETWORK_LAYER_MAP[model_tag] max_images = [ ] # to be filled with images that maximize cluster activations # min_images = [] # to be filled with images that minimize cluster activations random_max_images = [ ] # to be filled with images that maximize random units activations # random_min_images = [] # to be filled with images that minimize random units activations max_losses = [] # to be filled with losses # min_losses = [] # to be filled with losses random_max_losses = [] # to be filled with losses # random_min_losses = [] # to be filled with losses sm_sizes = [] # list of submodule sizes sm_layer_sizes = [] sm_layers = [] # list of layer names sm_clusters = [] # list of clusters for layer_name, labels, layer_size in zip(layer_names, labels_in_layers, layer_sizes): if layer_name not in layer_map.keys(): continue lucid_name = layer_map[layer_name] max_size = max_prop * layer_size for clust_i in range(n_clusters): sm_binary = labels == clust_i sm_size = sum(sm_binary) if sm_size <= min_size or sm_size >= max_size: # skip if too big or small continue sm_sizes.append(sm_size) sm_layer_sizes.append(layer_size) sm_layers.append(layer_name) sm_clusters.append(clust_i) print(f'{model_tag}, layer names: {layer_name}, {lucid_name}') print(f'submodule_size: {sm_size}, layer_size: {layer_size}') sm_idxs = [i for i in range(layer_size) if sm_binary[i]] max_obj = sum( [objectives.channel(lucid_name, unit) for unit in sm_idxs]) # min_obj = -1 * sum([objectives.channel(lucid_name, unit) for unit in sm_idxs]) max_im, max_loss = render_vis_with_loss(lucid_net, max_obj, size=IMAGE_SIZE_IMAGENET, thresholds=(256, )) max_images.append(max_im) max_losses.append(max_loss) # min_im, min_loss = render_vis_with_loss(lucid_net, min_obj) # min_images.append(min_im) # min_losses.append(min_loss) if display: print(f'loss: {round(max_loss, 3)}') show(max_im) rdm_losses = [] rdm_ims = [] for _ in range(n_random): # random max results rdm_idxs = np.random.choice(np.array(range(layer_size)), size=sm_size, replace=False) random_max_obj = sum([ objectives.channel(lucid_name, unit) for unit in rdm_idxs ]) random_max_im, random_max_loss = render_vis_with_loss( lucid_net, random_max_obj, size=IMAGE_SIZE_IMAGENET, thresholds=(256, )) random_max_images.append(random_max_im) random_max_losses.append(random_max_loss) rdm_losses.append(round(random_max_loss, 3)) rdm_ims.append(np.squeeze(random_max_im)) if display: print(f'random losses: {rdm_losses}') show(np.hstack(rdm_ims)) # for _ in range(n_random): # random min results # rdm_idxs = np.random.choice(np.array(range(layer_size)), size=sm_size, replace=False) # random_min_obj = -1 * sum([objectives.channel(lucid_name, unit) for unit in rdm_idxs]) # random_min_im, random_min_loss = render_vis_with_loss(lucid_net, random_min_obj) # random_min_images.append(random_min_im) # random_min_losses.append(random_min_loss) max_images = np.squeeze(np.array(max_images)) # min_images = np.squeeze(np.array(min_images)) random_max_images = np.squeeze(np.array(random_max_images)) # random_min_images = np.squeeze(np.array(random_min_images)) max_losses = np.array(max_losses) # min_losses = np.array(min_losses) random_max_losses = np.array(random_max_losses) # random_min_losses = np.array(random_min_losses) results = { 'max_images': max_images, # 'min_images': min_images, 'random_max_images': random_max_images, # 'random_min_images': random_min_images, 'max_losses': max_losses, # 'min_losses': min_losses, 'random_max_losses': random_max_losses, # 'random_min_losses': random_min_losses, 'sm_sizes': sm_sizes, 'sm_layer_sizes': sm_layer_sizes, 'sm_layers': sm_layers, 'sm_clusters': sm_clusters } with open(savedir + model_tag + '_max_data.pkl', 'wb') as f: pickle.dump(results, f)
def make_lucid_dataset( model_tag, lucid_net, all_labels, is_unpruned, transforms=[], n_random=9, min_size=5, max_prop=0.8, display=True, savedir='/project/clusterability_in_neural_networks/datasets/', savetag=''): if 'cnn' in model_tag.lower(): cnn_params = CNN_VGG_MODEL_PARAMS if 'vgg' in str( model_tag).lower() else CNN_MODEL_PARAMS layer_sizes = [cl['filters'] for cl in cnn_params['conv']] layer_names = ['conv2d/Relu'] + [ f'conv2d_{i}/Relu' for i in range(1, len(layer_sizes)) ] else: # it's an mlp layer_sizes = [256, 256, 256, 256] layer_names = ['dense/Relu'] + [ f'dense_{i}/Relu' for i in range(1, len(layer_sizes)) ] if not is_unpruned: layer_names = ['prune_low_magnitude_' + ln for ln in layer_names] labels_in_layers = [ np.array(lyr_labels) for lyr_labels in list(splitter(all_labels, layer_sizes)) ] max_images = [ ] # to be filled with images that maximize cluster activations random_max_images = [ ] # to be filled with images that maximize random units activations max_losses = [] # to be filled with losses random_max_losses = [] # to be filled with losses sm_sizes = [] # list of submodule sizes sm_layer_sizes = [] sm_layers = [] # list of layer names sm_clusters = [] # list of clusters imsize = IMAGE_SIZE_CIFAR10 if 'vgg' in model_tag.lower() else IMAGE_SIZE for layer_name, labels, layer_size in zip(layer_names, labels_in_layers, layer_sizes): max_size = max_prop * layer_size for clust_i in range(max(all_labels) + 1): sm_binary = labels == clust_i sm_size = sum(sm_binary) if sm_size <= min_size or sm_size >= max_size: # skip if too big or small continue sm_sizes.append(sm_size) sm_layer_sizes.append(layer_size) sm_layers.append(layer_name) sm_clusters.append(clust_i) # print(f'{model_tag}, layer: {layer_name}') # print(f'submodule_size: {sm_size}, layer_size: {layer_size}') sm_idxs = [i for i in range(layer_size) if sm_binary[i]] max_obj = sum( [objectives.channel(layer_name, unit) for unit in sm_idxs]) max_im, max_loss = render_vis_with_loss(lucid_net, max_obj, size=imsize, transforms=transforms) max_images.append(max_im) max_losses.append(max_loss) if display: print(f'loss: {round(max_loss, 3)}') show(max_im) rdm_losses = [] rdm_ims = [] for _ in range(n_random): # random max results rdm_idxs = np.random.choice(np.array(range(layer_size)), size=sm_size, replace=False) random_max_obj = sum([ objectives.channel(layer_name, unit) for unit in rdm_idxs ]) random_max_im, random_max_loss = render_vis_with_loss( lucid_net, random_max_obj, size=imsize, transforms=transforms) random_max_images.append(random_max_im) random_max_losses.append(random_max_loss) rdm_ims.append(np.squeeze(random_max_im)) rdm_losses.append(round(random_max_loss, 3)) if display: print(f'random losses: {rdm_losses}') show(np.hstack(rdm_ims)) max_images = np.squeeze(np.array(max_images)) random_max_images = np.squeeze(np.array(random_max_images)) max_losses = np.array(max_losses) random_max_losses = np.array(random_max_losses) results = { 'max_images': max_images, 'random_max_images': random_max_images, 'max_losses': max_losses, 'random_max_losses': random_max_losses, 'sm_sizes': sm_sizes, 'sm_layer_sizes': sm_layer_sizes, 'sm_layers': sm_layers, 'sm_clusters': sm_clusters } if is_unpruned: suff = '_unpruned_max_data' else: suff = '_pruned_max_data' with open(savedir + model_tag + suff + savetag + '.pkl', 'wb') as f: pickle.dump(results, f)
image_value_range = (0, 1) input_name = 'input_1' network = FrozenNetwork() network.load_graphdef() if LAYER == "-": images = [] layers = [] for l in sys.stdin: layers.append(l.strip()) for layer in layers: for i in range(COLUMNS): obj = objectives.channel(layer, i) renders = render.render_vis(network, obj) assert len(renders) == 1 image = renders[0] assert len(image) == 1 image = image[0] images.append(image) images = np.array(images) height, width = 128, 128 rows = len(layers) print(images.shape) assert images.shape == (rows * COLUMNS, 128, 128, 3) grid = (images.reshape(rows, COLUMNS, height, width, 3).swapaxes(1, 2).reshape(height * rows, width * COLUMNS, 3)) scipy.misc.imsave(OUTPUT_PREFIX + ".png", grid)
print(t_input, t_prep_input) tf.import_graph_def(self.graph_def, {self.input_name: t_prep_input}, name=scope) self.post_import(scope) # lucid_model = vision_base.Model() # lucid_model.model_path = tf_model lucid_model = PbModel() lucid_model.load_graphdef() temp_graph_def = graph.as_graph_def() # fd = dg # print(lucid_model.graph_def) # model = models.InceptionV1().load_graphdef() # print(model) # print(dir(model)) obj = objectives.channel("discriminator/dis_n_conv_1_4/Conv2D", 2) param_f = lambda: tf.concat([ param.rgb_sigmoid(param.naive([1, 128, 128, 3])), param.fancy_colors(param.naive([1, 128, 128, 8]) / 1.3), param.rgb_sigmoid(param.laplacian_pyramid([1, 128, 128, 3]) / 2.), param.fancy_colors(param.laplacian_pyramid([1, 128, 128, 8]) / 2. / 1.3), ], 0) render.render_vis(lucid_model, obj, param_f) # _ = render.render_vis(lucid_model, "discriminator/dis_n_conv_1_4/Conv2D:0")
def test_channel(inceptionv1): objective = objectives.channel("mixed4a_pre_relu", 42) assert_gradient_ascent(objective, inceptionv1)
if keras_model.layers[i].name == "conv2d_{}".format(layer_idx): keras_layer_idx = i break layer_name = "conv2d_{}/convolution".format(layer_idx) filter_idx = args.filter_idx n_of_filter = args.num_of_filters if filter_idx is None: filters = [ int(idx) for idx in get_strongest_filter(inceptionv3, layer_name, x, n_of_filter) ] else: filters = [filter_idx] for idx in filters: obj = objectives.channel(layer_name, idx) file_name = "Conv{}_{}".format(layer_idx, idx) img_file = file_name + ".png" render_vis(inceptionv3, obj, img_file, idx) saliency_img = visualize_saliency(keras_model, keras_layer_idx, filter_indices=idx, seed_input=y) plt.imshow(img) plt.imshow(saliency_img, alpha=.7) plt.axis('off') plt.savefig("{}_saliency.png".format(file_name))
def get_vm_model_image_losses(args, layer=None): model_name_scope = None if args.model_type == 'vm_model': model_name_scope = 'encode' elif args.model_type == 'simclr_model': model_name_scope = 'base_model' else: raise NotImplementedError('Model type %s not supported!' % args.model_type) def model(t_image): t_image = t_image * 255 ending_points, _ = get_network_outputs({'images': t_image}, prep_type=args.prep_type, model_type=args.model_type, setting_name=args.setting_name, module_name=['encode'], **json.loads(args.cfg_kwargs)) all_vars = tf.global_variables() var_list = [x for x in all_vars if x.name.startswith(model_name_scope)] saver = tf.train.Saver(var_list=var_list) if not args.from_scratch: if not args.load_from_ckpt: model_ckpt_path = tf_model_loader.load_model_from_mgdb( db=args.load_dbname, col=args.load_colname, exp=args.load_expId, port=args.load_port, cache_dir=args.model_cache_dir, step_num=args.load_step, ) else: model_ckpt_path = args.load_from_ckpt saver.restore(tf.get_default_session(), model_ckpt_path) else: SESS = tf.get_default_session() init_op_global = tf.global_variables_initializer() SESS.run(init_op_global) init_op_local = tf.local_variables_initializer() SESS.run(init_op_local) all_train_ref = tf.get_collection_ref(tf.GraphKeys.TRAINABLE_VARIABLES) def _remove_others(vars_ref): cp_vars_ref = copy.copy(vars_ref) for each_v in cp_vars_ref: if each_v.op.name.startswith(model_name_scope): vars_ref.remove(each_v) _remove_others(all_train_ref) return ending_points layer = layer or "encode_9" batch_size = 16 param_f = lambda: param.image(224, batch=batch_size) num_of_units = 64 images = [] all_losses = [] for start_idx in range(0, num_of_units, batch_size): obj = objectives.channel(layer, 0 + start_idx, 0) for idx in range(1, batch_size): obj += objectives.channel(layer, idx + start_idx, idx) image, losses = render_vis(model, obj, param_f, model_name_scope=model_name_scope) images.append(image) all_losses.append(losses) images = np.concatenate(images, axis=0) all_losses = np.sum(all_losses, axis=0) return layer, images, all_losses
type=str, help='path to output folder') parser.add_argument('--layer', default='conv5_block3_out/add') parser.add_argument('--depth', default=2048, type=int) args = parser.parse_args() if __name__ == '__main__': model = CovidNetB() model.load_graphdef() for i in range(args.depth): param_f = lambda: tf.tile( tf.math.reduce_mean(param.image(128, fft=True), axis=3, keepdims=True), [1, 1, 1, 3]) obj = -objectives.channel("conv5_block3_out/add", i) deprecation._PRINT_DEPRECATION_WARNINGS = False tf.logging.set_verbosity(tf.logging.ERROR) print("\nProgress: " + str(i) + '/' + str(args.depth) + '\n') img = render.render_vis(model, obj, param_f, transforms=transform.standard_transforms, thresholds=[ 200, ]) img = np.uint8(255 * (img[0])) cv2.imwrite(
class FrozenNetwork(Model): model_path = MODEL_PATH image_shape = [224, 224, 3] image_value_range = (0, 1) input_name = 'input_1' network = FrozenNetwork() network.load_graphdef() #for layer in network: # print(layer.get_shape()) pixels = 224 param_f = lambda: param.image(pixels, fft=True, decorrelate=True) #obj_test = objectives.channel(LAYER, NEURON_INDEX).get_shape() #print(obj_test) obj = objectives.channel(LAYER, NEURON_INDEX) images = render.render_vis(network, obj, param_f, thresholds=(1024, )) assert len(images) == 1 image = images[0] assert len(image) == 1 image = image[0] out_filename = LAYER.replace( "/", "-") + "_" + str(NEURON_INDEX) + "_" + MODEL_PATH.zfill(10) + ".png" scipy.misc.imsave(out_filename, image)
def build_sprites(local_root='.', graph_version=None, model_loader=None, vis=None, layers=None, vis_filename='vis.js'): if graph_version is None: raise ValueError("graph_version cannot be None") if model_loader is None: raise ValueError("model_loader cannot be None") graph_version_path = os.path.join(local_root, graph_version) if not os.path.isdir(graph_version_path): raise ValueError( "No graph vis directory: {}".format(graph_version_path)) if vis is None: vis = {} update_dict_from_json(json_path=os.path.join(graph_version_path, vis_filename), updatee=vis) # graph_steps = vis['steps'] if 'steps' in vis else [] if len(graph_steps) == 0: print("no graph instances") return for graph_step in graph_steps: graph_step_dir = os.path.join(graph_version_path, graph_step) image_dir = os.path.join(graph_step_dir, 'sprites') image_consumed_dir = os.path.join(graph_step_dir, 'sprites_consumed') image_scum_dir = os.path.join(graph_step_dir, 'sprites_scum') sprite_map_dir = os.path.join(graph_step_dir, 'spritemaps') log_path = os.path.join(graph_step_dir, 'losses.csv') if not os.path.isdir(graph_step_dir): os.mkdir(graph_step_dir) if not os.path.isdir(image_dir): os.mkdir(image_dir) if not os.path.isdir(image_consumed_dir): os.mkdir(image_consumed_dir) if not os.path.isdir(sprite_map_dir): os.mkdir(sprite_map_dir) sprite_dirs = [ d for d in [image_dir, image_consumed_dir, image_scum_dir] if os.path.isdir(d) ] # graph step specific config step_vis = deepcopy(vis) update_dict_from_json(json_path=os.path.join(graph_step_dir, vis_filename), updatee=step_vis) # max_index = step_vis['max_index'] if 'max_index' in step_vis else 2048 scale = step_vis['scale'] if 'scale' in step_vis else 64 thresholds = step_vis['thresholds'] if 'thresholds' in step_vis else [ 64 ] vis_loss = step_vis['loss'] if 'loss' in step_vis else {} batch_id = get_next_batch_id(loss_log_path=log_path) # drives off model json - as might be customised graph_model = get_graph_model(graph_version=graph_version, model_loader=model_loader) layers = graph_model['layers'] # if not None and not empty then only build sprites for these layers/indexes if 'target_layers' in vis: target_layers = vis['target_layers'] layers = [ layer for layer in layers if target_layers is None or len(target_layers) == 0 or layer['index'] in target_layers ] target_indexes = [] if 'target_indexes' not in vis else vis[ 'target_indexes'] use_cppn = True if 'param' in vis and vis['param'] == 'cppn' else False # load existing sprite details existing_sprite_details = get_existing_sprite_details( sprite_dirs=sprite_dirs, scale=scale) print("\nBUILDING SPRITES: graph_version={} steps={}".format( graph_version, graph_step)) print(" layers={}".format([layer['index'] for layer in layers])) for layer in layers: layer_name = layer['name'] layer_index = layer['index'] adam = layer['adam'] transform_id = layer['transform_id'] model = None optimizer = tf.train.AdamOptimizer(adam) transforms = get_transforms(transform_id) # existing_layer_sprites = existing_sprite_details[ layer_index] if layer_index in existing_sprite_details else [] try: print("\nLAYER: {}\n".format(layer)) num_processed = 0 for index in range(0, max_index): # check for abort in vis files vf_abort = check_abort(dirs=[ os.path.join(graph_version_path, vis_filename), os.path.join(graph_step_dir, vis_filename) ]) if len(vf_abort) > 0: print("\nDetected abort in vis files: {}".format( vf_abort)) return # check any target indexes if not (target_indexes is None or len(target_indexes) == 0 or index in target_indexes): continue # existing_index_sprite_thresholds = existing_layer_sprites[ index] if index in existing_layer_sprites else [] # calculate work to do # do all thresholds already existing thresholds_to_generate = [ t for t in thresholds if t not in existing_index_sprite_thresholds ] if len(thresholds_to_generate) == 0: continue # can start from an existing threshold max_existing_threshold = max( existing_index_sprite_thresholds ) if len(existing_index_sprite_thresholds) > 0 else None if max_existing_threshold is not None and max_existing_threshold <= min( thresholds_to_generate): threshold_start = max_existing_threshold + 1 img_path = [ ip for ip in [ get_image_path(sd, layer_index, index, max_existing_threshold, scale) for sd in sprite_dirs ] if os.path.isfile(ip) ][0] with Image.open(img_path) as im: im.load() # make array im_1 = np.array(im) # add dummy batch dimension im_2 = np.expand_dims(im_1, axis=0) # reduce less than one init_val = im_2.astype(np.float32) / 256 param_f = lambda: lucid_images.image( scale, fft=False, init_val=init_val) elif use_cppn: threshold_start = 0 adam = 0.00055 optimizer = tf.train.AdamOptimizer(adam) param_f = lambda: param.cppn(scale) else: threshold_start = 0 param_f = lambda: param.image( scale, fft=True, decorrelate=True) # drop the model regularly if num_processed % 100 == 0: print("Reloading model ...") model = None num_processed = 0 if model is None: model = model_loader(graph_step) model.load_graphdef() # start the feature print("\nFEATURE: {}:{}\n".format(layer['name'], index)) log_item = { "batch_id": batch_id, "timestamp": current_milli_time(), "scale": scale, "adam": adam, "transforms": transform_id, "layer": layer_index, "index": index } visualizations = [] try: visualization = get_visualizations_and_losses( model, objectives.channel(layer_name, index), transforms=transforms, param_f=param_f, optimizer=optimizer, threshold_start=threshold_start, thresholds=thresholds, visualization_index=index, visualization_layer=layer_index, minimum_loss=vis_loss['minimum_loss_threshold'] if 'minimum_loss_threshold' in vis_loss else 0, num_bins=vis_loss['num_bins'] if 'num_bins' in vis_loss else 0, max_bin_hits=vis_loss['max_bin_hits'] if 'max_bin_hits' in vis_loss else 0, bin_factor=vis_loss['bin_factor'] if 'bin_factor' in vis_loss else 0, loss_logger=lambda l, t, s: loss_logger( log_file=log_path, item=log_item, threshold=t, loss=l, status=s)) num_processed = num_processed + 1 if len(visualization) == 0: continue # check losses losses = [v[2] for v in visualization] print("\nLOSSES: feature={}:{}; {}\n".format( layer_index, index, losses)) visualizations.append(visualization) finally: if len(visualizations) > 0: store_visualizations_and_losses( visualizations, output_dir=image_dir, scale=scale) except ValueError as e: msg = "{}".format(e) if 'slice index' in msg and 'out of bounds' in msg: print( "Closing layer: slice index out of bounds: {}".format( e)) else: raise e