def neuron_groups(model, img, layer, n_groups=6, attr_classes=[]): # Compute activations with tf.Graph().as_default(), tf.Session(): t_input = tf.placeholder_with_default(img, [None, None, 3]) T = render.import_model(model, t_input, t_input) acts = T(layer).eval() # We'll use ChannelReducer (a wrapper around scikit learn's factorization tools) # to apply Non-Negative Matrix factorization (NMF). nmf = ChannelReducer(n_groups, "PCA") print(layer, n_groups) spatial_factors = nmf.fit_transform(acts)[0].transpose(2, 0, 1).astype("float32") channel_factors = nmf._reducer.components_.astype("float32") # Let's organize the channels based on their horizontal position in the image x_peak = np.argmax(spatial_factors.max(1), 1) ns_sorted = np.argsort(x_peak) spatial_factors = spatial_factors[ns_sorted] channel_factors = channel_factors[ns_sorted] # And create a feature visualziation of each group param_f = lambda: param.image(80, batch=n_groups) obj = sum(objectives.direction(layer, channel_factors[i], batch=i) for i in range(n_groups)) group_icons = render.render_vis(model, obj, param_f, verbose=False)[-1] # We'd also like to know about attribution # # First, let's turn each group into a vector over activations group_vecs = [spatial_factors[i, ..., None] * channel_factors[i] for i in range(n_groups)] attrs = np.asarray([raw_class_group_attr(img, layer, attr_class, model, group_vecs) for attr_class in attr_classes]) gray_scale_groups = [skimage.color.rgb2gray(icon) for icon in group_icons] # Let's render the visualization! data = { "img": _image_url(img), "n_groups": n_groups, "spatial_factors": [_image_url(factor[..., None] / np.percentile(spatial_factors, 99) * [1, 0, 0]) for factor in spatial_factors], "group_icons": [_image_url(icon) for icon in gray_scale_groups] } # with open('ng.pickle', 'wb') as handle: # pickle.dump(data, handle, protocol=pickle.HIGHEST_PROTOCOL) # with open('./svelte_python/ng.pickle', 'rb') as p_file: # data = pickle.load(p_file) generate_html('neuron_groups', data)
def spatial_spatial_attr(model, img, layer1, layer2, hint_label_1=None, hint_label_2=None, override=None): hint1 = orange_blue(raw_class_spatial_attr(model, img, layer1, hint_label_1, override=override), raw_class_spatial_attr(model, img, layer1, hint_label_2, override=override), clip=True) hint2 = orange_blue(raw_class_spatial_attr(model, img, layer2, hint_label_1, override=override), raw_class_spatial_attr(model, img, layer2, hint_label_2, override=override), clip=True) attrs = raw_spatial_spatial_attr(model, img, layer1, layer2, override=override) attrs = attrs / attrs.max() data = { "spritemap1": image_url_grid(attrs), "spritemap2": image_url_grid(attrs.transpose(2, 3, 0, 1)), "size1": attrs.shape[3], "layer1": layer1, "size2": attrs.shape[0], "layer2": layer2, "img": _image_url(img), "hint1": _image_url(hint1), "hint2": _image_url(hint2) } generate_html('spatial_attr', data)
def show_image(image): html = "" data_url = _image_url(image) html += '<img width=\"100\" style=\"margin: 10px\" src=\"' + data_url + '\">' with open("img.html", "w") as f: f.write(html) _display_html(html)
def image_url_grid(grid): return [[_image_url(img) for img in line] for line in grid]
def show_images(self, images): html = "" for image in images: data_url = _image_url(image) html += '<img width=\"100\" style=\"margin: 10px\" src=\"' + data_url + '\">' _display_html(html)
text.append(line) else: if "<image" in line and "image/png" in line and ( len(line) > 10000 or (key != "images/rot-features" and len(line) > 10000)): image_str = line.split("xlink:href=\"")[1].split("\"")[0] image_content = image_str[len("data:image/png;base64,"):] image_hash = hex(hash(image_content))[4:20] with open(f"public/generated_images/{image_hash}.png", "wb") as f: f.write(base64.b64decode(image_content)) line = line.replace(image_str, f"generated_images/{image_hash}.png") line = line.replace("/>", " style='image-rendering: auto;' />") print(len(image_content) // 1024, end=", ") elif "<image" in line: image_str = line.split("xlink:href=\"")[1].split("\"")[0] arr = png_url2im(image_str) if arr.shape[0] == 5 and arr.shape[1] == 5: print(arr.shape) arr = np.repeat(arr, 4, axis=0) arr = np.repeat(arr, 4, axis=1) new_image_str = _image_url(arr) line = line.replace(image_str, new_image_str) text.append(line) figure_html[key] = "\n".join(text) index_template = open("index_template.html", "r").read() index_html = index_template.format(**figure_html) open("public/index.html", "w").write(index_html) print("")
def spatial_spatial_attr(imglist, filenamelist, layer1, layer2, hint_label_1=None, hint_label_2=None, override=None): filename = '' for f in filenamelist: filename += f with open('result/' + filename + '.html', 'a') as f: f.write('''<!DOCTYPE html> <html> <head > <title>%s</title> <script src='GroupWidget_1cb0e0d.js'></script> </head> <body>''' % (filename)) for key, img in enumerate(imglist): hint1 = orange_blue(raw_class_spatial_attr(img, layer1, hint_label_1, override=override), raw_class_spatial_attr(img, layer1, hint_label_2, override=override), clip=True) hint2 = orange_blue(raw_class_spatial_attr(img, layer2, hint_label_1, override=override), raw_class_spatial_attr(img, layer2, hint_label_2, override=override), clip=True) attrs = raw_spatial_spatial_attr(img, layer1, layer2, override=override) attrs = attrs / attrs.max() with open('result/' + filename + '.html', 'a') as f: f.write(''' <main%s></main%s> <script> var app = new GroupWidget_1cb0e0d({ target: document.querySelector( 'main%s' ),''' % (key, key, key)) f.write(''' data: { "layer2": "''' + layer2 + '''", "layer1": "''' + layer1 + '''",''') f.write('"spritemap1"' + ":" + str(image_url_grid(attrs)) + ',\n') f.write('"spritemap2"' + ":" + str(image_url_grid(attrs.transpose(2, 3, 0, 1))) + ',\n') f.write('"size1"' + ":" + str(attrs.shape[3]) + ',\n') f.write('"size2"' + ":" + str(attrs.shape[0]) + ',\n') f.write('"img"' + ":" + '"' + str(_image_url(img)) + '"' + ',\n') f.write('"hint1"' + ":" + '"' + str(_image_url(hint1)) + '"' + ',\n') f.write('"hint2"' + ":" + '"' + str(_image_url(hint2)) + '"' + '\n') f.write('''} });''') f.write('''</script>''') with open('result/' + filename + '.html', 'a') as f: f.write('''</body></html >''') print(filename)
def neuron_groups(imglist, filenamelist, layer, n_groups=6, attr_classes=None): # Compute activations filename = '' for f in filenamelist: filename += f with open('result/' + filename + '.html', 'a') as f: f.write('''<!DOCTYPE html> <html> <head > <title>%s</title> <script src='GroupWidget_1cb0e0d.js'></script> </head> <body>''' % (filename)) for key, img in enumerate(imglist): if attr_classes is None: attr_classes = [] with tf.Graph().as_default(), tf.Session(): t_input = tf.placeholder_with_default(img, [None, None, 3]) T = render.import_model(model, t_input, t_input) acts = T(layer).eval() # We'll use ChannelReducer (a wrapper around scikit learn's factorization tools) # to apply Non-Negative Matrix factorization (NMF). nmf = ChannelReducer(n_groups, "NMF") spatial_factors = nmf.fit_transform(acts)[0].transpose( 2, 0, 1).astype("float32") channel_factors = nmf._reducer.components_.astype("float32") # Let's organize the channels based on their horizontal position in the image x_peak = np.argmax(spatial_factors.max(1), 1) ns_sorted = np.argsort(x_peak) spatial_factors = spatial_factors[ns_sorted] channel_factors = channel_factors[ns_sorted] # And create a feature visualziation of each group param_f = lambda: param.image(80, batch=n_groups) obj = sum( objectives.direction(layer, channel_factors[i], batch=i) for i in range(n_groups)) group_icons = render.render_vis(model, obj, param_f, verbose=False)[-1] # We'd also like to know about attribution # First, let's turn each group into a vector over activations group_vecs = [ spatial_factors[i, ..., None] * channel_factors[i] for i in range(n_groups) ] attrs = np.asarray([ raw_class_group_attr(img, layer, attr_class, group_vecs) for attr_class in attr_classes ]) print(attrs) # Let's render the visualization! with open('result/' + filename + '.html', 'a') as f: f.write(''' <main%s></main%s> <script> var app = new GroupWidget_1cb0e0d({ target: document.querySelector( 'main%s' ),''' % (key, key, key)) f.write('''data: {''') f.write('"img":"%s",\n' % str(_image_url(img))) f.write('"n_groups"' + ":" + str(n_groups) + ',\n') f.write('"spatial_factors"' + ":" + str([ _image_url(factor[..., None] / np.percentile(spatial_factors, 99) * [1, 0, 0]) for factor in spatial_factors ]) + ',\n') f.write('"group_icons"' + ":" + str([_image_url(icon) for icon in group_icons]) + ',\n') f.write('''} });''') f.write('''</script>''') with open('result/' + filename + '.html', 'a') as f: f.write('''</body></html >''') print(filename)