def load_and_saliency(model_path,
input_paths,
baseline=-1,
dictionary_path=None,
strip_alpha=False,
smooth_factor=7,
save=False,
save_dir=None):
"""A helper class to load input and invoke the saliency api
Args:
model_path: The path the model file (str)
input_paths: The paths to model input files [(str),...] or to a folder of inputs [(str)]
baseline: Either a number corresponding to the baseline for integration, or a path to a baseline file
dictionary_path: The path to a dictionary file encoding a 'class_idx'->'class_name' mapping
strip_alpha: Whether to collapse alpha channels when loading an input (bool)
smooth_factor: How many iterations of the smoothing algorithm to run (int)
save: Whether to save (True) or display (False) the resulting image
save_dir: Where to save the image if save=True
"""
model_dir = os.path.dirname(model_path)
if save_dir is None:
save_dir = model_dir
if not save:
save_dir = None
network = keras.models.load_model(model_path, compile=False)
input_type = network.input.dtype
input_shape = network.input.shape
n_channels = 0 if len(input_shape) == 3 else input_shape[3]
dic = load_dict(dictionary_path)
if len(input_paths) == 1 and os.path.isdir(input_paths[0]):
loader = PathLoader(input_paths[0])
input_paths = [path[0] for path in loader.path_pairs]
inputs = [
load_image(input_paths[i],
strip_alpha=strip_alpha,
channels=n_channels) for i in range(len(input_paths))
]
max_shapes = np.maximum.reduce([inp.shape for inp in inputs], axis=0)
tf_image = tf.stack([
tf.image.resize_with_crop_or_pad(
tf.convert_to_tensor(im, dtype=input_type), max_shapes[0],
max_shapes[1]) for im in inputs
],
axis=0)
if is_number(baseline):
baseline_gen = tf.constant_initializer(float(baseline))
baseline_image = baseline_gen(shape=tf_image.shape, dtype=input_type)
else:
baseline_image = load_image(baseline)
baseline_image = tf.convert_to_tensor(baseline_image, dtype=input_type)
visualize_saliency(network,
tf_image,
baseline_input=baseline_image,
decode_dictionary=dic,
smooth=smooth_factor,
save_path=save_dir)
def load_and_gradcam(model_path,
input_paths,
layer_id=None,
dictionary_path=None,
strip_alpha=False,
save=False,
save_dir=None):
"""A helper class to load input and invoke the gradcam api
Args:
model_path: The path the model file (str)
input_paths: The paths to model input files [(str),...] or to a folder of inputs [(str)]
layer_id: The layer id to be used. None defaults to the last conv layer in the model
dictionary_path: The path to a dictionary file encoding a 'class_idx'->'class_name' mapping
strip_alpha: Whether to collapse alpha channels when loading an input (bool)
save: Whether to save (True) or display (False) the resulting image
save_dir: Where to save the image if save=True
"""
model_dir = os.path.dirname(model_path)
if save_dir is None:
save_dir = model_dir
if not save:
save_dir = None
network = keras.models.load_model(model_path, compile=False)
input_type = network.input.dtype
input_shape = network.input.shape
n_channels = 0 if len(input_shape) == 3 else input_shape[3]
dic = load_dict(dictionary_path)
if len(input_paths) == 1 and os.path.isdir(input_paths[0]):
loader = PathLoader(input_paths[0])
input_paths = [path[0] for path in loader.path_pairs]
inputs = [
load_image(input_paths[i],
strip_alpha=strip_alpha,
channels=n_channels) for i in range(len(input_paths))
]
max_shapes = np.maximum.reduce([inp.shape for inp in inputs], axis=0)
tf_image = tf.stack([
tf.image.resize_with_crop_or_pad(
tf.convert_to_tensor(im, dtype=input_type), max_shapes[0],
max_shapes[1]) for im in inputs
],
axis=0)
visualize_gradcam(inputs=tf_image,
model=network,
layer_id=layer_id,
decode_dictionary=dic,
save_path=save_dir)
def load_and_caricature(model_path,
input_paths,
dictionary_path=None,
save=False,
save_dir=None,
strip_alpha=False,
layer_ids=None,
print_layers=False,
n_steps=512,
learning_rate=0.05,
blur=1,
cossim_pow=0.5,
sd=0.01,
fft=True,
decorrelate=True,
sigmoid=True):
"""
Args:
model_path (str): The path to a keras model to be inspected by the Caricature visualization
layer_ids (int, list): The layer(s) of the model to be inspected by the Caricature visualization
input_paths (list): Strings corresponding to image files to be visualized
dictionary_path (string): A path to a dictionary mapping model outputs to class names
save (bool): Whether to save (True) or display (False) the result
save_dir (str): Where to save the image if save is True
strip_alpha (bool): Whether to strip the alpha channel from input images
print_layers (bool): Whether to skip visualization and instead just print out the available layers in a model \
(useful for deciding which layers you might want to caricature)
n_steps (int): How many steps of optimization to run when computing caricatures (quality vs time trade)
learning_rate (float): The learning rate of the caricature optimizer. Should be higher than usual
blur (float): How much blur to add to images during caricature generation
cossim_pow (float): How much should similarity in form be valued versus creative license
sd (float): The standard deviation of the noise used to seed the caricature
fft (bool): Whether to use fft space (True) or image space (False) to create caricatures
decorrelate (bool): Whether to use an ImageNet-derived color correlation matrix to de-correlate
colors in the caricature. Parameter has no effect on grey scale images.
sigmoid (bool): Whether to use sigmoid (True) or clipping (False) to bound the caricature pixel values
"""
model_dir = os.path.dirname(model_path)
if save_dir is None and save:
save_dir = model_dir
network = keras.models.load_model(model_path, compile=False)
input_type = network.input.dtype
input_shape = network.input.shape
n_channels = 0 if len(input_shape) == 3 else input_shape[3]
input_height = input_shape[
1] or 224 # If the model doesn't specify width and height, just guess 224
input_width = input_shape[2] or 224
if print_layers:
for idx, layer in enumerate(network.layers):
print("{}: {} --- output shape: {}".format(idx, layer.name,
layer.output_shape))
return
dic = load_dict(dictionary_path)
if len(input_paths) == 1 and os.path.isdir(input_paths[0]):
loader = PathLoader(input_paths[0])
input_paths = [path[0] for path in loader.path_pairs]
inputs = [
load_image(input_paths[i],
strip_alpha=strip_alpha,
channels=n_channels) for i in range(len(input_paths))
]
tf_image = tf.stack([
tf.image.resize_with_pad(tf.convert_to_tensor(im, dtype=input_type),
input_height,
input_width,
method='lanczos3') for im in inputs
])
tf_image = tf.clip_by_value(tf_image, -1, 1)
visualize_caricature(network,
tf_image,
layer_ids=layer_ids,
decode_dictionary=dic,
save_path=save_dir,
n_steps=n_steps,
learning_rate=learning_rate,
blur=blur,
cossim_pow=cossim_pow,
sd=sd,
fft=fft,
decorrelate=decorrelate,
sigmoid=sigmoid)
def load_and_umap(model_path,
input_root_path,
print_layers=False,
strip_alpha=False,
layers=None,
input_extension=None,
batch=10,
use_cache=True,
cache_dir=None,
dictionary_path=None,
save=False,
save_dir=None,
legend_mode='shared',
umap_parameters=None):
if umap_parameters is None:
umap_parameters = {}
if save is True and save_dir is None:
save_dir = os.path.dirname(model_path)
if cache_dir is None:
# If the user passes the input dir as a relative path without ./ then dirname will contain all path info
if os.path.basename(input_root_path) == "":
cache_dir = os.path.dirname(input_root_path) + "__layer_outputs"
else:
cache_dir = os.path.join(
os.path.dirname(input_root_path),
os.path.basename(input_root_path) + "__layer_outputs")
network = keras.models.load_model(model_path, compile=False)
if print_layers:
for idx, layer in enumerate(network.layers):
print("{}: {} --- output shape: {}".format(idx, layer.name,
layer.output_shape))
return
evaluator = Evaluator(network, layers=layers)
loader = ImageLoader(input_root_path,
network,
batch=batch,
input_extension=input_extension,
strip_alpha=strip_alpha)
cache = FileCache(cache_dir, evaluator.layers) if use_cache else None
plotter = UmapPlotter(load_dict(dictionary_path, True), **umap_parameters)
classes = []
layer_outputs = None
for batch_id, (batch_inputs, batch_classes) in enumerate(
tqdm(loader, desc='Computing Outputs', unit='batch')):
if use_cache and cache.batch_cached(batch_id):
continue
batch_layer_outputs = evaluator.evaluate(batch_inputs)
if use_cache:
cache.save(batch_layer_outputs, batch_classes)
else:
if layer_outputs is None:
layer_outputs = batch_layer_outputs
else:
for i, (layer, batch_layer) in enumerate(
zip(layer_outputs, batch_layer_outputs)):
layer_outputs[i] = np.concatenate((layer, batch_layer),
axis=0)
classes.extend(batch_classes)
if use_cache:
layer_outputs, classes = cache.load(len(loader))
plotter.visualize_umap(layer_outputs,
labels=classes,
legend_loc=legend_mode,
save_path=save_dir,
title=[
"Layer {}: {}".format(
evaluator.layers[idx],
network.layers[evaluator.layers[idx]].name)
for idx in range(len(layer_outputs))
])
def umap_layers(model_path,
input_root_path,
print_layers=False,
strip_alpha=False,
layers=None,
input_extension=None,
batch=10,
use_cache=True,
cache_dir=None,
dictionary_path=None,
save=False,
save_dir=None,
legend_mode='shared',
umap_parameters=None):
if umap_parameters is None:
umap_parameters = {}
if save_dir is None:
save_dir = os.path.dirname(model_path)
if cache_dir is None:
# If the user passes the input dir as a relative path without ./ then dirname will contain all path info
if os.path.basename(input_root_path) == "":
cache_dir = os.path.dirname(input_root_path) + "__layer_outputs"
else:
cache_dir = os.path.join(
os.path.dirname(input_root_path),
os.path.basename(input_root_path) + "__layer_outputs")
network = keras.models.load_model(model_path)
if print_layers:
for idx, layer in enumerate(network.layers):
print("{}: {} --- output shape: {}".format(idx, layer.name,
layer.output_shape))
return
evaluator = Evaluator(network, layers=layers)
loader = ImageLoader(input_root_path,
network,
batch=batch,
input_extension=input_extension,
strip_alpha=strip_alpha)
cache = FileCache(cache_dir, evaluator.layers,
umap_parameters) if use_cache else None
classes = []
layer_outputs = None
for batch_id, (batch_inputs, batch_classes) in enumerate(
tqdm(loader, desc='Computing Outputs', unit='batch')):
if use_cache and cache.batch_cached(batch_id):
continue
batch_layer_outputs = evaluator.evaluate(batch_inputs)
if use_cache:
cache.save(batch_layer_outputs, batch_classes)
else:
if layer_outputs is None:
layer_outputs = batch_layer_outputs
else:
for i, (layer, batch_layer) in enumerate(
zip(layer_outputs, batch_layer_outputs)):
layer_outputs[i] = np.concatenate((layer, batch_layer),
axis=0)
classes.extend(batch_classes)
if use_cache:
layer_outputs, classes = cache.load_and_transform(len(loader))
else:
fit = umap.UMAP(**umap_parameters)
with Suppressor(): # Silence a bunch of numba warnings
layer_outputs = [
fit.fit_transform(layer) for layer in layer_outputs
]
draw_umaps(layer_outputs,
classes,
layer_ids=layers,
layers=network.layers,
save=save,
save_path=save_dir,
dictionary=load_dict(dictionary_path, True),
legend_mode=legend_mode)