def load_and_saliency(model_path,
input_paths,
baseline=-1,
dictionary_path=None,
strip_alpha=False,
smooth_factor=7,
save=False,
save_dir=None):
"""A helper class to load input and invoke the saliency api
Args:
model_path: The path the model file (str)
input_paths: The paths to model input files [(str),...] or to a folder of inputs [(str)]
baseline: Either a number corresponding to the baseline for integration, or a path to a baseline file
dictionary_path: The path to a dictionary file encoding a 'class_idx'->'class_name' mapping
strip_alpha: Whether to collapse alpha channels when loading an input (bool)
smooth_factor: How many iterations of the smoothing algorithm to run (int)
save: Whether to save (True) or display (False) the resulting image
save_dir: Where to save the image if save=True
"""
model_dir = os.path.dirname(model_path)
if save_dir is None:
save_dir = model_dir
if not save:
save_dir = None
network = keras.models.load_model(model_path, compile=False)
input_type = network.input.dtype
input_shape = network.input.shape
n_channels = 0 if len(input_shape) == 3 else input_shape[3]
dic = load_dict(dictionary_path)
if len(input_paths) == 1 and os.path.isdir(input_paths[0]):
loader = PathLoader(input_paths[0])
input_paths = [path[0] for path in loader.path_pairs]
inputs = [
load_image(input_paths[i],
strip_alpha=strip_alpha,
channels=n_channels) for i in range(len(input_paths))
]
max_shapes = np.maximum.reduce([inp.shape for inp in inputs], axis=0)
tf_image = tf.stack([
tf.image.resize_with_crop_or_pad(
tf.convert_to_tensor(im, dtype=input_type), max_shapes[0],
max_shapes[1]) for im in inputs
],
axis=0)
if is_number(baseline):
baseline_gen = tf.constant_initializer(float(baseline))
baseline_image = baseline_gen(shape=tf_image.shape, dtype=input_type)
else:
baseline_image = load_image(baseline)
baseline_image = tf.convert_to_tensor(baseline_image, dtype=input_type)
visualize_saliency(network,
tf_image,
baseline_input=baseline_image,
decode_dictionary=dic,
smooth=smooth_factor,
save_path=save_dir)
def __next__(self):
paths = super().__next__()
inputs = [
load_image(paths[i][0], strip_alpha=self.strip_alpha, channels=self.n_channels) for i in range(len(paths))
]
batch_inputs = tf.stack([
tf.image.resize_with_crop_or_pad(tf.convert_to_tensor(im, dtype=self.input_type), self.input_shape[1],
self.input_shape[2]) for im in inputs
], axis=0)
batch_classes = [paths[i][1] for i in range(len(paths))]
return batch_inputs, batch_classes
def load_and_gradcam(model_path,
input_paths,
layer_id=None,
dictionary_path=None,
strip_alpha=False,
save=False,
save_dir=None):
"""A helper class to load input and invoke the gradcam api
Args:
model_path: The path the model file (str)
input_paths: The paths to model input files [(str),...] or to a folder of inputs [(str)]
layer_id: The layer id to be used. None defaults to the last conv layer in the model
dictionary_path: The path to a dictionary file encoding a 'class_idx'->'class_name' mapping
strip_alpha: Whether to collapse alpha channels when loading an input (bool)
save: Whether to save (True) or display (False) the resulting image
save_dir: Where to save the image if save=True
"""
model_dir = os.path.dirname(model_path)
if save_dir is None:
save_dir = model_dir
if not save:
save_dir = None
network = keras.models.load_model(model_path, compile=False)
input_type = network.input.dtype
input_shape = network.input.shape
n_channels = 0 if len(input_shape) == 3 else input_shape[3]
dic = load_dict(dictionary_path)
if len(input_paths) == 1 and os.path.isdir(input_paths[0]):
loader = PathLoader(input_paths[0])
input_paths = [path[0] for path in loader.path_pairs]
inputs = [
load_image(input_paths[i],
strip_alpha=strip_alpha,
channels=n_channels) for i in range(len(input_paths))
]
max_shapes = np.maximum.reduce([inp.shape for inp in inputs], axis=0)
tf_image = tf.stack([
tf.image.resize_with_crop_or_pad(
tf.convert_to_tensor(im, dtype=input_type), max_shapes[0],
max_shapes[1]) for im in inputs
],
axis=0)
visualize_gradcam(inputs=tf_image,
model=network,
layer_id=layer_id,
decode_dictionary=dic,
save_path=save_dir)
def load_and_caricature(model_path,
input_paths,
dictionary_path=None,
save=False,
save_dir=None,
strip_alpha=False,
layer_ids=None,
print_layers=False,
n_steps=512,
learning_rate=0.05,
blur=1,
cossim_pow=0.5,
sd=0.01,
fft=True,
decorrelate=True,
sigmoid=True):
"""
Args:
model_path (str): The path to a keras model to be inspected by the Caricature visualization
layer_ids (int, list): The layer(s) of the model to be inspected by the Caricature visualization
input_paths (list): Strings corresponding to image files to be visualized
dictionary_path (string): A path to a dictionary mapping model outputs to class names
save (bool): Whether to save (True) or display (False) the result
save_dir (str): Where to save the image if save is True
strip_alpha (bool): Whether to strip the alpha channel from input images
print_layers (bool): Whether to skip visualization and instead just print out the available layers in a model \
(useful for deciding which layers you might want to caricature)
n_steps (int): How many steps of optimization to run when computing caricatures (quality vs time trade)
learning_rate (float): The learning rate of the caricature optimizer. Should be higher than usual
blur (float): How much blur to add to images during caricature generation
cossim_pow (float): How much should similarity in form be valued versus creative license
sd (float): The standard deviation of the noise used to seed the caricature
fft (bool): Whether to use fft space (True) or image space (False) to create caricatures
decorrelate (bool): Whether to use an ImageNet-derived color correlation matrix to de-correlate
colors in the caricature. Parameter has no effect on grey scale images.
sigmoid (bool): Whether to use sigmoid (True) or clipping (False) to bound the caricature pixel values
"""
model_dir = os.path.dirname(model_path)
if save_dir is None and save:
save_dir = model_dir
network = keras.models.load_model(model_path, compile=False)
input_type = network.input.dtype
input_shape = network.input.shape
n_channels = 0 if len(input_shape) == 3 else input_shape[3]
input_height = input_shape[
1] or 224 # If the model doesn't specify width and height, just guess 224
input_width = input_shape[2] or 224
if print_layers:
for idx, layer in enumerate(network.layers):
print("{}: {} --- output shape: {}".format(idx, layer.name,
layer.output_shape))
return
dic = load_dict(dictionary_path)
if len(input_paths) == 1 and os.path.isdir(input_paths[0]):
loader = PathLoader(input_paths[0])
input_paths = [path[0] for path in loader.path_pairs]
inputs = [
load_image(input_paths[i],
strip_alpha=strip_alpha,
channels=n_channels) for i in range(len(input_paths))
]
tf_image = tf.stack([
tf.image.resize_with_pad(tf.convert_to_tensor(im, dtype=input_type),
input_height,
input_width,
method='lanczos3') for im in inputs
])
tf_image = tf.clip_by_value(tf_image, -1, 1)
visualize_caricature(network,
tf_image,
layer_ids=layer_ids,
decode_dictionary=dic,
save_path=save_dir,
n_steps=n_steps,
learning_rate=learning_rate,
blur=blur,
cossim_pow=cossim_pow,
sd=sd,
fft=fft,
decorrelate=decorrelate,
sigmoid=sigmoid)
