def test_sequential_call():
"""Test keras.models.Sequential.__call__"""
nb_samples, input_dim, output_dim = 3, 10, 5
model = Sequential()
model.add(Dense(output_dim=output_dim, input_dim=input_dim))
model.compile('sgd', 'mse')
# test flat model
X = K.placeholder(ndim=2)
Y = model(X)
f = K.function([X], [Y])
x = np.ones((nb_samples, input_dim)).astype(K.floatx())
y1 = f([x])[0].astype(K.floatx())
y2 = model.predict(x)
# results of __call__ should match model.predict
assert_allclose(y1, y2)
# test nested model
model2 = Sequential()
model2.add(model)
model2.compile('sgd', 'mse')
Y2 = model2(X)
f = K.function([X], [Y2])
y1 = f([x])[0].astype(K.floatx())
y2 = model2.predict(x)
# results of __call__ should match model.predict
assert_allclose(y1, y2)
def next(self):
"""
Get the next batch.
"""
if self.batch_idx >= self.n:
self.reset()
# incomplete batch
if self.batch_idx + self.batch_size >= self.n:
batch_size = self.n - self.batch_idx
else:
batch_size = self.batch_size
indices = self.indices[self.batch_idx:self.batch_idx + batch_size]
paths, labels = self.paths[indices], self.labels[indices]
x_batch = np.zeros((batch_size,) + self.output_shape, dtype=K.floatx())
# load and process the image batch
for idx in xrange(len(paths)):
img = imread(paths[idx], mode='RGB')
img = np.asarray(img, dtype=K.floatx())
img = self.transform(img)
if K.image_data_format() == 'channels_first':
img = img.transpose(2, 0, 1)
x_batch[idx] = img
self.batch_idx += self.batch_size
return x_batch, labels
def test_rotates_images():
bs = 3
shape = (1, 8, 8)
img = np.zeros((bs, 1, 8, 8), dtype=K.floatx())
angle = np.asarray([0, math.pi / 2, math.pi], dtype=K.floatx())
img_input = Input(shape=shape)
rot_input = Input(shape=(1,))
rot_layer = RotationTransformer()([img_input, rot_input])
model = Model(input=[img_input, rot_input], output=rot_layer)
model.compile('adam', 'mse')
_, theta = model.predict([img, angle])
np.testing.assert_almost_equal(
theta.reshape(-1, 2, 3),
np.asarray([
[[1, 0, 0],
[0, 1, 0]],
[[0, -1, 0],
[1, 0, 0]],
[[-1, 0, 0],
[0, -1, 0]],
]),
verbose=True
)
def call(self, x, mask=None):
eij = dot_product(x, self.W)
if self.bias:
eij += self.b
eij = K.tanh(eij)
a = K.exp(eij)
# apply mask after the exp. will be re-normalized next
if mask is not None:
# Cast the mask to floatX to avoid float64 upcasting in theano
a *= K.cast(mask, K.floatx())
# in some cases especially in the early stages of training the sum may be almost zero
# and this results in NaN's. A workaround is to add a very small positive number ε to the sum.
# a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())
a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())
weighted_input = x * K.expand_dims(a)
result = K.sum(weighted_input, axis=1)
if self.return_attention:
return [result, a]
return result
def call(self, x, mask=None):
# eij = K.dot(x, self.W) TF backend doesn't support it
# features_dim = self.W.shape[0]
# step_dim = x._keras_shape[1]
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)), K.reshape(self.W, (features_dim, 1))), (-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
a = K.exp(eij)
# apply mask after the exp. will be re-normalized next
if mask is not None:
# Cast the mask to floatX to avoid float64 upcasting in theano
a *= K.cast(mask, K.floatx())
# in some cases especially in the early stages of training the sum may be almost zero
a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())
a = K.expand_dims(a)
weighted_input = x * a
# print weigthted_input.shape
return K.sum(weighted_input, axis=1)
def next(self):
"""For python 2.x.
# Returns
The next batch.
"""
# Keeps under lock only the mechanism which advances
# the indexing of each batch.
with self.lock:
index_array, current_index, current_batch_size = next(
self.index_generator)
# The transformation of images is not under thread lock
# so it can be done in parallel
batch_x = np.zeros(
tuple([current_batch_size] + list(self.x.shape)[1:]), dtype=K.floatx())
for i, j in enumerate(index_array):
x = self.x[j]
x = self.image_data_generator.random_transform(
x.astype(K.floatx()))
x = self.image_data_generator.standardize(x)
batch_x[i] = x
if self.save_to_dir:
for i in range(current_batch_size):
img = array_to_img(batch_x[i], self.data_format, scale=True)
fname = '{prefix}_{index}_{hash}.{format}'.format(prefix=self.save_prefix,
index=current_index + i,
hash=np.random.randint(
1e4),
format=self.save_format)
img.save(os.path.join(self.save_to_dir, fname))
if self.y is None:
return batch_x
batch_y = self.y[index_array]
return batch_x, batch_y
def call(self, x):
r = K.cast(K.arange(self.num), K.floatx()) / float(self.num - 1)
r = self.start + (self.stop - self.start) * r
r = K.expand_dims(K.expand_dims(r), axis=0)
r = K.cast(r, dtype=K.floatx())
r = K.tile(r, (K.shape(x)[0], 1, 1))
return r
def _get_batches_of_transformed_samples(self, index_array):
batch_x = np.zeros((len(index_array),) + self.image_shape, dtype=K.floatx())
if self.with_labels:
batch_y = np.zeros((len(batch_x), self.num_class), dtype=K.floatx())
for i, j in enumerate(index_array):
# Protect file and dataframe access with a lock.
with self.lock:
image_row = self.images_df.iloc[j]
product_id = image_row["product_id"]
offset_row = self.offsets_df.loc[product_id]
# Read this product's data from the BSON file.
self.file.seek(offset_row["offset"])
item_data = self.file.read(offset_row["length"])
# Grab the image from the product.
item = bson.BSON.decode(item_data)
img_idx = image_row["img_idx"]
bson_img = item["imgs"][img_idx]["picture"]
# Load the image.
img = load_img(io.BytesIO(bson_img), target_size=self.target_size)
# Preprocess the image.
x = img_to_array(img)
x = preprocess_image(x)
#x = self.image_data_generator.random_transform(x)
#x = self.image_data_generator.standardize(x)
# Add the image and the label to the batch (one-hot encoded).
batch_x[i] = x
if self.with_labels:
batch_y[i, image_row["category_idx"]] = 1
if self.with_labels:
return batch_x, batch_y
else:
return batch_x
def rec_L(y_true, y_pred): s_flow = K.variable(np.array([1,0])) p = K.cast(K.equal(K.argmax(s_flow, axis=-1), K.argmax(y_pred, axis=-1)), K.floatx()) n = K.cast(K.not_equal(K.argmax(s_flow, axis=-1), K.argmax(y_pred, axis=-1)), K.floatx()) t = K.cast(K.equal(K.argmax(y_true, axis=-1), K.argmax(y_pred, axis=-1)), K.floatx()) f = K.cast(K.not_equal(K.argmax(y_true, axis=-1), K.argmax(y_pred, axis=-1)), K.floatx()) tn = t*n fp = f*p return K.sum(tn) / (K.sum(tn) + K.sum(fp))
def test_sparse_categorical_accuracy_correctness():
y_a = K.variable(np.random.randint(0, 7, (6,)), dtype=K.floatx())
y_b = K.variable(np.random.random((6, 7)), dtype=K.floatx())
# use one_hot embedding to convert sparse labels to equivalent dense labels
y_a_dense_labels = K.cast(K.one_hot(K.cast(y_a, dtype='int32'), num_classes=7),
dtype=K.floatx())
sparse_categorical_acc = metrics.sparse_categorical_accuracy(y_a, y_b)
categorical_acc = metrics.categorical_accuracy(y_a_dense_labels, y_b)
assert np.allclose(K.eval(sparse_categorical_acc), K.eval(categorical_acc))
def rec_S(y_true, y_pred): s_flow = K.variable(np.array([1,0])) p = K.cast(K.equal(K.argmax(s_flow, axis=-1), K.argmax(y_pred, axis=-1)), K.floatx()) n = K.cast(K.not_equal(K.argmax(s_flow, axis=-1), K.argmax(y_pred, axis=-1)), K.floatx()) t = K.cast(K.equal(K.argmax(y_true, axis=-1), K.argmax(y_pred, axis=-1)), K.floatx()) f = K.cast(K.not_equal(K.argmax(y_true, axis=-1), K.argmax(y_pred, axis=-1)), K.floatx()) tp = t*p fn = f*n return K.sum(tp) / (K.sum(tp) + K.sum(fn))
def test_setfloatx_correct_values(self):
# Keep track of the old value
old_floatx = floatx()
# Check correct values
for value in ['float16', 'float32', 'float64']:
set_floatx(value)
assert floatx() == value
# Restore old value
set_floatx(old_floatx)
def test_embedding():
layer_test(Embedding,
kwargs={'output_dim': 4, 'input_dim': 10, 'input_length': 2},
input_shape=(3, 2),
input_dtype='int32',
expected_output_dtype=K.floatx())
layer_test(Embedding,
kwargs={'output_dim': 4, 'input_dim': 10, 'mask_zero': True},
input_shape=(3, 2),
input_dtype='int32',
expected_output_dtype=K.floatx())
def call(self, x, mask=None):
if mask is not None:
mask = K.cast(mask, K.floatx())
mask = K.expand_dims(mask, axis=-1)
s = K.sum(mask, axis=1)
if K.equal(s, K.zeros_like(s)) is None:
return K.mean(x, axis=1)
else:
return K.cast(K.sum(x * mask, axis=1) / K.sqrt(s), K.floatx())
else:
return K.sum(x, axis=1)/K.sqrt(len(x))
def test_setfloatx_incorrect_values(self):
# Keep track of the old value
old_floatx = floatx()
# Try some incorrect values
initial = floatx()
for value in ['', 'beerfloat', 123]:
with pytest.raises(Exception):
set_floatx(value)
assert floatx() == initial
# Restore old value
set_floatx(old_floatx)
def softmax(x, axis, mask=None):
if mask is None:
mask = K.constant(True)
mask = K.cast(mask, K.floatx())
if K.ndim(x) is K.ndim(mask) + 1:
mask = K.expand_dims(mask)
m = K.max(x, axis=axis, keepdims=True)
e = K.exp(x - m) * mask
s = K.sum(e, axis=axis, keepdims=True)
s += K.cast(K.cast(s < K.epsilon(), K.floatx()) * K.epsilon(), K.floatx())
return e / s
def call(self, x, mask=None):
num = self.input_length
if mask:
num = mask.sum(-1,keepdims=True)
num = K.cast(num,K.floatx())
mask = K.expand_dims(mask,-1)
_x = x*K.cast(mask,K.floatx())
else:
_x = x
if not self.ave:
num = 1
return K.sum(_x,1)/num
def call(self, inputs, mask=None):
def norm_scale(a):
return (a + 1) / 2
x, scale_black, scale_white, shift = inputs
scale_black = norm_scale(scale_black)
black_selection = K.cast(x < 0.5, K.floatx())
white_selection = K.cast(x >= 0.5, K.floatx())
black_scaled = x*black_selection*(1 - scale_black*self.scale_factor)
white_scaled = x*white_selection*(1 - scale_white*self.scale_factor)
scaled = black_scaled + white_scaled
return scaled + self.shift_factor*shift
def test_layer_call():
"""Test keras.layers.core.Layer.__call__"""
nb_samples, input_dim, output_dim = 3, 10, 5
layer = Dense(output_dim, input_dim=input_dim)
W = np.asarray(K.eval(layer.W)).astype(K.floatx())
X = K.placeholder(ndim=2)
Y = layer(X)
F = K.function([X], [Y])
x = np.ones((nb_samples, input_dim)).astype(K.floatx())
y = F([x])[0].astype(K.floatx())
t = np.dot(x, W).astype(K.floatx())
assert_allclose(t, y, rtol=.2)
def get_w(self, x, mask=None):
input_shape = K.int_shape(x)
features_dim = self.features_dim
step_dim = input_shape[1]
eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)), K.reshape(self.W, (features_dim, 1))), (-1, step_dim))
if self.bias:
eij += self.b[:input_shape[1]]
eij = K.tanh(eij)
a = K.exp(eij)
if mask is not None:
a *= K.cast(mask, K.floatx())
a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())
return a
def call(self, x, mask=None):
e = K.dot(x, self.W)
if self.bias:
e += self.b
e = K.tanh(e)
e = K.reshape(K.dot(e, self.U), (-1, self.timesteps))
a = K.exp(e)
if mask is not None:
a *= K.cast(mask, K.floatx())
a_weights = a / K.cast(K.sum(a, axis=-1, keepdims=True) + K.epsilon(), K.floatx())
weighted_output = x * K.expand_dims(a_weights, axis=-1)
return [K.mean(weighted_output, axis=1), a_weights]
def loss(y_true, y_pred):
from plasma.conf import conf
fac = MaxHingeTarget.fac
#overall_fac = np.prod(np.array(K.shape(y_pred)[1:]).astype(np.float32))
overall_fac = K.prod(K.cast(K.shape(y_pred)[1:],K.floatx()))
max_val = K.max(y_pred,axis=-2) #temporal axis!
max_val1 = K.repeat(max_val,K.shape(y_pred)[-2])
mask = K.cast(K.equal(max_val1,y_pred),K.floatx())
y_pred1 = mask * y_pred + (1-mask) * y_true
weight_mask = K.mean(y_true,axis=-1)
weight_mask = K.cast(K.greater(weight_mask,0.0),K.floatx()) #positive label!
weight_mask = fac*weight_mask + (1 - weight_mask)
#return weight_mask*squared_hinge(y_true,y_pred1)
return conf['model']['loss_scale_factor']*overall_fac*weight_mask*hinge(y_true,y_pred1)
def next(self):
"""For python 2.x.
# Returns
The next batch.
"""
with self.lock:
index_array, current_index, current_batch_size = next(
self.index_generator)
# The transformation of images is not under thread lock
# so it can be done in parallel
batch_x = np.zeros((current_batch_size,) +
self.image_shape, dtype=K.floatx())
grayscale = self.color_mode == 'grayscale'
# build batch of image data
for i, j in enumerate(index_array):
fname = self.filenames[j]
img = load_img(os.path.join(self.directory, fname),
grayscale=grayscale,
target_size=self.target_size)
x = img_to_array(img, data_format=self.data_format)
x = self.image_data_generator.random_transform(x)
x = self.image_data_generator.standardize(x)
batch_x[i] = x
# optionally save augmented images to disk for debugging purposes
if self.save_to_dir:
for i in range(current_batch_size):
img = array_to_img(batch_x[i], self.data_format, scale=True)
fname = '{prefix}_{index}_{hash}.{format}'.format(prefix=self.save_prefix,
index=current_index + i,
hash=np.random.randint(
1e4),
format=self.save_format)
img.save(os.path.join(self.save_to_dir, fname))
# build batch of labels
if self.class_mode == 'input':
batch_y = batch_x.copy()
elif self.class_mode == 'sparse':
batch_y = self.classes[index_array]
elif self.class_mode == 'binary':
batch_y = self.classes[index_array].astype(K.floatx())
elif self.class_mode == 'categorical':
batch_y = np.zeros(
(len(batch_x), self.num_class), dtype=K.floatx())
for i, label in enumerate(self.classes[index_array]):
batch_y[i, label] = 1.
else:
return batch_x
return batch_x, batch_y
def __init__(self, folder, transforms=None, shuffle=True, batch_size=32,
seed=None):
if transforms is None:
transforms = []
paths, labels, label_names = get_paths_with_labels(folder)
self.n = len(paths)
self.paths = np.asarray(paths)
self.labels = to_categorical(labels, num_classes=len(label_names))
self.label_names = label_names
self.shuffle = shuffle
self.seed = seed
self.transform = get_transform(*transforms)
self.batch_size = batch_size
self.batch_idx = 0
self.num_batches_so_far = -1
self.indices = np.arange(self.n)
# calculate output shape by loading an image and
# passing it through the functions
img = imread(paths[0])
img = np.asarray(img, dtype=K.floatx())
self.output_shape = self.transform(img).shape
if K.image_data_format() == 'channels_first':
self.output_shape = (self.output_shape[2],
self.output_shape[0],
self.output_shape[1])
self.reset()
def get_augmented_generator(gen, transform_func, new_size=None):
"""
Yield batches from a given generator with specified transformations.
"""
for X_batch, y_batch in gen:
if new_size is None:
X_batch_new = np.zeros(X_batch.shape, dtype=K.floatx())
else:
X_batch_new = np.zeros(
(X_batch.shape[0], new_size, new_size, 3), dtype=K.floatx())
for idx in xrange(len(X_batch)):
X_batch_new[idx] = transform_func(X_batch[idx])
yield X_batch_new, y_batch
def _sample_weights(y, mask=None):
"""Compute sample weights."""
if mask is None:
weights = K.ones_like(y)
else:
weights = 1 - K.cast(K.equal(y, mask), K.floatx())
return weights
def main():
"""Generate different test models and save them to the given directory."""
if len(sys.argv) != 3:
print('usage: [model name] [destination file path]')
sys.exit(1)
else:
model_name = sys.argv[1]
dest_path = sys.argv[2]
get_model_functions = {
'small': get_test_model_small,
'sequential': get_test_model_sequential,
'full': get_test_model_full
}
if not model_name in get_model_functions:
print('unknown model name: ', model_name)
sys.exit(2)
assert K.backend() == "tensorflow"
assert K.floatx() == "float32"
assert K.image_data_format() == 'channels_last'
np.random.seed(0)
model_func = get_model_functions[model_name]
model = model_func()
model.save(dest_path, include_optimizer=False)
# Make sure models can be loaded again,
# see https://github.com/fchollet/keras/issues/7682
model = load_model(dest_path)
print(model.summary())
def img_to_array(img, data_format=None):
"""Converts a PIL Image instance to a Numpy array.
# Arguments
img: PIL Image instance.
data_format: Image data format.
# Returns
A 3D Numpy array.
# Raises
ValueError: if invalid `img` or `data_format` is passed.
"""
if data_format is None:
data_format = K.image_data_format()
if data_format not in {'channels_first', 'channels_last'}:
raise ValueError('Unknown data_format: ', data_format)
# Numpy array x has format (height, width, channel)
# or (channel, height, width)
# but original PIL image has format (width, height, channel)
x = np.asarray(img, dtype=K.floatx())
if len(x.shape) == 3:
if data_format == 'channels_first':
x = x.transpose(2, 0, 1)
elif len(x.shape) == 2:
if data_format == 'channels_first':
x = x.reshape((1, x.shape[0], x.shape[1]))
else:
x = x.reshape((x.shape[0], x.shape[1], 1))
else:
raise ValueError('Unsupported image shape: ', x.shape)
return x
def _loss_tensor(y_true, y_pred):
max_val = K.max(y_pred,axis=-2) #temporal axis!
max_val = K.repeat(max_val,K.shape(y_pred)[-2])
print(K.eval(max_val))
mask = K.cast(K.equal(max_val,y_pred),K.floatx())
y_pred = mask * y_pred + (1-mask) * y_true
return squared_hinge(y_true,y_pred)
def call(self, x, mask=None):
if hasattr(x, '_keras_shape'):
input_shape = x._keras_shape
else:
input_shape = self._input_shape
#import pdb
#pdb.set_trace()
#if self.last_two is not None:
# last2 = self.last_two
#else:
# input_shape = x._keras_shape
# last2 = input_shape[-2:]
#out_shape = K.shape(x)[:-2]
x = K.reshape(x, (-1,) + input_shape[-2:]) # (batch * d1 * ... * dn-2, dn-1, dn)
if mask is not None:
mask_shape = (K.shape(x)[0], -1)
mask = K.reshape(mask, mask_shape) # give it the same first dim
y = self.layer.call(x, mask)
#try:
#output_shape = self.get_output_shape_for(K.shape(x))
#except:
output_shape = self.get_output_shape_for(input_shape)
#import pdb
#pdb.set_trace()
return K.cast(K.reshape(y, output_shape), K.floatx())
def test_model(model, x_test, y_test, batch_size=100):
"""Run the benchmarks for the given model
:param model:
:param x_test: MNIST images scaled to [0, 1]
:param y_test: MNIST labels, raw values, not one-hot vectors
:param batch_size: batch size to use for evaluation
:return: None
"""
rng = np.random.RandomState(0)
classifier.load_weights('hw2\mnist_cnn.h5')
baseline_score = 0
correct_score = 0
ssim_score = 0
N = len(x_test)
assert N % batch_size == 0, 'N should be divisible by batch_size'
num_batches = N // batch_size
for i in range(num_batches):
imgs_orig = x_test[batch_size * i:batch_size * (i + 1)].astype(
K.floatx())
labels = y_test[batch_size * i:batch_size * (i + 1)]
# Create corruption masks
masks = []
for _ in range(batch_size):
# Choose square size
s = rng.randint(7, 15)
# Choose top-left corner position
x = rng.randint(0, 29 - s)
y = rng.randint(0, 29 - s)
mask = np.zeros(imgs_orig.shape[1:], dtype=np.bool)
# Set mask area
mask[y:y + s, x:x + s] = True
masks.append(mask)
masks = np.stack(masks)
# Add channel dimension
channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
imgs_orig = np.expand_dims(imgs_orig, channel_dim)
masks = np.expand_dims(masks, channel_dim)
# Generate corrupted versions
imgs_corrupted = imgs_orig.copy()
imgs_corrupted[masks] = 1.
# Generate restored images
imgs_restored = model.predict_on_batch(imgs_corrupted)
predicted_labels_orig = classifier.predict_on_batch(
_preprocess_for_classifier(imgs_orig)).argmax(axis=-1).astype(
labels.dtype)
predicted_labels_restored = classifier.predict_on_batch(
_preprocess_for_classifier(imgs_restored)).argmax(axis=-1).astype(
labels.dtype)
# Calculate classifier score:
# baseline corresponds to the original samples which the classifier is able to correctly predict
baseline = labels == predicted_labels_orig
# Since the classifier is NOT 100% accurate, we ignore the prediction results
# from the original samples which were misclassified by masking it using the baseline.
correct = (labels == predicted_labels_restored) & baseline
baseline_score += int(baseline.sum())
correct_score += int(correct.sum())
# Compute SSIM over the uncorrupted pixels
imgs_orig[masks] = 0.
imgs_restored[masks] = 0.
imgs_orig = imgs_orig.squeeze()
imgs_restored = imgs_restored.squeeze()
for j in range(batch_size):
ssim_score += ssim(imgs_orig[j], imgs_restored[j])
classifier_score = correct_score / baseline_score
ssim_score /= N
print('Classifier score: {:.2f}\nSSIM score: {:.2f}'.format(
100 * classifier_score, 100 * ssim_score))
def visualize_cam_with_losses(input_tensor,
losses,
seed_input,
penultimate_layer,
grad_modifier=None):
"""Generates a gradient based class activation map (CAM) by using positive gradients of `input_tensor`
with respect to weighted `losses`.
For details on grad-CAM, see the paper:
[Grad-CAM: Why did you say that? Visual Explanations from Deep Networks via Gradient-based Localization]
(https://arxiv.org/pdf/1610.02391v1.pdf).
Unlike [class activation mapping](https://arxiv.org/pdf/1512.04150v1.pdf), which requires minor changes to
network architecture in some instances, grad-CAM has a more general applicability.
Compared to saliency maps, grad-CAM is class discriminative; i.e., the 'cat' explanation exclusively highlights
cat regions and not the 'dog' region and vice-versa.
Args:
input_tensor: An input tensor of shape: `(samples, channels, image_dims...)` if `image_data_format=
channels_first` or `(samples, image_dims..., channels)` if `image_data_format=channels_last`.
losses: List of ([Loss](vis.losses#Loss), weight) tuples.
seed_input: The model input for which activation map needs to be visualized.
penultimate_layer: The pre-layer to `layer_idx` whose feature maps should be used to compute gradients
with respect to filter output.
grad_modifier: gradient modifier to use. See [grad_modifiers](vis.grad_modifiers.md). If you don't
specify anything, gradients are unchanged (Default value = None)
Returns:
The heatmap image indicating the `seed_input` regions whose change would most contribute towards minimizing the
weighted `losses`.
"""
penultimate_output = penultimate_layer.output
opt = Optimizer(input_tensor,
losses,
wrt_tensor=penultimate_output,
norm_grads=False)
_, grads, penultimate_output_value = opt.minimize(
seed_input, max_iter=1, grad_modifier=grad_modifier, verbose=False)
# For numerical stability. Very small grad values along with small penultimate_output_value can cause
# w * penultimate_output_value to zero out, even for reasonable fp precision of float32.
grads = grads / (np.max(grads) + K.epsilon())
# Average pooling across all feature maps.
# This captures the importance of feature map (channel) idx to the output.
channel_idx = 1 if K.image_data_format() == 'channels_first' else -1
other_axis = np.delete(np.arange(len(grads.shape)), channel_idx)
weights = np.mean(grads, axis=tuple(other_axis))
# Generate heatmap by computing weight * output over feature maps
output_dims = utils.get_img_shape(penultimate_output)[2:]
heatmap = np.zeros(shape=output_dims, dtype=K.floatx())
for i, w in enumerate(weights):
if channel_idx == -1:
heatmap += w * penultimate_output_value[0, ..., i]
else:
heatmap += w * penultimate_output_value[0, i, ...]
# ReLU thresholding to exclude pattern mismatch information (negative gradients).
heatmap = np.maximum(heatmap, 0)
# The penultimate feature map size is definitely smaller than input image.
input_dims = utils.get_img_shape(input_tensor)[2:]
# Figure out the zoom factor.
zoom_factor = [
i / (j * 1.0) for i, j in iter(zip(input_dims, output_dims))
]
heatmap = zoom(heatmap, zoom_factor)
# Normalize and create heatmap.
heatmap = utils.normalize(heatmap)
return np.uint8(cm.jet(heatmap)[..., :3] * 255)
def run(argv=None):
parser = argparse.ArgumentParser()
parser.add_argument("-tr",
"--train",
dest="train_path",
type=str,
metavar='<str>',
required=True,
help="The path to the training set")
parser.add_argument("-tu",
"--tune",
dest="dev_path",
type=str,
metavar='<str>',
required=True,
help="The path to the development set")
parser.add_argument("-ts",
"--test",
dest="test_path",
type=str,
metavar='<str>',
required=True,
help="The path to the test set")
parser.add_argument("-o",
"--out-dir",
dest="out_dir_path",
type=str,
metavar='<str>',
required=True,
help="The path to the output directory")
parser.add_argument(
"-p",
"--prompt",
dest="prompt_id",
type=int,
metavar='<int>',
required=False,
help="Promp ID for ASAP dataset. '0' means all prompts.")
parser.add_argument("-t",
"--type",
dest="model_type",
type=str,
metavar='<str>',
default='regp',
help="Model type (reg|regp|breg|bregp) (default=regp)")
parser.add_argument(
"-u",
"--rec-unit",
dest="recurrent_unit",
type=str,
metavar='<str>',
default='lstm',
help="Recurrent unit type (lstm|gru|simple) (default=lstm)")
parser.add_argument(
"-a",
"--algorithm",
dest="algorithm",
type=str,
metavar='<str>',
default='rmsprop',
help=
"Optimization algorithm (rmsprop|sgd|adagrad|adadelta|adam|adamax) (default=rmsprop)"
)
parser.add_argument("-l",
"--loss",
dest="loss",
type=str,
metavar='<str>',
default='mse',
help="Loss function (mse|mae) (default=mse)")
parser.add_argument("-e",
"--embdim",
dest="emb_dim",
type=int,
metavar='<int>',
default=50,
help="Embeddings dimension (default=50)")
parser.add_argument(
"-c",
"--cnndim",
dest="cnn_dim",
type=int,
metavar='<int>',
default=0,
help="CNN output dimension. '0' means no CNN layer (default=0)")
parser.add_argument("-w",
"--cnnwin",
dest="cnn_window_size",
type=int,
metavar='<int>',
default=3,
help="CNN window size. (default=3)")
parser.add_argument(
"-r",
"--rnndim",
dest="rnn_dim",
type=int,
metavar='<int>',
default=300,
help="RNN dimension. '0' means no RNN layer (default=300)")
parser.add_argument("-b",
"--batch-size",
dest="batch_size",
type=int,
metavar='<int>',
default=32,
help="Batch size (default=32)")
parser.add_argument("-v",
"--vocab-size",
dest="vocab_size",
type=int,
metavar='<int>',
default=4000,
help="Vocab size (default=4000)")
parser.add_argument(
"--aggregation",
dest="aggregation",
type=str,
metavar='<str>',
default='mot',
help=
"The aggregation method for regp and bregp types (mot|attsum|attmean) (default=mot)"
)
parser.add_argument(
"--dropout",
dest="dropout_prob",
type=float,
metavar='<float>',
default=0.5,
help=
"The dropout probability. To disable, give a negative number (default=0.5)"
)
parser.add_argument(
"--vocab-path",
dest="vocab_path",
type=str,
metavar='<str>',
help="(Optional) The path to the existing vocab file (*.pkl)")
parser.add_argument("--skip-init-bias",
dest="skip_init_bias",
action='store_true',
help="Skip initialization of the last layer bias")
parser.add_argument(
"--emb",
dest="emb_path",
type=str,
metavar='<str>',
help="The path to the word embeddings file (Word2Vec format)")
parser.add_argument("--epochs",
dest="epochs",
type=int,
metavar='<int>',
default=100,
help="Number of epochs (default=50)")
parser.add_argument(
"--maxlen",
dest="maxlen",
type=int,
metavar='<int>',
default=0,
help=
"Maximum allowed number of words during training. '0' means no limit (default=0)"
)
parser.add_argument("--seed",
dest="seed",
type=int,
metavar='<int>',
default=1234,
help="Random seed (default=1234)")
## dsv
parser.add_argument("--min-word-freq",
dest="min_word_freq",
type=int,
metavar='<int>',
default=2,
help="Min word frequency")
parser.add_argument("--stack",
dest="stack",
type=float,
metavar='<float>',
default=1,
help="how deep to stack core RNN")
parser.add_argument("--skip-emb-preload",
dest="skip_emb_preload",
action='store_true',
help="Skip preloading embeddings")
parser.add_argument("--tokenize-old",
dest="tokenize_old",
action='store_true',
help="use old tokenizer")
parser.add_argument("-ar",
"--abs-root",
dest="abs_root",
type=str,
metavar='<str>',
required=False,
help="Abs path to root directory")
parser.add_argument("-ad",
"--abs-data",
dest="abs_data",
type=str,
metavar='<str>',
required=False,
help="Abs path to data directory")
parser.add_argument("-ao",
"--abs-out",
dest="abs_out",
type=str,
metavar='<str>',
required=False,
help="Abs path to output directory")
##
if argv is None:
args = parser.parse_args()
else:
args = parser.parse_args(argv)
setattr(args, 'abs_emb_path', args.abs_root + args.emb_path)
out_dir = args.out_dir_path
U.mkdir_p(out_dir + '/preds')
U.set_logger(out_dir)
U.print_args(args)
assert args.model_type in {'reg', 'regp', 'breg', 'bregp', 'rwa'}
assert args.algorithm in {
'rmsprop', 'sgd', 'adagrad', 'adadelta', 'adam', 'adamax'
}
assert args.loss in {'mse', 'mae', 'kappa', 'soft_kappa'}
assert args.recurrent_unit in {'lstm', 'gru', 'simple', 'rwa'}
assert args.aggregation in {'mot', 'attsum', 'attmean'}
if args.seed > 0:
np.random.seed(args.seed)
if args.tokenize_old:
asap_reader.token = 0
logger.info('using OLD tokenizer!')
if args.prompt_id >= 0:
from deepats.asap_evaluator import Evaluator
import deepats.asap_reader as dataset
else:
raise NotImplementedError
###############################################################################################################################
## Prepare data
#
emb_words = None
if not args.skip_emb_preload: #if args.emb_path:
from deepats.w2vEmbReader import W2VEmbReader as EmbReader
logger.info('Loading embedding vocabulary...')
emb_reader = EmbReader(args.emb_path, emb_dim=args.emb_dim)
emb_words = emb_reader.load_words()
from keras.preprocessing import sequence
# data_x is a list of lists
(train_x, train_y, train_pmt), (dev_x, dev_y, dev_pmt), (
test_x, test_y, test_pmt
), vocab, vocab_size, overal_maxlen, num_outputs = dataset.get_data(
(args.train_path, args.dev_path, args.test_path),
args.prompt_id,
args.vocab_size,
args.maxlen,
tokenize_text=True,
to_lower=True,
sort_by_len=False,
vocab_path=args.vocab_path,
min_word_freq=args.min_word_freq,
emb_words=emb_words)
# Dump vocab
with open(out_dir + '/vocab.pkl', 'wb') as vocab_file:
pk.dump(vocab, vocab_file)
if args.recurrent_unit == 'rwa':
setattr(args, 'model_type', 'rwa')
# Pad sequences for mini-batch processing
if args.model_type in {'breg', 'bregp', 'rwa'}:
assert args.rnn_dim > 0
train_x = sequence.pad_sequences(train_x, maxlen=overal_maxlen)
dev_x = sequence.pad_sequences(dev_x, maxlen=overal_maxlen)
test_x = sequence.pad_sequences(test_x, maxlen=overal_maxlen)
#assert args.recurrent_unit == 'lstm'
else:
train_x = sequence.pad_sequences(train_x)
dev_x = sequence.pad_sequences(dev_x)
test_x = sequence.pad_sequences(test_x)
###############################################################################################################################
## Some statistics
#
import keras.backend as K
train_y = np.array(train_y, dtype=K.floatx())
dev_y = np.array(dev_y, dtype=K.floatx())
test_y = np.array(test_y, dtype=K.floatx())
if args.prompt_id:
train_pmt = np.array(train_pmt, dtype='int32')
dev_pmt = np.array(dev_pmt, dtype='int32')
test_pmt = np.array(test_pmt, dtype='int32')
bincounts, mfs_list = U.bincounts(train_y)
with open('%s/bincounts.txt' % out_dir, 'w') as output_file:
for bincount in bincounts:
output_file.write(str(bincount) + '\n')
train_mean = train_y.mean(axis=0)
train_std = train_y.std(axis=0)
dev_mean = dev_y.mean(axis=0)
dev_std = dev_y.std(axis=0)
test_mean = test_y.mean(axis=0)
test_std = test_y.std(axis=0)
logger.info('Statistics:')
logger.info(' train_x shape: ' + str(np.array(train_x).shape))
logger.info(' dev_x shape: ' + str(np.array(dev_x).shape))
logger.info(' test_x shape: ' + str(np.array(test_x).shape))
logger.info(' train_y shape: ' + str(train_y.shape))
logger.info(' dev_y shape: ' + str(dev_y.shape))
logger.info(' test_y shape: ' + str(test_y.shape))
logger.info(' train_y mean: %s, stdev: %s, MFC: %s' %
(str(train_mean), str(train_std), str(mfs_list)))
logger.info(' overal_maxlen: ' + str(overal_maxlen))
# We need the dev and test sets in the original scale for evaluation
dev_y_org = dev_y.astype(dataset.get_ref_dtype())
test_y_org = test_y.astype(dataset.get_ref_dtype())
train_y_org = train_y.astype(dataset.get_ref_dtype())
# Convert scores to boundary of [0 1] for training and evaluation (loss calculation)
train_y = dataset.get_model_friendly_scores(train_y, train_pmt)
dev_y = dataset.get_model_friendly_scores(dev_y, dev_pmt)
test_y = dataset.get_model_friendly_scores(test_y, test_pmt)
###############################################################################################################################
## Optimizaer algorithm
#
from deepats.optimizers import get_optimizer
#optimizer = get_optimizer(args)
from keras import optimizers
## RMS-PROP
#optimizer = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-6, clipnorm=10)
optimizer = optimizers.RMSprop(lr=0.0018,
rho=0.88,
epsilon=1e-6,
clipnorm=10)
#optimizer = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-8, clipnorm=10)
#optimizer = optimizers.RMSprop(lr=0.004, rho=0.85, epsilon=1e-6, clipnorm=10)# best 2.1 (RWA)
#optimizer = optimizers.RMSprop(lr=0.0025, rho=0.8, epsilon=1e-8, clipnorm=10) # best 2.1 (RWA)
#optimizer = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-8, clipnorm=10) # best 2.3 (RWA)
#optimizer = optimizers.RMSprop(lr=0.0025, rho=0.88, epsilon=1e-8, clipnorm=10) # best 2.3 (RWA)
#optimizer = optimizers.RMSprop(lr=0.004, rho=0.85, epsilon=1e-8, clipnorm=10) # best 2.10 (RWA)
## OTHER METHODS
#optimizer = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=10)
#optimizer = optimizers.Adam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-06, clipnorm=10)
#optimizer = optimizers.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=10)
#optimizer = optimizers.SGD(lr=0.05, momentum=0, decay=0.0, nesterov=False, clipnorm=10)
#optimizer = optimizers.Adagrad(lr=0.03, epsilon=1e-08, clipnorm=10)
#optimizer = optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=1e-06, clipnorm=10)
###############################################################################################################################
## Building model
#
from deepats.models import create_model
N = 1
L = 0
if args.loss == 'mse':
loss = 'mean_squared_error'
#metric = 'mean_absolute_error'; metric_name = metric
metric = kappa_metric
metric_name = 'kappa_metric'
elif args.loss == 'mae':
loss = 'mean_absolute_error'
#metric = 'mean_squared_error'; metric_name = metric
metric = kappa_metric
metric_name = 'kappa_metric'
elif args.loss == 'kappa':
loss = kappa_loss
metric = kappa_metric
metric_name = 'kappa_metric'
########################################################
if N > 1:
train_y_hot = np.eye(N)[train_y_org - L].astype('float32')
train_y = train_y_hot
########################################################
model = create_model(args, train_y.mean(axis=0), overal_maxlen, vocab)
############################################
'''
# test yaml serialization/de-serialization
yaml = model.to_yaml()
print yaml
from deepats.my_layers import MeanOverTime
from deepats.rwa import RWA
model = model_from_yaml(yaml, custom_objects={'MeanOverTime': MeanOverTime, 'RWA':RWA})
'''
############################################
model.compile(loss=loss, optimizer=optimizer, metrics=[metric])
print(model.summary())
###############################################################################################################################
## Plotting model
#
# from keras.utils.visualize_util import plot
# plot(model, to_file = out_dir + '/model.png')
###############################################################################################################################
## Save model architecture
#
logger.info('Saving model architecture')
with open(out_dir + '/model_arch.json', 'w') as arch:
arch.write(model.to_json(indent=2))
logger.info(' Done')
###############################################################################################################################
## Evaluator
#
evl = Evaluator(dataset, args.prompt_id, out_dir, dev_x, test_x, dev_y,
test_y, dev_y_org, test_y_org, N, L)
###############################################################################################################################
## Training
#
logger.info(
'----------------------------------------------------------------')
logger.info('Initial Evaluation:')
evl.evaluate(model, -1, print_info=True)
total_train_time = 0
total_eval_time = 0
for ii in range(args.epochs):
# Training
t0 = time()
train_history = model.fit(train_x,
train_y,
batch_size=args.batch_size,
epochs=1,
verbose=0)
tr_time = time() - t0
total_train_time += tr_time
# Evaluate
t0 = time()
evl.evaluate(model, ii)
evl_time = time() - t0
total_eval_time += evl_time
# Print information
train_loss = train_history.history['loss'][0]
train_metric = train_history.history[metric_name][0]
logger.info('Epoch %d, train: %is, evaluation: %is' %
(ii, tr_time, evl_time))
logger.info('[Train] loss: %.4f, metric: %.4f' %
(train_loss, train_metric))
evl.print_info()
###############################################################################################################################
## Summary of the results
#
logger.info('Training: %i seconds in total' % total_train_time)
logger.info('Evaluation: %i seconds in total' % total_eval_time)
evl.print_final_info()
def test_sparse_metrics():
for metric in all_sparse_metrics:
y_a = K.variable(np.random.randint(0, 7, (6,)), dtype=K.floatx())
y_b = K.variable(np.random.random((6, 7)), dtype=K.floatx())
assert K.eval(metric(y_a, y_b)).shape == (6,)
def masked_loss_function(y_true, y_pred):
mask = K.cast(K.not_equal(y_true, mask_value), K.floatx())
return K.binary_crossentropy(y_true * mask, y_pred * mask)
def recursion(self,
input_energy,
mask=None,
go_backwards=False,
return_sequences=True,
return_logZ=True,
input_length=None):
"""Forward (alpha) or backward (beta) recursion
If `return_logZ = True`, compute the logZ, the normalization constant:
\[ Z = \sum_{y1, y2, y3} exp(-E) # energy
= \sum_{y1, y2, y3} exp(-(u1' y1 + y1' W y2 + u2' y2 + y2' W y3 + u3' y3))
= sum_{y2, y3} (exp(-(u2' y2 + y2' W y3 + u3' y3))
sum_{y1} exp(-(u1' y1' + y1' W y2))) \]
Denote:
\[ S(y2) := sum_{y1} exp(-(u1' y1 + y1' W y2)), \]
\[ Z = sum_{y2, y3} exp(log S(y2) - (u2' y2 + y2' W y3 + u3' y3)) \]
\[ logS(y2) = log S(y2) = log_sum_exp(-(u1' y1' + y1' W y2)) \]
Note that:
yi's are one-hot vectors
u1, u3: boundary energies have been merged
If `return_logZ = False`, compute the Viterbi's best path lookup table.
"""
chain_energy = self.chain_kernel
# shape=(1, F, F): F=num of output features. 1st F is for t-1, 2nd F for t
chain_energy = K.expand_dims(chain_energy, 0)
# shape=(B, F), dtype=float32
prev_target_val = K.zeros_like(input_energy[:, 0, :])
if go_backwards:
input_energy = K.reverse(input_energy, 1)
if mask is not None:
mask = K.reverse(mask, 1)
initial_states = [
prev_target_val,
K.zeros_like(prev_target_val[:, :1])
]
constants = [chain_energy]
if mask is not None:
mask = K.cast(mask, K.floatx())
mask2 = K.cast(
K.concatenate([mask, K.zeros_like(mask[:, :1])], axis=1),
K.floatx())
constants.append(mask2)
def _step(input_energy_i, states):
return self.step(input_energy_i, states, return_logZ)
target_val_last, target_val_seq, _ = K.rnn(_step,
input_energy,
initial_states,
constants=constants,
input_length=input_length,
unroll=self.unroll)
if return_sequences:
if go_backwards:
target_val_seq = K.reverse(target_val_seq, 1)
return target_val_seq
else:
return target_val_last
def masked_mse(y_true, y_pred):
mask_true = K.cast(K.not_equal(y_true, y_pred), K.floatx())
masked_squared_error = K.square(mask_true * (y_true - y_pred))
r = K.sum(masked_squared_error,
axis=-1) # / K.sum(mask_true, axis=-1)
return r
def _smooth_labels(y_true, label_smoothing):
num_classes = tf.cast(K.shape(y_true)[-1], dtype=K.floatx())
label_smoothing = K.constant(label_smoothing, dtype=K.floatx())
return y_true * (1.0 - label_smoothing) + label_smoothing / num_classes
def count_matches(a, b):
tmp = cmp.concatenate([a, b])
return cmp.sum(cmp.cast(cmp.all(tmp, -1), cmp.floatx()))
def _get_batches_of_transformed_samples(self, index_array) :
"""
Public function to fetch next batch.
# Returns
The next batch of images and labels.
"""
current_batch_size = index_array.shape[0]
# Image transformation is not under thread lock, so it can be done in
# parallel
batch_x = np.zeros((current_batch_size,) + self.image_shape,
dtype=K.floatx())
batch_steer = np.zeros((current_batch_size, 2,),
dtype=K.floatx())
batch_coll = np.zeros((current_batch_size, 2,),
dtype=K.floatx())
grayscale = self.color_mode == 'grayscale'
# Build batch of image data
for i, j in enumerate(index_array):
fname = self.filenames[j]
#x = img_utils.load_img(os.path.join(self.directory, fname),
# grayscale=grayscale,
# crop_size=self.crop_size,
# target_size=self.target_size)
#x = self.image_data_generator.random_transform(x)
#x = self.image_data_generator.standardize(x)
x = np.loadtxt(os.path.join(self.directory, fname), delimiter=',', dtype=np.float32)
#x[x==0]=200.
if x.shape == (500,700) :
x = x.reshape(500,700,1)
x = self.image_data_generator.random_transform(x)
x = self.image_data_generator.standardize(x)
batch_x[i] = x#x.reshape(500,700,1)
#batch_x[i] = x.reshape(500,700,1)
else :
continue
#if x.shape == (500,700) :
# batch_x[i] = x.reshape(500,700,1)
#else :
# continue
#print(type(x))
# Build batch of steering and collision data
if self.exp_type[index_array[i]] == 1:
# Steering experiment (t=1)
batch_steer[i,0] =1.0
batch_steer[i,1] = self.ground_truth[index_array[i]]
batch_coll[i] = np.array([1.0, 0.0])
#batch_steer[i] = self.ground_truth[index_array[i]]
#batch_coll[i] = np.array([1.0])
else:
# Collision experiment (t=0)
batch_steer[i] = np.array([0.0, 0.0])
batch_coll[i,0] = 0.0
batch_coll[i,1] = self.ground_truth[index_array[i]]
#batch_y = [batch_steer, batch_coll]
batch_y = [batch_steer]
return batch_x, batch_y
def step(self, inputs, states):
h_tm1 = states[0]
c_tm1 = states[1]
t_tm1 = states[2]
dp_mask = states[3]
rec_dp_mask = states[4]
# time related variables, simply add +1 to t for now...starting from 0
# need to find better way if asynchronous/irregular time input is desired
# such as slicing input where first index is time and using that instead.
t = t_tm1 + 1
# using timegate_constraint = 'non_neg' instead
# self.timegate_kernel = K.abs(self.timegate_kernel)
period = self.timegate_kernel[0]
shift = self.timegate_kernel[1]
r_on = self.timegate_kernel[2]
# modulo operation not implemented in Tensorflow backend, so write explicitly.
# a mod n = a - (n * int(a/n))
# phi = ((t - shift) % period) / period
phi = ((t - shift) - (period * ((t - shift) // period))) / period
# K.switch not consistent between Theano and Tensorflow backend, so write explicitly.
up = K.cast(K.less_equal(phi, r_on * 0.5), K.floatx()) * 2 * phi / r_on
mid = K.cast(K.less_equal(phi, r_on), K.floatx()) * \
K.cast(K.greater(phi, r_on * 0.5), K.floatx()) * (2 - (2 * phi / r_on))
end = K.cast(K.greater(phi, r_on), K.floatx()) * self.alpha * phi
k = up + mid + end
if self.implementation == 2:
z = K.dot(inputs * dp_mask[0], self.kernel)
z += K.dot(h_tm1 * rec_dp_mask[0], self.recurrent_kernel)
if self.use_bias:
z = K.bias_add(z, self.bias)
z0 = z[:, :self.units]
z1 = z[:, self.units:2 * self.units]
z2 = z[:, 2 * self.units:3 * self.units]
z3 = z[:, 3 * self.units:]
i = self.recurrent_activation(z0)
f = self.recurrent_activation(z1)
# intermediate cell update
c_hat = f * c_tm1 + i * self.activation(z2)
# final cell update
c = k * c_hat + (1 - k) * c_tm1
o = self.recurrent_activation(z3)
else:
if self.implementation == 0:
x_i = inputs[:, :self.units]
x_f = inputs[:, self.units:2 * self.units]
x_c = inputs[:, 2 * self.units:3 * self.units]
x_o = inputs[:, 3 * self.units:]
elif self.implementation == 1:
x_i = K.dot(inputs * dp_mask[0], self.kernel_i) + self.bias_i
x_f = K.dot(inputs * dp_mask[1], self.kernel_f) + self.bias_f
x_c = K.dot(inputs * dp_mask[2], self.kernel_c) + self.bias_c
x_o = K.dot(inputs * dp_mask[3], self.kernel_o) + self.bias_o
else:
raise ValueError('Unknown `implementation` mode.')
i = self.recurrent_activation(
x_i + K.dot(h_tm1 * rec_dp_mask[0], self.recurrent_kernel_i))
f = self.recurrent_activation(
x_f + K.dot(h_tm1 * rec_dp_mask[1], self.recurrent_kernel_f))
# intermediate cell update
c_hat = f * c_tm1 + i * self.activation(
x_c + K.dot(h_tm1 * rec_dp_mask[2], self.recurrent_kernel_c))
# final cell update
c = k * c_hat + (1 - k) * c_tm1
o = self.recurrent_activation(
x_o + K.dot(h_tm1 * rec_dp_mask[3], self.recurrent_kernel_o))
# intermediate hidden update
h_hat = o * self.activation(c_hat)
# final hidden update
h = k * h_hat + (1 - k) * h_tm1
if 0 < self.dropout + self.recurrent_dropout:
h._uses_learning_phase = True
return h, [h, c, t]
pickle_safe=True,
callbacks=[checkpointer])
#tf.reset_default_graph()
#print(weights[0][55//2])
tStart = time.time()
print('load model')
MdNamePath = 'FCN_b2a' #the model path
with open(MdNamePath + '.json') as f:
model = model_from_json(f.read())
model.load_weights(MdNamePath + '.hdf5')
model.summary()
pdb.set_trace()
print(K.floatx())
print('testing...')
for path in Test_Bone_lists: # Ex: /mnt/Nas/Corpus/TMHINT/Testing/Noisy/car_noise_idle_noise_60_mph/b4/1dB/TMHINT_12_10.wav
if path.split("/")[-1][-4:] == ".wav":
#pdb.set_trace()
S = path.split('/')
noise = S[-4]
speaker = S[-3]
dB = S[-2]
wave_name = S[-1]
rate, noisy = wavfile.read(path)
noisy = noisy.astype('float32')
if len(noisy.shape) == 2:
noisy = (noisy[:, 0] + noisy[:, 1]) / 2
def test_recursion():
####################################################
# test recursion
a = Input(shape=(32, ), name='input_a')
b = Input(shape=(32, ), name='input_b')
dense = Dense(16, name='dense_1')
a_2 = dense(a)
b_2 = dense(b)
merged = merge([a_2, b_2], mode='concat', name='merge')
c = Dense(64, name='dense_2')(merged)
d = Dense(5, name='dense_3')(c)
model = Model(input=[a, b], output=[c, d], name='model')
e = Input(shape=(32, ), name='input_e')
f = Input(shape=(32, ), name='input_f')
g, h = model([e, f])
# g2, h2 = model([e, f])
assert g._keras_shape == c._keras_shape
assert h._keras_shape == d._keras_shape
# test separate manipulation of different layer outputs
i = Dense(7, name='dense_4')(h)
final_model = Model(input=[e, f], output=[i, g], name='final')
assert len(final_model.inputs) == 2
assert len(final_model.outputs) == 2
assert len(final_model.layers) == 4
# we don't check names of first 2 layers (inputs) because
# ordering of same-level layers is not fixed
print('final_model layers:', [layer.name for layer in final_model.layers])
assert [layer.name
for layer in final_model.layers][2:] == ['model', 'dense_4']
print(model.compute_mask([e, f], [None, None]))
assert model.compute_mask([e, f], [None, None]) == [None, None]
print(final_model.get_output_shape_for([(10, 32), (10, 32)]))
assert final_model.get_output_shape_for([(10, 32), (10, 32)]) == [(10, 7),
(10, 64)]
# run recursive model
fn = K.function(final_model.inputs, final_model.outputs)
input_a_np = np.random.random((10, 32))
input_b_np = np.random.random((10, 32))
fn_outputs = fn([input_a_np, input_b_np])
assert [x.shape for x in fn_outputs] == [(10, 7), (10, 64)]
# test serialization
model_config = final_model.get_config()
print(json.dumps(model_config, indent=4))
recreated_model = Model.from_config(model_config)
fn = K.function(recreated_model.inputs, recreated_model.outputs)
input_a_np = np.random.random((10, 32))
input_b_np = np.random.random((10, 32))
fn_outputs = fn([input_a_np, input_b_np])
assert [x.shape for x in fn_outputs] == [(10, 7), (10, 64)]
####################################################
# test multi-input multi-output
j = Input(shape=(32, ), name='input_j')
k = Input(shape=(32, ), name='input_k')
m, n = model([j, k])
o = Input(shape=(32, ), name='input_o')
p = Input(shape=(32, ), name='input_p')
q, r = model([o, p])
assert n._keras_shape == (None, 5)
assert q._keras_shape == (None, 64)
s = merge([n, q], mode='concat', name='merge_nq')
assert s._keras_shape == (None, 64 + 5)
# test with single output as 1-elem list
multi_io_model = Model([j, k, o, p], [s])
fn = K.function(multi_io_model.inputs, multi_io_model.outputs)
fn_outputs = fn([
np.random.random((10, 32)),
np.random.random((10, 32)),
np.random.random((10, 32)),
np.random.random((10, 32))
])
assert [x.shape for x in fn_outputs] == [(10, 69)]
# test with single output as tensor
multi_io_model = Model([j, k, o, p], s)
fn = K.function(multi_io_model.inputs, multi_io_model.outputs)
fn_outputs = fn([
np.random.random((10, 32)),
np.random.random((10, 32)),
np.random.random((10, 32)),
np.random.random((10, 32))
])
# note that the output of the K.function will still be a 1-elem list
assert [x.shape for x in fn_outputs] == [(10, 69)]
# test serialization
print('multi_io_model.layers:', multi_io_model.layers)
print('len(model.inbound_nodes):', len(model.inbound_nodes))
print('len(model.outbound_nodes):', len(model.outbound_nodes))
model_config = multi_io_model.get_config()
print(model_config)
print(json.dumps(model_config, indent=4))
recreated_model = Model.from_config(model_config)
fn = K.function(recreated_model.inputs, recreated_model.outputs)
fn_outputs = fn([
np.random.random((10, 32)),
np.random.random((10, 32)),
np.random.random((10, 32)),
np.random.random((10, 32))
])
# note that the output of the K.function will still be a 1-elem list
assert [x.shape for x in fn_outputs] == [(10, 69)]
config = model.get_config()
new_model = Model.from_config(config)
model.summary()
json_str = model.to_json()
new_model = model_from_json(json_str)
yaml_str = model.to_yaml()
new_model = model_from_yaml(yaml_str)
####################################################
# test invalid graphs
# input is not an Input tensor
j = Input(shape=(32, ), name='input_j')
j = Dense(32)(j)
k = Input(shape=(32, ), name='input_k')
m, n = model([j, k])
with pytest.raises(Exception):
invalid_model = Model([j, k], [m, n])
# disconnected graph
j = Input(shape=(32, ), name='input_j')
k = Input(shape=(32, ), name='input_k')
m, n = model([j, k])
with pytest.raises(Exception) as e:
invalid_model = Model([j], [m, n])
# redudant outputs
j = Input(shape=(32, ), name='input_j')
k = Input(shape=(32, ), name='input_k')
m, n = model([j, k])
# this should work lol
# TODO: raise a warning
invalid_model = Model([j, k], [m, n, n])
# redundant inputs
j = Input(shape=(32, ), name='input_j')
k = Input(shape=(32, ), name='input_k')
m, n = model([j, k])
with pytest.raises(Exception):
invalid_model = Model([j, k, j], [m, n])
# i have not idea what I'm doing: garbage as inputs/outputs
j = Input(shape=(32, ), name='input_j')
k = Input(shape=(32, ), name='input_k')
m, n = model([j, k])
with pytest.raises(Exception):
invalid_model = Model([j, k], [m, n, 0])
####################################################
# test calling layers/models on TF tensors
if K._BACKEND == 'tensorflow':
import tensorflow as tf
j = Input(shape=(32, ), name='input_j')
k = Input(shape=(32, ), name='input_k')
m, n = model([j, k])
tf_model = Model([j, k], [m, n])
# magic
j_tf = tf.placeholder(dtype=K.floatx())
k_tf = tf.placeholder(dtype=K.floatx())
m_tf, n_tf = tf_model([j_tf, k_tf])
assert not hasattr(m_tf, '_keras_shape')
assert not hasattr(n_tf, '_keras_shape')
assert K.int_shape(m_tf) == (None, 64)
assert K.int_shape(n_tf) == (None, 5)
# test merge
o_tf = merge([j_tf, k_tf], mode='concat', concat_axis=1)
# test tensor input
x = tf.placeholder(shape=(None, 2), dtype=K.floatx())
input_layer = InputLayer(input_tensor=x)
x = Input(tensor=x)
y = Dense(2)(x)
def call(self, x, mask=None):
if self.mask_zero:
return K.cast(
x.sum(axis=1) / mask.sum(axis=1, keepdims=True), K.floatx())
else:
return K.mean(x, axis=1)
def sphere_uniform_sample(shape, r_max): num_samples, dim = shape X = tf.random_normal(shape=shape, dtype=K.floatx()) X_norm = K.sqrt(K.sum(K.square(X), axis=-1, keepdims=True)) U = tf.random_uniform(shape=(num_samples, 1), dtype=K.floatx()) return r_max * U ** (1./dim) * X / X_norm
def _preprocess_numpy_input(x, data_format, mode, **kwargs):
"""Preprocesses a Numpy array encoding a batch of images.
# Arguments
x: Input array, 3D or 4D.
data_format: Data format of the image array.
mode: One of "caffe", "tf" or "torch".
- caffe: will convert the images from RGB to BGR,
then will zero-center each color channel with
respect to the ImageNet dataset,
without scaling.
- tf: will scale pixels between -1 and 1,
sample-wise.
- torch: will scale pixels between 0 and 1 and then
will normalize each channel with respect to the
ImageNet dataset.
# Returns
Preprocessed Numpy array.
"""
# backend, _, _, _ = get_submodules_from_kwargs(kwargs)
from keras import backend
if not issubclass(x.dtype.type, np.floating):
x = x.astype(backend.floatx(), copy=False)
if mode == 'tf':
x /= 127.5
x -= 1.
return x
if mode == 'torch':
x /= 255.
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
else:
if data_format == 'channels_first':
# 'RGB'->'BGR'
if x.ndim == 3:
x = x[::-1, ...]
else:
x = x[:, ::-1, ...]
else:
# 'RGB'->'BGR'
x = x[..., ::-1]
mean = [103.939, 116.779, 123.68]
std = None
# Zero-center by mean pixel
if data_format == 'channels_first':
if x.ndim == 3:
x[0, :, :] -= mean[0]
x[1, :, :] -= mean[1]
x[2, :, :] -= mean[2]
if std is not None:
x[0, :, :] /= std[0]
x[1, :, :] /= std[1]
x[2, :, :] /= std[2]
else:
x[:, 0, :, :] -= mean[0]
x[:, 1, :, :] -= mean[1]
x[:, 2, :, :] -= mean[2]
if std is not None:
x[:, 0, :, :] /= std[0]
x[:, 1, :, :] /= std[1]
x[:, 2, :, :] /= std[2]
else:
x[..., 0] -= mean[0]
x[..., 1] -= mean[1]
x[..., 2] -= mean[2]
if std is not None:
x[..., 0] /= std[0]
x[..., 1] /= std[1]
x[..., 2] /= std[2]
return x
def update_network(self, if_pretrain, use_average, current_time):
''' update Q network '''
if current_time - self.update_outdated < self.dic_agent_conf["UPDATE_PERIOD"]:
return
self.update_outdated = current_time
# prepare the samples
if if_pretrain:
gamma = self.dic_agent_conf["GAMMA_PRETRAIN"]
print("precision ", K.floatx())
else:
gamma = self.dic_agent_conf["GAMMA"]
dic_state_feature_arrays = {}
for feature_name in self.dic_traffic_env_conf["LIST_STATE_FEATURE"]:
dic_state_feature_arrays[feature_name] = []
Y = []
# get average state-action reward
if self.dic_agent_conf["SEPARATE_MEMORY"]:
self.average_reward = self._cal_average_separate(self.memory)
else:
self.average_reward = self._cal_average(self.memory)
# ================ sample memory ====================
if self.dic_agent_conf["SEPARATE_MEMORY"]:
for phase_i in range(self.num_phases):
for action_i in range(self.num_actions):
sampled_memory = self._sample_memory(
gamma=gamma,
with_priority=self.dic_agent_conf["PRIORITY_SAMPLING"],
memory=self.memory[phase_i][action_i],
if_pretrain=if_pretrain)
dic_state_feature_arrays, Y = self.get_sample(
sampled_memory, dic_state_feature_arrays, Y, gamma, current_time, use_average)
else:
sampled_memory = self._sample_memory(
gamma=gamma,
with_priority=self.dic_agent_conf["PRIORITY_SAMPLING"],
memory=self.memory,
if_pretrain=if_pretrain)
dic_state_feature_arrays, Y = self.get_sample(
sampled_memory, dic_state_feature_arrays, Y, gamma, current_time, use_average)
# ================ sample memory ====================
Xs = [np.array(dic_state_feature_arrays[feature_name]) for feature_name in self.dic_traffic_env_conf["LIST_STATE_FEATURE"]]
Y = np.array(Y)
sample_weight = np.ones(len(Y))
# shuffle the training samples, especially for different phases and actions
Xs, Y, _ = self._unison_shuffled_copies(Xs, Y, sample_weight)
if if_pretrain:
pickle.dump(Xs, open(os.path.join(self.path_set.PATH_TO_OUTPUT, "Xs.pkl"), "wb"))
pickle.dump(Y, open(os.path.join(self.path_set.PATH_TO_OUTPUT, "Y.pkl"), "wb"))
# ============================ training =======================================
self.train_network(Xs, Y, current_time, if_pretrain)
self.q_bar_outdated += 1
self.forget(if_pretrain=if_pretrain)
def fit(self, x,
augment=False,
rounds=1,
seed=None):
"""Fits the data generator to some sample data.
只有当featurewise_normalization和zca_whitening为True是才需要调用
# Arguments
x: Sample data. Should have rank 4.
In case of grayscale data,
the channels axis should have value 1, in case
of RGB data, it should have value 3, and in case
of RGBA data, it should have value 4.
augment: Boolean (default: False).
Whether to fit on randomly augmented samples.
rounds: Int (default: 1).
If using data augmentation (`augment=True`),
this is how many augmentation passes over the data to use.
seed: Int (default: None). Random seed.
"""
x = np.asarray(x, dtype=backend.floatx())
if x.ndim != 4:
raise ValueError('Input to `.fit()` should have rank 4. '
'Got array with shape: ' + str(x.shape))
if x.shape[self.channel_axis] not in {1, 3, 4}:
warnings.warn(
'Expected input to be images (as Numpy array) '
'following the data format convention "' +
self.data_format + '" (channels on axis ' +
str(self.channel_axis) + '), i.e. expected '
'either 1, 3 or 4 channels on axis ' +
str(self.channel_axis) + '. '
'However, it was passed an array with shape ' +
str(x.shape) + ' (' + str(x.shape[self.channel_axis]) +
' channels).')
if seed is not None:
np.random.seed(seed)
x = np.copy(x)
if augment:
ax = np.zeros(
tuple([rounds * x.shape[0]] + list(x.shape)[1:]),
dtype=backend.floatx())
for r in range(rounds):
for i in range(x.shape[0]):
ax[i + r * x.shape[0]] = self.random_transform(x[i])
x = ax
if self.featurewise_center:
self.mean = np.mean(x, axis=(0, self.row_axis, self.col_axis))
broadcast_shape = [1, 1, 1]
broadcast_shape[self.channel_axis - 1] = x.shape[self.channel_axis]
self.mean = np.reshape(self.mean, broadcast_shape)
x -= self.mean
if self.featurewise_std_normalization:
self.std = np.std(x, axis=(0, self.row_axis, self.col_axis))
broadcast_shape = [1, 1, 1]
broadcast_shape[self.channel_axis - 1] = x.shape[self.channel_axis]
self.std = np.reshape(self.std, broadcast_shape)
x /= (self.std + backend.epsilon())
if self.zca_whitening:
if scipy is None:
raise ImportError('Using zca_whitening requires SciPy. '
'Install SciPy.')
flat_x = np.reshape(
x, (x.shape[0], x.shape[1] * x.shape[2] * x.shape[3]))
sigma = np.dot(flat_x.T, flat_x) / flat_x.shape[0]
u, s, _ = scipy.linalg.svd(sigma)
s_inv = 1. / np.sqrt(s[np.newaxis] + self.zca_epsilon)
self.principal_components = (u * s_inv).dot(u.T)
def accuracy_round(y_true, y_pred):
y_true = (y_true + 1) / 2.0
y_pred = (y_pred + 1) / 2.0
equal = K.cast(K.equal(K.round(y_true), K.round(y_pred)), K.floatx())
acc = K.sum(equal) / K.cast(tf.size(equal), K.floatx())
return acc
def accuracy(y_true, y_pred):
equal = K.cast(K.equal(K.argmax(y_true, axis=-1), K.argmax(y_pred, axis=-1)), K.floatx())
acc = K.sum(equal) / K.cast(tf.size(equal), K.floatx())
return acc
def cartpole_loss(self, target, prediction):
r = K.cast(K.less_equal(target, -1e4), K.floatx())
return -K.mean(K.log(prediction) * (1-r) * target, axis=-1)
def test_graph_multiple_in_out_call():
"""Test keras.models.Graph.__call__ with multiple inputs"""
nb_samples, input_dim, output_dim = 3, 10, 5
model = Graph()
model.add_input('input1', input_shape=(input_dim, ))
model.add_input('input2', input_shape=(input_dim, ))
model.add_node(Dense(output_dim=output_dim, input_dim=input_dim),
inputs=['input1', 'input2'],
merge_mode='sum',
name='output',
create_output=True)
model.compile('sgd', {'output': 'mse'})
# test flat model
X1 = K.placeholder(ndim=2)
X2 = K.placeholder(ndim=2)
Y = model({'input1': X1, 'input2': X2})['output']
f = K.function([X1, X2], [Y])
x1 = np.ones((nb_samples, input_dim)).astype(K.floatx())
x2 = np.ones((nb_samples, input_dim)).astype(K.floatx()) * -2
y1 = f([x1, x2])[0].astype(K.floatx())
y2 = model.predict({'input1': x1, 'input2': x2})['output']
# results of __call__ should match model.predict
assert_allclose(y1, y2)
# test with single input, multiple outputs
model2 = Graph()
model2.add_input('input', input_shape=(input_dim, ))
model2.add_node(Dense(output_dim=output_dim, input_dim=input_dim),
input='input',
name='output1',
create_output=True)
model2.add_node(Dense(output_dim=output_dim, input_dim=input_dim),
input='input',
name='output2',
create_output=True)
model2.compile('sgd', {'output1': 'mse', 'output2': 'mse'})
# test flat model
X = K.placeholder(ndim=2)
Y = model2(X)
f = K.function([X], [Y['output1'], Y['output2']])
x = np.ones((nb_samples, input_dim)).astype(K.floatx())
out = f([x])
y1a = out[0].astype(K.floatx())
y1b = out[1].astype(K.floatx())
y2 = model2.predict({'input': x})
# results of __call__ should match model.predict
assert_allclose(y1a, y2['output1'])
assert_allclose(y1b, y2['output2'])
# test with multiple inputs, multiple outputs
model3 = Graph()
model3.add_input('input1', input_shape=(input_dim, ))
model3.add_input('input2', input_shape=(input_dim, ))
model3.add_shared_node(Dense(output_dim=output_dim, input_dim=input_dim),
inputs=['input1', 'input2'],
name='output',
outputs=['output1', 'output2'],
create_output=True)
model3.compile('sgd', {'output1': 'mse', 'output2': 'mse'})
# test flat model
Y = model3({'input1': X1, 'input2': X2})
f = K.function([X1, X2], [Y['output1'], Y['output2']])
x1 = np.ones((nb_samples, input_dim)).astype(K.floatx())
x2 = np.ones((nb_samples, input_dim)).astype(K.floatx()) * -2
out = f([x1, x2])
y1a = out[0].astype(K.floatx())
y1b = out[1].astype(K.floatx())
y2 = model3.predict({'input1': x1, 'input2': x2})
# results of __call__ should match model.predict
assert_allclose(y1a, y2['output1'])
assert_allclose(y1b, y2['output2'])
# calculate the total energy across all rows
total_energy = Lambda(lambda x: K.reshape(K.sum(x, axis=-1), (-1, 1)),
name='total_energy')(energies)
nb_features = 10
vspace_dim = 10
minibatch_featurizer = Lambda(minibatch_discriminator,
output_shape=minibatch_output_shape)
K_energy = Dense3D(nb_features, vspace_dim)(energies)
mbd_energy = Activation('tanh')(minibatch_featurizer(K_energy))
# # binary y/n if it is over the input energy
energy_well = Lambda(lambda x: K.abs(x[0] - x[1]))(
[total_energy, input_energy])
well_too_big = Lambda(lambda x: 10 * K.cast(x > 5, K.floatx()))(
energy_well)
# p = concatenate([features, energies, total_energy])
p = concatenate([
features,
scale(energies, 10),
scale(total_energy, 100),
# scale(input_energy, 100),
energy_well,
well_too_big,
mbd_energy
])
fake = Dense(1, activation='sigmoid', name='fakereal_output')(p)
discriminator_outputs = [fake, total_energy]
def compute_output_signature(self, input_spec):
output_shape = self.compute_output_shape(input_spec.shape.as_list())
output_dtype = (tf.int64
if self._output_mode == INT else backend.floatx())
return tf.TensorSpec(shape=output_shape, dtype=output_dtype)
def part_cls_acc(y_true, y_pred):
y_true = K.reshape(y_true, (-1, num_classes))
y_pred = K.reshape(y_pred, (-1, num_classes))
return K.cast(
K.equal(K.argmax(y_true, axis=-1), K.argmax(y_pred, axis=-1)),
K.floatx())
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr * (1. / (1. + self.decay * K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx()) + 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = K.maximum(self.total_steps - warmup_steps, 1)
decay_rate = (self.min_lr - lr) / decay_steps
lr = K.switch(
t <= warmup_steps,
lr * (t / warmup_steps),
lr + decay_rate * K.minimum(t - warmup_steps, decay_steps),
)
ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for (i, p) in enumerate(params)]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for (i, p) in enumerate(params)]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for (i, p) in enumerate(params)]
else:
vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params))]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
beta_2_t = K.pow(self.beta_2, t)
sma_inf = 2.0 / (1.0 - self.beta_2) - 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t / (1.0 - beta_2_t)
for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
m_corr_t = m_t / (1.0 - beta_1_t)
if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
v_corr_t = K.sqrt(vhat_t / (1.0 - beta_2_t))
self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t / (1.0 - beta_2_t))
r_t = K.sqrt((sma_t - 4.0) / (sma_inf - 4.0) *
(sma_t - 2.0) / (sma_inf - 2.0) *
sma_inf / sma_t)
p_t = K.switch(sma_t >= 5, r_t * m_corr_t / (v_corr_t + self.epsilon), m_corr_t)
if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
def accuracy_with_threshold(y_true, y_pred):
y_pred = K.cast(K.greater(y_pred, threshold), K.floatx())
return K.mean(K.equal(y_true, y_pred))
def load_source_model(models=['xception','resnet50'], \
pred_models=None, \
is_targeted=False, \
aug_or_rotate='aug', \
grad_dict=None, \
make_model=False, \
load_weights=True):
"""
Loads the models and set up grad_dict. grad_dict is a dictionary
indexed by model names, and the values are tuples
(input, output, K.function(input, output)). There is a special
entry PRED that is for the prediction function. All other
functions are gradients w.r.t. input pixels.
The gradients are always supposed to be subtracted, so the sign
will be different depending on whether it is a targeted attack.
Arguments
models models used in the ensemble
pred_models models used for prediction
is_targeted
aug_or_rotate
grad_dict the main output of this function
make_model
load_weights
"""
grad_sign = 1 if is_targeted else -1
ph = K.learning_phase()
if grad_dict is not None:
grad_dict.clear()
input_img = Input(shape=(299, 299, 3), name='src_input_img')
noise_input = Input(batch_shape=(1, ), name='augmentation_scale')
if aug_or_rotate not in ['aug', 'rot', 'none']:
raise ValueError('invalid value for aug_or_rotate')
target_labels = Input(shape=(1000, ), name='target_labels')
pred_max = Lambda(lambda x: K.max(x, axis=1, keepdims=True),
name='target_labels_pred_max')(target_labels)
y = Equal(name='target_pred_equal')([target_labels, pred_max])
y = Lambda(lambda x: K.cast(x, K.floatx()), name='type_cast')(y)
y = Lambda(lambda x: x / K.sum(x, axis=1, keepdims=True),
name='pseudo_label')(y)
model_outputs = []
for model_name in models:
print('preparing', model_name)
model_pred = load_branch(model_name,
input_img,
aug_or_rotate,
noise_input,
False,
load_weights=load_weights)
if pred_models is not None and model_name in pred_models:
model_outputs.append(model_pred)
print('preparing gradients', model_name)
branch_loss = CategoricalCrossEntropy(from_logits=False,
name=model_name +
'_ce')([y, model_pred])
branch_grads = Gradients(grad_sign, name='grad_' +
model_name)([input_img, branch_loss])
if grad_dict is not None:
if make_model:
grad_dict[model_name] = branch_grads
else:
grad_dict[model_name] = (\
[input_img, noise_input, target_labels, ph], \
[branch_grads], \
K.function(\
[input_img, noise_input, target_labels, ph], \
[branch_grads]))
print('finished', model_name)
print('loading of source models finished')
if pred_models and grad_dict is not None:
if make_model:
if len(model_outputs) == 1:
grad_dict['PRED'] = Lambda(lambda x: x,
name='PRED')(model_outputs[0])
else:
grad_dict['PRED'] = Average(name='PRED')(model_outputs)
else:
pred_avg = sum(model_outputs) / len(model_outputs)
pred_func = K.function([input_img, noise_input, K.learning_phase()], \
[pred_avg])
grad_dict['PRED'] = ([input_img, noise_input, K.learning_phase()], \
[pred_avg], pred_func)
if make_model:
output_list = models
if pred_models:
output_list.append('PRED')
grad_inputs = [input_img, noise_input, target_labels]
model = Model(inputs=grad_inputs, \
outputs=[grad_dict[m] \
for m in output_list])
else:
model = None
print('gradient computation finished')
return model
def fit(self, x,
augment=False,
rounds=1,
seed=None):
"""Fits internal statistics to some sample data.
Required for featurewise_center, featurewise_std_normalization
and zca_whitening.
# Arguments
x: Numpy array, the data to fit on. Should have rank 4.
In case of grayscale data,
the channels axis should have value 1, and in case
of RGB data, it should have value 3.
augment: Whether to fit on randomly augmented samples
rounds: If `augment`,
how many augmentation passes to do over the data
seed: random seed.
# Raises
ValueError: in case of invalid input `x`.
"""
x = np.asarray(x, dtype=K.floatx())
if x.ndim != 4:
raise ValueError('Input to `.fit()` should have rank 4. '
'Got array with shape: ' + str(x.shape))
if x.shape[self.channel_axis] not in {1, 3, 4}:
warnings.warn(
'Expected input to be images (as Numpy array) '
'following the data format convention "' + self.data_format + '" '
'(channels on axis ' + str(self.channel_axis) + '), i.e. expected '
'either 1, 3 or 4 channels on axis ' + str(self.channel_axis) + '. '
'However, it was passed an array with shape ' + str(x.shape) +
' (' + str(x.shape[self.channel_axis]) + ' channels).')
if seed is not None:
np.random.seed(seed)
x = np.copy(x)
if augment:
ax = np.zeros(tuple([rounds * x.shape[0]] + list(x.shape)[1:]), dtype=K.floatx())
for r in range(rounds):
for i in range(x.shape[0]):
ax[i + r * x.shape[0]] = self.random_transform(x[i])
x = ax
if self.featurewise_center:
self.mean = np.mean(x, axis=(0, self.row_axis, self.col_axis))
broadcast_shape = [1, 1, 1]
broadcast_shape[self.channel_axis - 1] = x.shape[self.channel_axis]
self.mean = np.reshape(self.mean, broadcast_shape)
x -= self.mean
if self.featurewise_std_normalization:
self.std = np.std(x, axis=(0, self.row_axis, self.col_axis))
broadcast_shape = [1, 1, 1]
broadcast_shape[self.channel_axis - 1] = x.shape[self.channel_axis]
self.std = np.reshape(self.std, broadcast_shape)
x /= (self.std + K.epsilon())
if self.zca_whitening:
flat_x = np.reshape(x, (x.shape[0], x.shape[1] * x.shape[2] * x.shape[3]))
sigma = np.dot(flat_x.T, flat_x) / flat_x.shape[0]
u, s, _ = linalg.svd(sigma)
self.principal_components = np.dot(np.dot(u, np.diag(1. / np.sqrt(s + self.zca_epsilon))), u.T)
def class_percent_2buckRange(
y_true, y_pred
): # percent of times the prediction is within 2 buckets of true value
return K.cast(
K.equal(K.argmax(y_true, axis=-1), K.argmax(y_pred, axis=-1)),
K.floatx()) + K.cast(
K.equal(K.cast(K.argmax(y_true, axis=-1), K.floatx()),
K.cast(K.argmax(y_pred, axis=-1), K.floatx()) - 1.0),
K.floatx()) + K.cast(
K.equal(K.cast(K.argmax(y_true, axis=-1), K.floatx()),
K.cast(K.argmax(y_pred, axis=-1), K.floatx()) + 1.0),
K.floatx()) + K.cast(
K.equal(
K.cast(K.argmax(y_true, axis=-1), K.floatx()),
K.cast(K.argmax(y_pred, axis=-1), K.floatx()) - 2.0),
K.floatx()) + K.cast(
K.equal(
K.cast(K.argmax(y_true, axis=-1), K.floatx()),
K.cast(K.argmax(y_pred, axis=-1), K.floatx()) +
2.0), K.floatx())
