def train(model, optimizer, x_train, y_train, x_test, epoch, batch_size):
# placeholder
x_shape, y_shape = list(x_train.shape), list(y_train.shape)
x_shape[0], y_shape[0] = None, None
_x_batch = K.placeholder(name='x', shape=x_shape)
_y_batch = K.placeholder(name='y', shape=y_shape)
loss = model.custom_loss(_x_batch, _y_batch)
variables = model.variables
updates = optimizer.get_updates(params=variables, loss=loss)
_train = K.function([_x_batch, _y_batch], [loss], updates=updates)
for e in range(epoch):
n_train = x_train.shape[0]
iter_rate_1to0 = np.exp(-4 * ((e + 1.0) / epoch)**2)
iter_rate_0to1 = 1 - iter_rate_1to0
if model.SCHEDULE_SIG_MAX: # schedule sig_max
model.sig_rate = iter_rate_0to1
else:
model.sig_rate = iter_rate_0to1
r_idx = np.random.permutation(n_train)[:batch_size]
x_batch, y_batch = x_train[r_idx, :], y_train[
r_idx, :] # current batch
# Optimize the network
loss = _train([x_batch, y_batch])
print("= epoch : {} loss {} =".format(e, loss[0]))
# plot result
if e % 50 == 0:
model.plot_result(x_test, e, _x_train=x_train, _y_train=y_train)
model.plot_variances(x_test, e)
def get_symbolic_inputs(self, return_single_as_list=False):
"""Returns inputs to be set as self.inputs for a model."""
for i in range(len(self._flattened_inputs)):
k = self._input_names[i]
v = self._flattened_inputs[i]
if isinstance(v, (list, float, int)):
v = np.asarray(v)
if v.ndim == 1:
v = np.expand_dims(v, 1)
if isinstance(v, (np.ndarray, ops.EagerTensor)):
# We fix the placeholder shape except the batch size.
# This is suboptimal, but it is the best we can do with the info
# we have. The user should call `model._set_inputs(placeholders)`
# to specify custom placeholders if the need arises.
shape = (None,) + tuple(v.shape[1:])
v = K.placeholder(shape=shape, name=k)
elif isinstance(v, tensor_shape.TensorShape):
shape = (None,) + tuple(v.as_list()[1:])
v = K.placeholder(shape=shape, name=k)
self._flattened_inputs[i] = v
if self._is_dict:
return dict(zip(self._input_names, self._flattened_inputs))
if self._is_single_input and not return_single_as_list:
return self._flattened_inputs[0]
return self._flattened_inputs
def get_symbolic_inputs(self, return_single_as_list=False):
"""Returns inputs to be set as self.inputs for a model."""
for i in range(len(self._flattened_inputs)):
k = self._input_names[i]
v = self._flattened_inputs[i]
if isinstance(v, (list, float, int)):
v = np.asarray(v)
if v.ndim == 1:
v = np.expand_dims(v, 1)
if isinstance(v, (np.ndarray, ops.EagerTensor)):
# We fix the placeholder shape except the batch size.
# This is suboptimal, but it is the best we can do with the info
# we have. The user should call `model._set_inputs(placeholders)`
# to specify custom placeholders if the need arises.
shape = (None,) + tuple(v.shape[1:])
v = K.placeholder(shape=shape, name=k)
elif isinstance(v, tensor_shape.TensorShape):
shape = (None,) + tuple(v.as_list()[1:])
v = K.placeholder(shape=shape, name=k)
self._flattened_inputs[i] = v
if self._is_dict:
return dict(zip(self._input_names, self._flattened_inputs))
if self._is_single_input and not return_single_as_list:
return self._flattened_inputs[0]
return self._flattened_inputs
def compute_output_shape(self, input_shape):
if self._output_shape is None:
if context.executing_eagerly():
# Make use of existing autocomputation for Eager mode but provide
# Lambda-specific error message.
try:
return super(Lambda,
self).compute_output_shape(input_shape)
except NotImplementedError:
raise NotImplementedError(
'We could not automatically infer '
'the static shape of the Lambda\'s output.'
' Please specify the `output_shape` for'
' this Lambda.')
if isinstance(input_shape, list):
x = [K.placeholder(shape=shape) for shape in input_shape]
else:
x = K.placeholder(shape=input_shape)
x = self.call(x)
if isinstance(x, list):
return [
tensor_shape.TensorShape(K.int_shape(x_elem))
for x_elem in x
]
else:
return tensor_shape.TensorShape(K.int_shape(x))
elif isinstance(self._output_shape, (tuple, list)):
if isinstance(input_shape, list):
num_samples = input_shape[0][0]
else:
num_samples = input_shape[0] if input_shape else None
# List here represents multiple outputs.
if isinstance(self._output_shape, list):
return [
tensor_shape.TensorShape((num_samples, ) +
tuple(single_shape))
for single_shape in self._output_shape
]
return tensor_shape.TensorShape((num_samples, ) +
self._output_shape)
else:
shape = self._output_shape(input_shape)
if not isinstance(shape, (list, tuple)):
raise ValueError(
'`output_shape` function must return a tuple or a list of tuples.'
)
# List here can represent multiple outputs or single output.
if isinstance(shape, list):
# Convert list representing single output into a tuple.
if isinstance(shape[0], (int, type(None))):
shape = tuple(shape)
else:
return [
tensor_shape.TensorShape(single_shape)
for single_shape in shape
]
return tensor_shape.TensorShape(shape)
def test_softmax(self):
x = backend.placeholder(ndim=2)
f = backend.function([x], [activations.softmax(x)])
test_values = np.random.random((2, 5))
result = f([test_values])[0]
expected = _ref_softmax(test_values[0])
self.assertAllClose(result[0], expected, rtol=1e-05)
with self.assertRaises(ValueError):
x = backend.placeholder(ndim=1)
activations.softmax(x)
def compute_output_shape(self, input_shape):
if self._output_shape is None:
if context.executing_eagerly():
# Make use of existing autocomputation for Eager mode but provide
# Lambda-specific error message.
try:
return super(Lambda, self).compute_output_shape(input_shape)
except NotImplementedError:
raise NotImplementedError('We could not automatically infer '
'the static shape of the Lambda\'s output.'
' Please specify the `output_shape` for'
' this Lambda.')
if isinstance(input_shape, list):
x = [K.placeholder(shape=shape) for shape in input_shape]
else:
x = K.placeholder(shape=input_shape)
x = self.call(x)
if isinstance(x, list):
return [tensor_shape.TensorShape(K.int_shape(x_elem)) for x_elem in x]
else:
return tensor_shape.TensorShape(K.int_shape(x))
elif isinstance(self._output_shape, (tuple, list)):
if isinstance(input_shape, list):
num_samples = input_shape[0][0]
else:
num_samples = input_shape[0] if input_shape else None
# List here represents multiple outputs.
if isinstance(self._output_shape, list):
return [
tensor_shape.TensorShape((num_samples,) + tuple(single_shape))
for single_shape in self._output_shape
]
return tensor_shape.TensorShape((num_samples,) + self._output_shape)
else:
shape = self._output_shape(input_shape)
if not isinstance(shape, (list, tuple)):
raise ValueError(
'`output_shape` function must return a tuple or a list of tuples.')
# List here can represent multiple outputs or single output.
if isinstance(shape, list):
# Convert list representing single output into a tuple.
if isinstance(shape[0], (int, type(None))):
shape = tuple(shape)
else:
return [
tensor_shape.TensorShape(single_shape) for single_shape in shape
]
return tensor_shape.TensorShape(shape)
def test_softmax(self, shape):
x = backend.placeholder(ndim=len(shape))
f = backend.function([x], [activations.softmax(x, axis=-1)])
test_values = np.random.random(shape)
result = f([test_values])[0]
expected = _ref_softmax(test_values)
self.assertAllClose(result, expected, rtol=1e-05)
def test_temporal_softmax(self): x = backend.placeholder(shape=(2, 2, 3)) f = backend.function([x], [activations.softmax(x)]) test_values = np.random.random((2, 2, 3)) * 10 result = f([test_values])[0] expected = _ref_softmax(test_values[0, 0]) self.assertAllClose(result[0, 0], expected, rtol=1e-05)
def generate(self):
model_path = os.path.expanduser(self.model_path)
assert model_path.endswith(
'.h5'), 'Keras model or weights must be a .h5 file.'
self.yolo_model = load_model(model_path,
custom_objects={'Mish': Mish},
compile=False)
print('{} model, anchors, and classes loaded.'.format(model_path))
# Generate colors for drawing bounding boxes.
hsv_tuples = [(x / len(self.class_names), 1., 1.)
for x in range(len(self.class_names))]
self.colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
self.colors = list(
map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
self.colors))
np.random.seed(10101) # Fixed seed for consistent colors across runs.
np.random.shuffle(
self.colors) # Shuffle colors to decorrelate adjacent classes.
np.random.seed(None) # Reset seed to default.
# Generate output tensor targets for filtered bounding boxes.
self.input_image_shape = K.placeholder(shape=(2, ))
if self.gpu_num >= 2:
self.yolo_model = multi_gpu_model(self.yolo_model,
gpus=self.gpu_num)
boxes, scores, classes = yolo_eval(self.yolo_model.output,
self.anchors,
len(self.class_names),
self.input_image_shape,
score_threshold=self.score,
iou_threshold=self.iou)
return boxes, scores, classes
def do_activation(input_values, function_name, alpha=0.2):
"""Runs input array through activation function.
:param input_values: numpy array (any shape).
:param function_name: Name of activation function (must be accepted by ``).
:param alpha: Slope parameter (alpha) for activation function. This applies
only for eLU and ReLU.
:return: output_values: Same as `input_values` but post-activation.
"""
architecture_utils.check_activation_function(
activation_function_string=function_name,
alpha_for_elu=alpha,
alpha_for_relu=alpha)
input_object = K.placeholder()
if function_name == architecture_utils.ELU_FUNCTION_STRING:
function_object = K.function([input_object],
[layers.ELU(alpha=alpha)(input_object)])
elif function_name == architecture_utils.RELU_FUNCTION_STRING:
function_object = K.function(
[input_object], [layers.LeakyReLU(alpha=alpha)(input_object)])
else:
function_object = K.function(
[input_object], [layers.Activation(function_name)(input_object)])
return function_object([input_values])[0]
def compute_output_shape(self, input_shape):
if self._output_shape is None:
if context.executing_eagerly():
raise NotImplementedError
x = K.placeholder(shape=input_shape)
x = self.call(x)
if isinstance(x, list):
return [tensor_shape.TensorShape(K.int_shape(x_elem)) for x_elem in x]
else:
return tensor_shape.TensorShape(K.int_shape(x))
elif isinstance(self._output_shape, (tuple, list)):
if isinstance(input_shape, list):
num_samples = input_shape[0][0]
else:
num_samples = input_shape[0] if input_shape else None
return tensor_shape.TensorShape((num_samples,) +
tuple(self._output_shape))
else:
shape = self._output_shape(input_shape)
if not isinstance(shape, (list, tuple)):
raise ValueError(
'`output_shape` function must return a tuple or a list of tuples.')
if isinstance(shape, list):
if isinstance(shape[0], int) or shape[0] is None:
shape = tuple(shape)
return tensor_shape.TensorShape(shape)
def test_apply_complex_gradient(input_layer, batch_size):
with DEFAULT_TF_GRAPH.as_default():
machine = Linear(input_layer)
model = Model(inputs=[input_layer], outputs=machine.predictions)
if tensorflow.__version__ >= '1.14':
optimizer = ComplexValuesOptimizer(model,
machine.predictions_jacobian,
name='optimizer',
lr=1.0)
else:
optimizer = ComplexValuesOptimizer(model,
machine.predictions_jacobian,
lr=1.0)
complex_vector_t = K.placeholder(shape=(model.count_params() // 2, 1),
dtype=tensorflow.complex64)
predictions_function = K.function(inputs=[input_layer],
outputs=[machine.predictions])
sample = numpy.random.choice(
2, (batch_size, ) + K.int_shape(input_layer)[1:]) * 2 - 1
predictions_before = predictions_function([sample])[0]
updates = optimizer.apply_complex_gradient(complex_vector_t)
apply_gradients_function = K.function(
inputs=[input_layer, complex_vector_t],
outputs=[machine.predictions],
updates=[updates])
real_vector = numpy.random.normal(size=(model.count_params() // 2, 1,
2))
complex_vector = real_vector[..., 0] + 1j * real_vector[..., 1]
apply_gradients_function([sample, complex_vector])
predictions_after = predictions_function([sample])[0]
diff = predictions_after - predictions_before
manual_diff = sample.reshape((batch_size, -1)) @ complex_vector
diff_norm = numpy.linalg.norm(diff - manual_diff)
res_norm = numpy.linalg.norm(manual_diff)
assert (diff_norm / res_norm) < 1e-5
def generate(self, model_path):
num_anchors = len(self.anchors)
num_classes = len(self.class_names)
# self.yolo_model = load_model(model_path, compile=False)
self.yolo_model = yolo_body(
Input(shape=(None, None, self.num_channels)), num_anchors // 3,
num_classes)
self.yolo_model.load_weights(model_path)
hsv_tuples = [(x / len(self.class_names), 1., 1.)
for x in range(len(self.class_names))]
self.colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
self.colors = list(
map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
self.colors))
np.random.seed(10101) # Fixed seed for consistent colors across runs.
np.random.shuffle(
self.colors) # Shuffle colors to decorrelate adjacent classes.
np.random.seed(None) # Reset seed to default.
# Generate output tensor targets for filtered bounding boxes.
self.input_image_shape = K.placeholder(shape=(2, ))
if self.gpu_num >= 2:
self.yolo_model = multi_gpu_model(self.yolo_model,
gpus=self.gpu_num)
boxes, scores, classes = yolo_eval(self.yolo_model.output,
self.anchors,
len(self.class_names),
self.input_image_shape,
score_threshold=self.score,
iou_threshold=self.iou)
return boxes, scores, classes
def compute_output_shape(self, input_shape):
input_shape = tuple(tensor_shape.TensorShape(input_shape).as_list())
if self._output_shape is None:
if context.executing_eagerly():
raise NotImplementedError
x = K.placeholder(shape=input_shape)
x = self.call(x)
if isinstance(x, list):
return [tensor_shape.TensorShape(K.int_shape(x_elem)) for x_elem in x]
else:
return tensor_shape.TensorShape(K.int_shape(x))
elif isinstance(self._output_shape, (tuple, list)):
if isinstance(input_shape, list):
num_samples = input_shape[0][0]
else:
num_samples = input_shape[0] if input_shape else None
return tensor_shape.TensorShape((num_samples,) +
tuple(self._output_shape))
else:
shape = self._output_shape(input_shape)
if not isinstance(shape, (list, tuple)):
raise ValueError(
'`output_shape` function must return a tuple or a list of tuples.')
if isinstance(shape, list):
if isinstance(shape[0], int) or shape[0] is None:
shape = tuple(shape)
return tensor_shape.TensorShape(shape)
def __init__(self,
DIR,
num_classes,
record_every=None,
only_weights=False):
super(GradientActivationStore, self).__init__()
"""
Input:
- DIR: directory where activations and gradients are saved
- filename: name of file, preferably name of experiment for better recording
- num_classes: number of classes
- record_every: int, save parameters at what interval of epochs
- only_weights: boolean, True to save gradients for weights only
"""
# Creates directory with subfolders to save gradients and activations
if not os.path.exists(DIR):
os.makedirs(DIR)
self.DIR = DIR
# Placeholder is required to calculate gradients and activations
self.num_classes = num_classes
self.y_true = K.placeholder(shape=[None, num_classes])
self.record_every = record_every
self.only_weights = only_weights
# Initialize empty dictionaries
self.gradients = {}
self.activations = {}
# on_train_begin does not allow access to the validation set. Functions are called on first batch of first epoch
# Once complete, the flag is turned to False to prevent further use.
self.initial_flag = True
def test_exponential(self): test_values = np.random.random((2, 5)) x = backend.placeholder(ndim=2) exp = activations.exponential(x) f = backend.function([x], [exp]) result = f([test_values])[0] expected = np.exp(test_values) self.assertAllClose(result, expected, rtol=1e-05)
def __init__(self, discretization_range, training_data, test_data,
architecture, calculate_mi_for):
super().__init__(discretization_range, training_data, test_data,
architecture, calculate_mi_for)
Klayer_activity = K.placeholder(ndim=2) # Keras placeholder.
self._K_estimate_entropy = K.function(
[Klayer_activity],
[kde.entropy_estimator_bd(Klayer_activity, self.noise_variance)])
def test_softmax_3d_axis_tuple(self):
x = backend.placeholder(ndim=3)
f = backend.function([x], [activations.softmax(x, axis=(1, 2))])
test_values = np.random.random((2, 3, 5))
result = f([test_values])[0]
expected = np.zeros((2, 3, 5))
for i in range(2):
expected[i, :, :] = _ref_softmax(test_values[i, :, :])
self.assertAllClose(result, expected, rtol=1e-05)
def test_softmax_2d_axis_0(self):
x = backend.placeholder(ndim=2)
f = backend.function([x], [activations.softmax(x, axis=0)])
test_values = np.random.random((2, 5))
result = f([test_values])[0]
expected = np.zeros((2, 5))
for i in range(5):
expected[:, i] = _ref_softmax(test_values[:, i])
self.assertAllClose(result, expected, rtol=1e-05)
def generate(self):
model_path = os.path.expanduser(self.model_path)
assert model_path.endswith(
'.h5'), 'Keras model or weights must be a .h5 file.'
# Load model, or construct model and load weights.
start = timer()
num_anchors = len(self.anchors)
num_classes = len(self.class_names)
is_tiny_version = num_anchors == 6 # default setting
try:
self.yolo_model = load_model(model_path, compile=False)
except:
self.yolo_model = tiny_yolo_body(Input(shape=(None,None,3)), num_anchors//2, num_classes) \
if is_tiny_version else yolo_body(Input(shape=(None,None,3)), num_anchors//3, num_classes)
self.yolo_model.load_weights(
self.model_path) # make sure model, anchors and classes match
else:
assert self.yolo_model.layers[-1].output_shape[-1] == \
num_anchors/len(self.yolo_model.output) * (num_classes + 5), \
'Mismatch between model and given anchor and class sizes'
end = timer()
print('{} model, anchors, and classes loaded in {:.2f}sec.'.format(
model_path, end - start))
# Generate colors for drawing bounding boxes.
if len(self.class_names) == 1:
self.colors = ['GreenYellow']
else:
hsv_tuples = [(x / len(self.class_names), 1., 1.)
for x in range(len(self.class_names))]
self.colors = list(
map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
self.colors = list(
map(
lambda x:
(int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
self.colors))
np.random.seed(
10101) # Fixed seed for consistent colors across runs.
np.random.shuffle(
self.colors) # Shuffle colors to decorrelate adjacent classes.
np.random.seed(None) # Reset seed to default.
# Generate output tensor targets for filtered bounding boxes.
self.input_image_shape = K.placeholder(shape=(2, ))
if self.gpu_num >= 2:
self.yolo_model = multi_gpu_model(self.yolo_model,
gpus=self.gpu_num)
boxes, scores, classes = yolo_eval(self.yolo_model.output,
self.anchors,
len(self.class_names),
self.input_image_shape,
score_threshold=self.score,
iou_threshold=self.iou)
return boxes, scores, classes
def test_softsign(self):
def softsign(x):
return np.divide(x, np.ones_like(x) + np.absolute(x))
x = backend.placeholder(ndim=2)
f = backend.function([x], [activations.softsign(x)])
test_values = np.random.random((2, 5))
result = f([test_values])[0]
expected = softsign(test_values)
self.assertAllClose(result, expected, rtol=1e-05)
def test_elu(self): x = backend.placeholder(ndim=2) f = backend.function([x], [activations.elu(x, 0.5)]) test_values = np.random.random((2, 5)) result = f([test_values])[0] self.assertAllClose(result, test_values, rtol=1e-05) negative_values = np.array([[-1, -2]], dtype=backend.floatx()) result = f([negative_values])[0] true_result = (np.exp(negative_values) - 1) / 2 self.assertAllClose(result, true_result)
def test_relu(self):
x = backend.placeholder(ndim=2)
f = backend.function([x], [activations.relu(x)])
positive_values = np.random.random((2, 5))
result = f([positive_values])[0]
self.assertAllClose(result, positive_values, rtol=1e-05)
negative_values = np.random.uniform(-1, 0, (2, 5))
result = f([negative_values])[0]
expected = np.zeros((2, 5))
self.assertAllClose(result, expected, rtol=1e-05)
def test_hard_sigmoid(self):
def ref_hard_sigmoid(x):
x = (x * 0.2) + 0.5
z = 0.0 if x <= 0 else (1.0 if x >= 1 else x)
return z
hard_sigmoid = np.vectorize(ref_hard_sigmoid)
x = backend.placeholder(ndim=2)
f = backend.function([x], [activations.hard_sigmoid(x)])
test_values = np.random.random((2, 5))
result = f([test_values])[0]
expected = hard_sigmoid(test_values)
self.assertAllClose(result, expected, rtol=1e-05)
def get_symbolic_inputs(self, return_single_as_list=False):
"""Returns inputs to be set as self.inputs for a model."""
for i in range(len(self._flattened_inputs)):
k = self._input_names[i]
v = self._flattened_inputs[i]
if context.executing_eagerly():
v = K.placeholder((None, ) + tuple(v.shape[1:]), name=k)
else:
if isinstance(v, list):
v = np.asarray(v)
if v.ndim == 1:
v = np.expand_dims(v, 1)
if isinstance(v, (np.ndarray)):
# We fix the placeholder shape except the batch size.
# This is suboptimal, but it is the best we can do with the info
# we have. The user should call `model._set_inputs(placeholders)`
# to specify custom placeholders if the need arises.
shape = (None, ) + v.shape[1:]
v = K.placeholder(shape=shape, name=k)
self._flattened_inputs[i] = v
return self._get(return_single_as_list)
def __init__(self, args):
self.frames = []
self.X_train = []
self.X_val = []
self.X_test = []
self.encoder = []
self.decoder = []
self.network = []
self.dnb_net = []
self.encoder_widths = []
self.decoder_widths = []
self.z_mean = K.placeholder(shape=(8, ))
self.z_log_var = K.placeholder(shape=(8, ))
self.beta_changer = []
self.n_epochs = args.n_epochs
self.net_type = args.net_type
self.skip = args.skip
self.filename_in_1 = args.filename_in_1
self.filename_in_2 = args.filename_in_2
self.filename_out = args.filename_out
self.trained_model_name = args.trained_model_name
def test_selu(self):
x = backend.placeholder(ndim=2)
f = backend.function([x], [activations.selu(x)])
alpha = 1.6732632423543772848170429916717
scale = 1.0507009873554804934193349852946
positive_values = np.array([[1, 2]], dtype=backend.floatx())
result = f([positive_values])[0]
self.assertAllClose(result, positive_values * scale, rtol=1e-05)
negative_values = np.array([[-1, -2]], dtype=backend.floatx())
result = f([negative_values])[0]
true_result = (np.exp(negative_values) - 1) * scale * alpha
self.assertAllClose(result, true_result)
def test_sigmoid(self):
def ref_sigmoid(x):
if x >= 0:
return 1 / (1 + np.exp(-x))
else:
z = np.exp(x)
return z / (1 + z)
sigmoid = np.vectorize(ref_sigmoid)
x = backend.placeholder(ndim=2)
f = backend.function([x], [activations.sigmoid(x)])
test_values = np.random.random((2, 5))
result = f([test_values])[0]
expected = sigmoid(test_values)
self.assertAllClose(result, expected, rtol=1e-05)
def test_compute_wave_function_gradient_covariance_inverse_multiplication(
input_layer, batch_size, diag_shift, iterative):
with DEFAULT_TF_GRAPH.as_default():
machine = Linear(input_layer)
model = Model(inputs=[input_layer], outputs=machine.predictions)
if tensorflow.__version__ >= '1.14':
optimizer = ComplexValuesStochasticReconfiguration(
model,
machine.predictions_jacobian,
diag_shift=diag_shift,
conjugate_gradient_tol=1e-6,
iterative_solver=iterative,
iterative_solver_max_iterations=None,
name='optimizer')
else:
optimizer = ComplexValuesStochasticReconfiguration(
model,
machine.predictions_jacobian,
diag_shift=diag_shift,
conjugate_gradient_tol=1e-6,
iterative_solver=iterative,
iterative_solver_max_iterations=None)
complex_vector_t = K.placeholder(shape=(model.count_params() // 2, 1),
dtype=tensorflow.complex64)
jacobian_minus_mean = machine.manual_jacobian - tensorflow.reduce_mean(
machine.manual_jacobian, axis=0, keepdims=True)
manual_s = tensorflow.eye(model.count_params() // 2,
dtype=tensorflow.complex64) * diag_shift
manual_s += tensorflow.matmul(
jacobian_minus_mean, jacobian_minus_mean,
adjoint_a=True) / tensorflow.cast(batch_size, tensorflow.complex64)
manual_res_t = pinv(manual_s, complex_vector_t)
res_t = optimizer.compute_wave_function_gradient_covariance_inverse_multiplication(
complex_vector_t, jacobian_minus_mean)
res_function = K.function(inputs=[input_layer, complex_vector_t],
outputs=[res_t])
manual_res_function = K.function(
inputs=[input_layer, complex_vector_t], outputs=[manual_res_t])
sample = numpy.random.choice(
2, (batch_size, ) + K.int_shape(input_layer)[1:]) * 2 - 1
real_vector = numpy.random.normal(size=(model.count_params() // 2, 1,
2))
complex_vector = real_vector[..., 0] + 1j * real_vector[..., 1]
res = res_function([sample, complex_vector])[0]
manual_res = manual_res_function([sample, complex_vector])[0]
diff_norm = numpy.linalg.norm(res - manual_res)
res_norm = numpy.linalg.norm(manual_res)
assert (diff_norm / res_norm) < 1e-5
def generate(self):
model_path = os.path.expanduser(self.model_path)
assert model_path.endswith(
'.h5'), 'Keras model or weights must be a .h5 file.'
num_anchors = len(self.anchors)
num_classes = len(self.class_names)
if_tiny = num_anchors == 6
try:
self.yolo_model = load_model(model_path, compile=False)
except:
self.yolo_model = TinyYOLO(Input(shape=(None, None, 3)), num_anchors // 2, num_classes) if if_tiny else \
YOLOBody(Input(None, None, 3), num_anchors // 3, num_classes)
self.yolo_model.load_weights(self.model_path)
else:
assert self.yolo_model.layers[-1].output_shape[-1] == num_anchors / len(self.yolo_model.output) * (num_classes + 5), \
'MODEL AND GIVEN ANCHOR AND CLASS SIZE MISMATCH'
print('{} model, anchors, and classes loaded.'.format(model_path))
#Generate colors
hsv_tuple = [(x / len(self.class_names), 1., 1.)
for x in range(len(self.class_names))]
self.colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuple))
self.colors = list(
map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
self.colors))
np.random.shuffle(self.colors)
self.input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = YOLOEval(self.yolo_model.output,
self.anchors,
len(self.class_names),
self.input_image_shape,
score_threshold=self.score,
iou_threshold=self.iou)
return boxes, scores, classes
def test_gelu(self):
def gelu(x, approximate=False):
if approximate:
return 0.5 * x * (1.0 + np.tanh(
np.sqrt(2.0 / np.pi) * (x + 0.044715 * np.power(x, 3))))
else:
from scipy.stats import norm # pylint: disable=g-import-not-at-top
return x * norm.cdf(x)
x = backend.placeholder(ndim=2)
f = backend.function([x], [activations.gelu(x)])
test_values = np.random.random((2, 5))
result = f([test_values])[0]
expected = gelu(test_values)
self.assertAllClose(result, expected, rtol=1e-05)
f = backend.function([x], [activations.gelu(x, True)])
test_values = np.random.random((2, 5))
result = f([test_values])[0]
expected = gelu(test_values, True)
self.assertAllClose(result, expected, rtol=1e-05)
def get_symbolic_inputs(self, return_single_as_list=False):
"""Returns inputs to be set as self.inputs for a model."""
for i in range(len(self._flattened_inputs)):
k = self._input_names[i]
v = self._flattened_inputs[i]
if context.executing_eagerly():
v = base_layer.DeferredTensor(
shape=(None for _ in v.shape), dtype=v.dtype)
else:
if isinstance(v, list):
v = np.asarray(v)
if v.ndim == 1:
v = np.expand_dims(v, 1)
if isinstance(v, (np.ndarray)):
# We fix the placeholder shape except the batch size.
# This is suboptimal, but it is the best we can do with the info
# we have. The user should call `model._set_inputs(placeholders)`
# to specify custom placeholders if the need arises.
shape = (None,) + v.shape[1:]
v = K.placeholder(shape=shape, name=k)
self._flattened_inputs[i] = v
return self._get(return_single_as_list)
def clone_and_build_model(model,
input_tensors=None,
target_tensors=None,
custom_objects=None,
compile_clone=True,
in_place_reset=False,
optimizer_iterations=None):
"""Clone a `Model` and build/compile it with the same settings used before.
This function can be be run in the same graph or in a separate graph from the
model. When using a separate graph, `in_place_reset` must be `False`.
Note that, currently, the clone produced from this function may not work with
TPU DistributionStrategy. Try at your own risk.
Args:
model: `tf.keras.Model` object. Can be Functional, Sequential, or
sub-classed.
input_tensors: Optional list of input tensors to build the model upon. If
not provided, placeholders will be created.
target_tensors: Optional list of target tensors for compiling the model. If
not provided, placeholders will be created.
custom_objects: Optional dictionary mapping string names to custom classes
or functions.
compile_clone: Boolean, whether to compile model clone (default `True`).
in_place_reset: Boolean, whether to reset the model in place. Only used if
the model is not a graph network. If the model is a subclassed model, then
this argument must be set to `True` (default `False`). To restore the
original model, use the function
`in_place_subclassed_model_state_restoration(model)`.
optimizer_iterations: An iterations variable that will be incremented by the
optimizer if the clone is compiled. This argument is used when a Keras
model is cloned into an Estimator model function, because Estimators
create their own global step variable.
Returns:
Clone of the model.
Raises:
ValueError: Cloning fails in the following cases
- cloning a subclassed model with `in_place_reset` set to False.
- compiling the clone when the original model has not been compiled.
"""
# Grab optimizer now, as we reset-in-place for subclassed models, but
# want to maintain access to the original optimizer.
orig_optimizer = model.optimizer
if compile_clone and not orig_optimizer:
raise ValueError(
'Error when cloning model: compile_clone was set to True, but the '
'original model has not been compiled.')
if model._is_graph_network or isinstance(model, Sequential):
if custom_objects:
with CustomObjectScope(custom_objects):
clone = clone_model(model, input_tensors=input_tensors)
else:
clone = clone_model(model, input_tensors=input_tensors)
if all([
isinstance(clone, Sequential), not clone._is_graph_network,
getattr(model, '_build_input_shape', None) is not None
]):
# Set model inputs to build the model and add input/output properties.
# TODO(kathywu): Add multiple placeholders to handle edge case where
# sequential model has multiple inputs.
clone._set_inputs(
K.placeholder(model._build_input_shape,
dtype=model.inputs[0].dtype))
else:
if not in_place_reset:
raise ValueError(
'Model is not a graph network (usually means that it is a subclassed '
'model). The model cannot be cloned, but there is a workaround where '
'the model is reset in-place. To use this, please set the argument '
'`in_place_reset` to `True`. This will reset the attributes in the '
'original model. To restore the attributes, call '
'`in_place_subclassed_model_state_restoration(model)`.')
clone = model
_in_place_subclassed_model_reset(clone)
if input_tensors is not None:
if isinstance(input_tensors,
(list, tuple)) and len(input_tensors) == 1:
input_tensors = input_tensors[0]
clone._set_inputs(input_tensors)
if compile_clone:
if isinstance(orig_optimizer, optimizers.TFOptimizer):
optimizer = optimizers.TFOptimizer(orig_optimizer.optimizer,
optimizer_iterations)
K.track_tf_optimizer(optimizer)
else:
optimizer_config = orig_optimizer.get_config()
optimizer = orig_optimizer.__class__.from_config(optimizer_config)
if optimizer_iterations is not None:
optimizer.iterations = optimizer_iterations
clone.compile(optimizer,
model.loss,
metrics=metrics_module.clone_metrics(
model._compile_metrics),
loss_weights=model.loss_weights,
sample_weight_mode=model.sample_weight_mode,
weighted_metrics=metrics_module.clone_metrics(
model._compile_weighted_metrics),
target_tensors=target_tensors)
return clone
def create_placeholder(spec): return K.placeholder(shape=spec.shape, dtype=spec.dtype, name=spec.name)
def clone_and_build_model(
model, input_tensors=None, target_tensors=None, custom_objects=None,
compile_clone=True, in_place_reset=False, optimizer_iterations=None):
"""Clone a `Model` and build/compile it with the same settings used before.
This function can be be run in the same graph or in a separate graph from the
model. When using a separate graph, `in_place_reset` must be `False`.
Note that, currently, the clone produced from this function may not work with
TPU DistributionStrategy. Try at your own risk.
Args:
model: `tf.keras.Model` object. Can be Functional, Sequential, or
sub-classed.
input_tensors: Optional list of input tensors to build the model upon. If
not provided, placeholders will be created.
target_tensors: Optional list of target tensors for compiling the model. If
not provided, placeholders will be created.
custom_objects: Optional dictionary mapping string names to custom classes
or functions.
compile_clone: Boolean, whether to compile model clone (default `True`).
in_place_reset: Boolean, whether to reset the model in place. Only used if
the model is a subclassed model. In the case of a subclassed model,
this argument must be set to `True` (default `False`). To restore the
original model, use the function
`in_place_subclassed_model_state_restoration(model)`.
optimizer_iterations: An iterations variable that will be incremented by the
optimizer if the clone is compiled. This argument is used when a Keras
model is cloned into an Estimator model function, because Estimators
create their own global step variable.
Returns:
Clone of the model.
Raises:
ValueError: Cloning fails in the following cases
- cloning a subclassed model with `in_place_reset` set to False.
- compiling the clone when the original model has not been compiled.
"""
# Grab optimizer now, as we reset-in-place for subclassed models, but
# want to maintain access to the original optimizer.
orig_optimizer = model.optimizer
if compile_clone and not orig_optimizer:
raise ValueError(
'Error when cloning model: compile_clone was set to True, but the '
'original model has not been compiled.')
if model._is_graph_network or isinstance(model, Sequential):
if custom_objects:
with CustomObjectScope(custom_objects):
clone = clone_model(model, input_tensors=input_tensors)
else:
clone = clone_model(model, input_tensors=input_tensors)
if all([isinstance(clone, Sequential),
not clone._is_graph_network,
getattr(model, '_build_input_shape', None) is not None]):
# Set model inputs to build the model and add input/output properties.
# TODO(kathywu): Add multiple placeholders to handle edge case where
# sequential model has multiple inputs.
clone._set_inputs(
K.placeholder(model._build_input_shape, dtype=model.inputs[0].dtype))
else:
if not in_place_reset:
raise ValueError(
'This model is a subclassed model. '
'Such a model cannot be cloned, but there is a workaround where '
'the model is reset in-place. To use this, please set the argument '
'`in_place_reset` to `True`. This will reset the attributes in the '
'original model. To restore the attributes, call '
'`in_place_subclassed_model_state_restoration(model)`.')
clone = model
_in_place_subclassed_model_reset(clone)
if input_tensors is not None:
if isinstance(input_tensors, (list, tuple)) and len(input_tensors) == 1:
input_tensors = input_tensors[0]
clone._set_inputs(input_tensors)
if compile_clone:
if isinstance(orig_optimizer, optimizers.TFOptimizer):
optimizer = optimizers.TFOptimizer(
orig_optimizer.optimizer, optimizer_iterations)
K.track_tf_optimizer(optimizer)
else:
optimizer_config = orig_optimizer.get_config()
optimizer = orig_optimizer.__class__.from_config(optimizer_config)
if optimizer_iterations is not None:
optimizer.iterations = optimizer_iterations
clone.compile(
optimizer,
model.loss,
metrics=metrics_module.clone_metrics(model._compile_metrics),
loss_weights=model.loss_weights,
sample_weight_mode=model.sample_weight_mode,
weighted_metrics=metrics_module.clone_metrics(
model._compile_weighted_metrics),
target_tensors=target_tensors)
return clone
def __init__(self,
input_shape=None,
batch_size=None,
dtype=None,
input_tensor=None,
sparse=False,
name=None,
**kwargs):
strategy = distribution_strategy_context.get_strategy()
if strategy and batch_size is not None and \
distributed_training_utils.global_batch_size_supported(strategy):
if batch_size % strategy.num_replicas_in_sync != 0:
raise ValueError('The `batch_size` argument value {} cannot be '
'divisible by number of replicas {}'.format(
batch_size, strategy.num_replicas_in_sync))
batch_size = batch_size // strategy.num_replicas_in_sync
if 'batch_input_shape' in kwargs:
batch_input_shape = kwargs.pop('batch_input_shape')
if input_shape and batch_input_shape:
raise ValueError('Only provide the input_shape OR '
'batch_input_shape argument to '
'InputLayer, not both at the same time.')
batch_size = batch_input_shape[0]
input_shape = batch_input_shape[1:]
if kwargs:
raise ValueError('Unrecognized keyword arguments:', kwargs.keys())
if not name:
prefix = 'input'
name = prefix + '_' + str(backend.get_uid(prefix))
if not dtype:
if input_tensor is None:
dtype = backend.floatx()
else:
dtype = backend.dtype(input_tensor)
elif input_tensor is not None and input_tensor.dtype != dtype:
raise ValueError('`input_tensor.dtype` differs from `dtype`: %s vs. %s' %
(input_tensor.dtype, dtype))
super(InputLayer, self).__init__(dtype=dtype, name=name)
self.built = True
self.sparse = sparse
self.batch_size = batch_size
self.supports_masking = True
if isinstance(input_shape, tensor_shape.TensorShape):
input_shape = tuple(input_shape.as_list())
elif isinstance(input_shape, int):
input_shape = (input_shape,)
if input_tensor is None:
if input_shape is not None:
batch_input_shape = (batch_size,) + tuple(input_shape)
else:
batch_input_shape = None
graph = backend.get_graph()
with graph.as_default():
# In graph mode, create a graph placeholder to call the layer on.
if sparse:
input_tensor = backend.placeholder(
shape=batch_input_shape,
dtype=dtype,
name=self.name,
sparse=True)
else:
input_tensor = backend.placeholder(
shape=batch_input_shape,
dtype=dtype,
name=self.name)
self.is_placeholder = True
self._batch_input_shape = batch_input_shape
else:
if not tf_utils.is_symbolic_tensor(input_tensor):
raise ValueError('You should not pass an EagerTensor to `Input`. '
'For example, instead of creating an '
'InputLayer, you should instantiate your model and '
'directly call it on your input.')
self.is_placeholder = False
self._batch_input_shape = tuple(input_tensor.shape.as_list())
# Create an input node to add to self.outbound_node
# and set output_tensors' _keras_history.
input_tensor._keras_history = (self, 0, 0) # pylint: disable=protected-access
input_tensor._keras_mask = None
base_layer.Node(
self,
inbound_layers=[],
node_indices=[],
tensor_indices=[],
input_tensors=[input_tensor],
output_tensors=[input_tensor])
def __init__(self,
input_shape=None,
batch_size=None,
dtype=None,
input_tensor=None,
sparse=False,
name=None,
**kwargs):
if 'batch_input_shape' in kwargs:
batch_input_shape = kwargs.pop('batch_input_shape')
if input_shape and batch_input_shape:
raise ValueError('Only provide the input_shape OR '
'batch_input_shape argument to '
'InputLayer, not both at the same time.')
batch_size = batch_input_shape[0]
input_shape = batch_input_shape[1:]
if kwargs:
raise ValueError('Unrecognized keyword arguments:', kwargs.keys())
if not name:
prefix = 'input'
name = prefix + '_' + str(backend.get_uid(prefix))
if not dtype:
if input_tensor is None:
dtype = backend.floatx()
else:
dtype = backend.dtype(input_tensor)
super(InputLayer, self).__init__(dtype=dtype, name=name)
self.built = True
self.sparse = sparse
self.batch_size = batch_size
self.supports_masking = True
if isinstance(input_shape, tensor_shape.TensorShape):
input_shape = tuple(input_shape.as_list())
if input_tensor is None:
if input_shape is not None:
batch_input_shape = (batch_size,) + tuple(input_shape)
else:
batch_input_shape = None
graph = backend.get_graph()
with graph.as_default():
# In graph mode, create a graph placeholder to call the layer on.
if sparse:
input_tensor = backend.placeholder(
shape=batch_input_shape,
dtype=dtype,
name=self.name,
sparse=True)
else:
input_tensor = backend.placeholder(
shape=batch_input_shape,
dtype=dtype,
name=self.name)
self.is_placeholder = True
self._batch_input_shape = batch_input_shape
else:
if not tf_utils.is_symbolic_tensor(input_tensor):
raise ValueError('You should not pass an EagerTensor to `Input`. '
'For example, instead of creating an '
'InputLayer, you should instantiate your model and '
'directly call it on your input.')
self.is_placeholder = False
self._batch_input_shape = tuple(input_tensor.get_shape().as_list())
# Create an input node to add to self.outbound_node
# and set output_tensors' _keras_history.
input_tensor._keras_history = (self, 0, 0) # pylint: disable=protected-access
base_layer.Node(
self,
inbound_layers=[],
node_indices=[],
tensor_indices=[],
input_tensors=[input_tensor],
output_tensors=[input_tensor])
def __init__(self,
input_shape=None,
batch_size=None,
dtype=None,
input_tensor=None,
sparse=False,
name=None,
ragged=False,
**kwargs):
strategy = distribution_strategy_context.get_strategy()
if strategy and batch_size is not None and \
distributed_training_utils.global_batch_size_supported(strategy):
if batch_size % strategy.num_replicas_in_sync != 0:
raise ValueError(
'The `batch_size` argument value {} cannot be '
'divisible by number of replicas {}'.format(
batch_size, strategy.num_replicas_in_sync))
batch_size = batch_size // strategy.num_replicas_in_sync
if 'batch_input_shape' in kwargs:
batch_input_shape = kwargs.pop('batch_input_shape')
if input_shape and batch_input_shape:
raise ValueError('Only provide the input_shape OR '
'batch_input_shape argument to '
'InputLayer, not both at the same time.')
batch_size = batch_input_shape[0]
input_shape = batch_input_shape[1:]
if kwargs:
raise ValueError('Unrecognized keyword arguments:', kwargs.keys())
if not name:
prefix = 'input'
name = prefix + '_' + str(backend.get_uid(prefix))
if not dtype:
if input_tensor is None:
dtype = backend.floatx()
else:
dtype = backend.dtype(input_tensor)
elif input_tensor is not None and input_tensor.dtype != dtype:
raise ValueError(
'`input_tensor.dtype` differs from `dtype`: %s vs. %s' %
(input_tensor.dtype, dtype))
super(InputLayer, self).__init__(dtype=dtype, name=name)
self.built = True
self.sparse = sparse
self.batch_size = batch_size
self.supports_masking = True
if isinstance(input_shape, tensor_shape.TensorShape):
input_shape = tuple(input_shape.as_list())
elif isinstance(input_shape, int):
input_shape = (input_shape, )
if input_tensor is None:
if input_shape is not None:
batch_input_shape = (batch_size, ) + tuple(input_shape)
else:
batch_input_shape = None
graph = backend.get_graph()
with graph.as_default():
input_tensor = backend.placeholder(shape=batch_input_shape,
dtype=dtype,
name=self.name,
sparse=sparse,
ragged=ragged)
self.is_placeholder = True
self._batch_input_shape = batch_input_shape
else:
if not tf_utils.is_symbolic_tensor(input_tensor):
raise ValueError(
'You should not pass an EagerTensor to `Input`. '
'For example, instead of creating an '
'InputLayer, you should instantiate your model and '
'directly call it on your input.')
self.is_placeholder = False
self._batch_input_shape = tuple(input_tensor.shape.as_list())
# Create an input node to add to self.outbound_node
# and set output_tensors' _keras_history.
input_tensor._keras_history = base_layer.KerasHistory(self, 0, 0)
input_tensor._keras_mask = None
node_module.Node(self,
inbound_layers=[],
node_indices=[],
tensor_indices=[],
input_tensors=[input_tensor],
output_tensors=[input_tensor])
