def get(identifier):
"""Retrieves a Keras Optimizer instance.
# Arguments
identifier: Optimizer identifier, one of
- String: name of an optimizer
- Dictionary: configuration dictionary.
- Keras Optimizer instance (it will be returned unchanged).
- TensorFlow Optimizer instance
(it will be wrapped as a Keras Optimizer).
# Returns
A Keras Optimizer instance.
# Raises
ValueError: If `identifier` cannot be interpreted.
"""
if K.backend() == 'tensorflow':
# Wrap TF optimizer instances
if isinstance(identifier, tf.train.Optimizer):
return optimizers.TFOptimizer(identifier)
if isinstance(identifier, dict):
return deserialize(identifier)
elif isinstance(identifier, six.string_types):
config = {'class_name': str(identifier), 'config': {}}
return deserialize(config)
if isinstance(identifier, optimizers.Optimizer):
return identifier
else:
raise ValueError('Could not interpret optimizer identifier:',
identifier)
def test_tfoptimizer():
from keras import constraints
import tensorflow as tf
if tf.__version__.startswith('1.'):
optimizer = optimizers.TFOptimizer(tf.train.AdamOptimizer())
else:
optimizer = tf.keras.optimizers.Adam()
model = Sequential()
model.add(
Dense(num_classes,
input_shape=(3, ),
kernel_constraint=constraints.MaxNorm(1)))
model.compile(loss='mean_squared_error', optimizer=optimizer)
model.fit(np.random.random((5, 3)),
np.random.random((5, num_classes)),
epochs=1,
batch_size=5,
verbose=0)
if tf.__version__.startswith('1.'):
with pytest.raises(NotImplementedError):
optimizer.weights
with pytest.raises(NotImplementedError):
optimizer.get_config()
with pytest.raises(NotImplementedError):
optimizer.from_config(None)
def test_tfoptimizer_pass_correct_named_params_to_native_tensorflow_optimizer(
):
from keras import constraints
from tensorflow import train
class MyTfOptimizer(train.Optimizer):
wrapping_optimizer = train.AdamOptimizer()
def compute_gradients(self, loss, **kwargs):
return super(MyTfOptimizer, self).compute_gradients(loss, **kwargs)
def apply_gradients(self, grads_and_vars, **kwargs):
return self.wrapping_optimizer.apply_gradients(
grads_and_vars, **kwargs)
my_tf_optimizer = MyTfOptimizer(use_locking=False, name='MyTfOptimizer')
optimizer = optimizers.TFOptimizer(my_tf_optimizer)
model = Sequential()
model.add(
Dense(num_classes,
input_shape=(3, ),
kernel_constraint=constraints.MaxNorm(1)))
model.compile(loss='mean_squared_error', optimizer=optimizer)
model.fit(np.random.random((5, 3)),
np.random.random((5, num_classes)),
epochs=1,
batch_size=5,
verbose=0)
def __init__(self, feature_size, hidden_size, weight_decay, learning_rate):
self.feature_size = feature_size
self.hidden_size = hidden_size
self.weight_decay = weight_decay
self.learning_rate = learning_rate
self.model = Sequential()
self.model.add(
Dense(self.hidden_size,
input_dim=self.feature_size,
activation='tanh',
weights=layer_0_param,
kernel_regularizer=regularizers.l2(self.weight_decay),
kernel_initializer=initializers.random_normal(stddev=0.01)))
self.model.add(
Dense(1,
input_dim=self.hidden_size,
weights=layer_1_param,
kernel_regularizer=regularizers.l2(self.weight_decay)))
self.model.add(Reshape([-1]))
self.model.add(Activation('sigmoid'))
self.model.summary()
self.model.compile(loss='binary_crossentropy',
optimizer=optimizers.TFOptimizer(
tf.train.GradientDescentOptimizer(
self.learning_rate)),
metrics=['accuracy'])
def selu_network(input_dim, output_dim, retmodel):
"""
# Returns
A Keras model instance (compiled).
"""
#hyper params to be otpimized at some point
nnProps = {
'nlayers': 2,
'mlayers': 2,
'choke': 40,
'start': 100,
'end': 50,
'dropout_rate': .1,
'activation': 'selu',
"kernel_initializer": 'lecun_normal',
'optimizer': 'adam'
}
layers = hourglass(nnProps['nlayers'], nnProps['mlayers'],
nnProps['choke'], nnProps['start'], nnProps['end'])
model = Sequential()
for i, size in enumerate(layers):
if i == 0:
#add selu layer 1st
model.add(
Dense(size,
kernel_initializer=nnProps['kernel_initializer'],
input_shape=(input_dim, )))
model.add(Activation(nnProps['activation']))
model.add(AlphaDropout(nnProps['dropout_rate']))
else:
#add rectified linear units
model.add(
PReLU(alpha_initializer='zeros',
alpha_regularizer=None,
alpha_constraint=None,
shared_axes=None))
model.add(Dropout(nnProps['dropout_rate']))
#output layer is categorical
model.add(Dense(output_dim, activation='softmax'))
#compile
x = tf.placeholder(tf.float32, shape=(None, input_dim))
#placeholder y
y = model(x)
#use tensorflow optimizer
tfOptimizer = tf.train.AdamOptimizer
optimizer = optimizers.TFOptimizer(tfOptimizer)
lossfun = losses.categorical_crossentropy
model.compile(loss=lossfun, optimizer=optimizer, metrics=['accuracy'])
if retmodel == True:
return model
else:
return model, x, y, lossfun, tfOptimizer
def __init__(self,
feature_size,
hidden_size,
weight_decay,
learning_rate,
temperature=1.0,
layer_0_param=None,
layer_1_param=None):
self.feature_size = feature_size
self.hidden_size = hidden_size
self.weight_decay = weight_decay
self.learning_rate = learning_rate
self.temperature = temperature
self.pred_data = Input(shape=(None, self.feature_size))
self.reward = Input(shape=(None, ))
self.important_sampling = Input(shape=(None, ))
self.Dense_1_result = Dense(self.hidden_size,
input_dim=self.feature_size,
activation='tanh',
weights=layer_0_param,
kernel_regularizer=regularizers.l2(
self.weight_decay))(self.pred_data)
self.Dense_2_result = Dense(1,
input_dim=self.hidden_size,
weights=layer_1_param,
kernel_regularizer=regularizers.l2(
self.weight_decay))(
self.Dense_1_result)
# Given batch query-url pairs, calculate the matching score
# For all urls of one query
self.score = Lambda(lambda x: x / self.temperature)(
self.Dense_2_result)
self.score = Reshape([-1])(self.score)
self.prob = Activation('softmax')(self.score)
self.model = Model(
inputs=[self.pred_data, self.reward, self.important_sampling],
outputs=[self.prob])
self.model.summary()
self.model.compile(
loss=self.__loss(self.reward, self.important_sampling),
optimizer=optimizers.TFOptimizer(
tf.train.GradientDescentOptimizer(self.learning_rate)),
metrics=['accuracy'])
def test_tfoptimizer():
from keras import constraints
from tensorflow import train
optimizer = optimizers.TFOptimizer(train.AdamOptimizer())
model = Sequential()
model.add(Dense(num_classes, input_shape=(3,), kernel_constraint=constraints.MaxNorm(1)))
model.compile(loss='mean_squared_error', optimizer=optimizer)
model.fit(np.random.random((5, 3)), np.random.random((5, num_classes)),
epochs=1, batch_size=5, verbose=0)
# not supported
with pytest.raises(NotImplementedError):
optimizer.weights
with pytest.raises(NotImplementedError):
optimizer.get_config()
with pytest.raises(NotImplementedError):
optimizer.from_config(None)
def test_tf_optimizer():
with pytest.raises(NotImplementedError):
import tensorflow as tf
tf_opt = optimizers.TFOptimizer(tf.train.GradientDescentOptimizer(0.1))
NormalizedOptimizer(tf_opt, normalization='l2')
def __init__(self, vocab, config, hps):
super(SummarizationModel, self).__init__(config)
self.__name = 'pointer_generator_summarizer'
self.config = config
self.hps = hps
self.mode = config['mode']
self.use_coverage = config['use_coverage']
self.pointer_gen = config['pointer_gen']
self.embed_trainable = config['train_embed']
self.embedding_size = config['embed_size']
self.vsize = config['vocab_size']
self.rand_unif_init_mag = config['rand_unif_init_mag']
self.trunc_norm_init_std = config['trunc_norm_init_std']
self.hidden_units = self.config['hidden_units']
self.cov_loss_wt = self.config['cov_loss_wt']
# Initializers:
self.rand_unif_init = RandomUniform(minval=-self.rand_unif_init_mag,
maxval=self.rand_unif_init_mag,
seed=123)
self.trunc_norm_init = TruncatedNormal(stddev=self.trunc_norm_init_std)
# Optimizers:
self.adg = optimizers.TFOptimizer(
K.tf.train.AdagradOptimizer(
self.hps.lr,
initial_accumulator_value=self.hps.adagrad_init_acc))
# Layers
self.Emb = Embedding(self.vsize,
self.embedding_size,
weights=config['embed'],
trainable=self.embed_trainable)
# different dictionary for source and target
# Bi-directional lstm encoder, return (output, states)
# Dimension: 2*hidden_units
# concatenated forward and backward vectors
self.Encoder = Bidirectional(
CuDNNLSTM(self.hidden_units,
return_state=True,
return_sequences=True,
kernel_initializer=self.rand_unif_init))
# Decoder is not bi-directional, perform linear reduction...
# Dense_layer_dimension=encoder_hidden_units
# Encoder states and output tensors are separated...
# to initialize decoder
# Decoder cell input: [input, state_h, state_c]
self.DecoderCell = LSTMCell(self.hidden_units,
kernel_initializer=self.rand_unif_init,
bias_initializer="zeros",
recurrent_initializer=self.rand_unif_init)
# Decoder output projector
# to probabilities[word_index]
self.DecoderOutputProjector = Dense(
self.vsize,
kernel_initializer=self.trunc_norm_init,
bias_initializer=self.trunc_norm_init,
activation=None)
self.ConcatenateAxis1 = Concatenate(axis=1)
self.ConcatenateLastDim = Concatenate(axis=-1)
self.StackSecondDim = Lambda(lambda x: K.tf.stack(x, axis=1))
self.SoftmaxforScore = Softmax(axis=-1)
self._batch_size = None
self._enc_batch = None
self._enc_lens = None
self._enc_padding_mask = None
self._enc_batch_extend_vocab = None
self._max_art_oovs = None
self._max_art_oovs_inp = None
self._dec_batch = None
self._target_batch = None
self._dec_padding_mask = None
self._dec_in_state = None
self._enc_states = None
self._dec_out_state = None
self.p_gens = None
self.prev_coverage = None
self.coverage = None
self._coverage_loss = None
self.check_list = []
if not self.check():
pass
pass
