def testDense(self):
decay_steps = 10
sign_decay_fn = sign_decay.get_linear_decay_fn(decay_steps)
py_sign_decay_fn = py_linear_decay_fn(decay_steps)
self._testDense()
self._testDense(learning_rate=0.01, alpha=0.1, beta=0.8)
self._testDense(sign_decay_fn=sign_decay_fn,
py_sign_decay_fn=py_sign_decay_fn)
def testDense(self):
decay_steps = 10
sign_decay_fn = sign_decay.get_linear_decay_fn(decay_steps)
py_sign_decay_fn = py_linear_decay_fn(decay_steps)
self._testDense()
self._testDense(learning_rate=0.1, base=10.0, beta=0.8)
self._testDense(
sign_decay_fn=sign_decay_fn, py_sign_decay_fn=py_sign_decay_fn)
def testLinearDecay(self):
num_training_steps = 1000
linear_decay_fn = sign_decay.get_linear_decay_fn(num_training_steps)
for step in range(0, 1000, 100):
with self.test_session():
tf_decayed = linear_decay_fn(step).eval()
py_decayed = py_linear_decay_fn(num_training_steps)(step)
self.assertAlmostEqual(tf_decayed, py_decayed, places=4)
def testLinearDecay(self):
num_training_steps = 1000
linear_decay_fn = sign_decay.get_linear_decay_fn(num_training_steps)
for step in range(0, 1000, 100):
with self.test_session():
tf_decayed = linear_decay_fn(step).eval()
py_decayed = py_linear_decay_fn(num_training_steps)(step)
self.assertAlmostEqual(tf_decayed, py_decayed, places=4)
def get_opt(name, learning_rate, decay_steps=None):
if name == 'momentum':
optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9)
elif name == 'adam':
optimizer = tf.train.AdamOptimizer(learning_rate,
beta2=0.98,
epsilon=1e-9)
elif name == 'sgd':
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
elif name == 'rms':
optimizer = tf.train.RMSPropOptimizer(learning_rate)
elif name == 'adagrad':
optimizer = tf.train.AdagradOptimizer(learning_rate)
elif name == 'lazyadam':
optimizer = LazyAdamOptimizer(learning_rate)
elif name == 'powersign':
optimizer = PowerSignOptimizer(learning_rate)
elif name == 'powersign-ld':
optimizer = PowerSignOptimizer(
learning_rate, sign_decay_fn=get_linear_decay_fn(decay_steps))
elif name == 'powersign-cd':
optimizer = PowerSignOptimizer(
learning_rate, sign_decay_fn=get_cosine_decay_fn(decay_steps))
elif name == 'powersign-rd':
optimizer = PowerSignOptimizer(
learning_rate, sign_decay_fn=get_restart_decay_fn(decay_steps))
elif name == 'addsign':
optimizer = AddSignOptimizer(learning_rate)
elif name == 'addsign-ld':
optimizer = AddSignOptimizer(
learning_rate, sign_decay_fn=get_linear_decay_fn(decay_steps))
elif name == 'addsign-cd':
optimizer = AddSignOptimizer(
learning_rate, sign_decay_fn=get_cosine_decay_fn(decay_steps))
elif name == 'addsign-rd':
optimizer = AddSignOptimizer(
learning_rate, sign_decay_fn=get_restart_decay_fn(decay_steps))
else:
optimizer = None
return optimizer
def testSparse(self):
decay_steps = 10
sign_decay_fn = sign_decay.get_linear_decay_fn(decay_steps)
py_sign_decay_fn = py_linear_decay_fn(decay_steps)
self._testSparse(use_resource=False)
self._testSparse(use_resource=False,
learning_rate=0.01,
base=2.0,
beta=0.8)
self._testSparse(use_resource=False,
sign_decay_fn=sign_decay_fn,
py_sign_decay_fn=py_sign_decay_fn)
def testSparse(self):
decay_steps = 10
sign_decay_fn = sign_decay.get_linear_decay_fn(decay_steps)
py_sign_decay_fn = py_linear_decay_fn(decay_steps)
self._testSparse(use_resource=False)
self._testSparse(use_resource=False,
learning_rate=0.01,
base=2.0,
beta=0.8)
self._testSparse(use_resource=False,
sign_decay_fn=sign_decay_fn,
py_sign_decay_fn=py_sign_decay_fn)
def testDense(self):
decay_steps = 10
sign_decay_fn = sign_decay.get_linear_decay_fn(decay_steps)
py_sign_decay_fn = py_linear_decay_fn(decay_steps)
self._testDense(use_resource=False)
self._testDense(use_resource=False, learning_rate=0.01, alpha=0.1, beta=0.8)
self._testDense(use_resource=False,
sign_decay_fn=sign_decay_fn,
py_sign_decay_fn=py_sign_decay_fn)
self._testDense(use_resource=True)
self._testDense(use_resource=True, learning_rate=0.01, alpha=0.1, beta=0.8)
self._testDense(use_resource=True,
sign_decay_fn=sign_decay_fn,
py_sign_decay_fn=py_sign_decay_fn)
def testDense(self):
decay_steps = 10
sign_decay_fn = sign_decay.get_linear_decay_fn(decay_steps)
py_sign_decay_fn = py_linear_decay_fn(decay_steps)
self._testDense(use_resource=False)
self._testDense(use_resource=False, learning_rate=0.01, alpha=0.1, beta=0.8)
self._testDense(use_resource=False,
sign_decay_fn=sign_decay_fn,
py_sign_decay_fn=py_sign_decay_fn)
self._testDense(use_resource=True)
self._testDense(use_resource=True, learning_rate=0.01, alpha=0.1, beta=0.8)
self._testDense(use_resource=True,
sign_decay_fn=sign_decay_fn,
py_sign_decay_fn=py_sign_decay_fn)
def get_optimizer(opt_params, lr):
"""Helper to get optimizer from text params
Parameters
----------
opt_params: dict
Dictionary containing optimization function name and learning rate decay
lr: float
Initial learning rate
Return
------
opt_function: Keras optimizer
"""
if opt_params['opt_func'] == 'sgd':
return SGD(lr=lr, momentum=opt_params['momentum'])
elif opt_params['opt_func'] == 'adam':
return Adam(lr=lr)
elif opt_params['opt_func'] == 'rmsprop':
return rmsprop(lr=lr)
elif opt_params['opt_func'] == 'nadam':
return Nadam(lr=lr)
elif opt_params['opt_func'] == 'powersign':
from tensorflow.contrib.opt.python.training import sign_decay as sd
d_steps = opt_params['pwr_sign_decay_steps']
# Define the decay function (if specified)
if opt_params['pwr_sign_decay_func'] == 'lin':
decay_func = sd.get_linear_decay_fn(d_steps)
elif opt_params['pwr_sign_decay_func'] == 'cos':
decay_func = sd.get_consine_decay_fn(d_steps)
elif opt_params['pwr_sign_decay_func'] == 'res':
decay_func = sd.get_restart_decay_fn(d_steps,
num_periods=opt_params['pwr_sign_decay_periods'])
elif opt_params['decay_func'] is None:
decay_func = None
else:
raise ValueError('decay function not specified correctly')
# Use decay function in TF optimizer
return TFOptimizer(PowerSignOptimizer(learning_rate=lr,
sign_decay_fn=decay_func))
else:
raise ValueError
