def test_items_values_keys():
def fun(input_dict):
A = 0.
B = 0.
for i, (k, v) in enumerate(input_dict.items()):
A = A + np.sum(np.sin(v)) * (i + 1.0)
B = B + np.sum(np.cos(v))
for v in input_dict.values():
A = A + np.sum(np.sin(v))
for k in input_dict.keys():
A = A + np.sum(np.cos(input_dict[k]))
return A + B
def d_fun(input_dict):
g = grad(fun)(input_dict)
A = np.sum(g['item_1'])
B = np.sum(np.sin(g['item_1']))
C = np.sum(np.sin(g['item_2']))
return A + B + C
input_dict = {
'item_1': npr.randn(5, 6),
'item_2': npr.randn(4, 3),
'item_X': npr.randn(2, 4)
}
check_grads(fun, input_dict)
check_grads(d_fun, input_dict)
def check_fun_and_grads(fun, args, kwargs, argnums):
wrt_args = [args[i] for i in argnums]
try:
if isinstance(fun, primitive):
check_equivalent(fun(*args, **kwargs),
fun.fun(*args, **kwargs))
except:
print("Value test failed! Args were", args, kwargs)
raise
with warnings.catch_warnings(record=True) as w:
try:
def scalar_fun(*new_args):
full_args = list(args)
for i, argnum in enumerate(argnums):
full_args[argnum] = new_args[i]
return to_scalar(fun(*full_args, **kwargs))
check_grads(scalar_fun, *wrt_args)
except:
print("First derivative test failed! Args were", args, kwargs)
raise
try:
for i in range(len(argnums)):
def d_scalar_fun(*args):
return to_scalar(grad(scalar_fun, argnum=i)(*args))
check_grads(d_scalar_fun, *wrt_args)
except:
print("Second derivative test failed! Args were", args, kwargs)
raise
def test_meta_gradient_with_langevin():
num_samples = 4
num_langevin_steps = 3
D = 2
init_mean = npr.randn(D) * 0.01
init_log_stddevs = np.log(1*np.ones(D)) + npr.randn(D) * 0.01
init_log_stepsizes = np.log(0.01*np.ones(num_langevin_steps)) + npr.randn(num_langevin_steps) * 0.01
init_log_noise_sizes = np.log(.001*np.ones(num_langevin_steps)) + npr.randn(num_langevin_steps) * 0.01
init_log_gradient_scales = np.log(1*np.ones(D))
init_gradient_power = 0.9
sample_and_run_langevin, parser = build_langevin_sampler(logprob_two_moons, D, num_langevin_steps, approx=False)
sampler_params = np.zeros(len(parser))
parser.put(sampler_params, 'mean', init_mean)
parser.put(sampler_params, 'log_stddev', init_log_stddevs)
parser.put(sampler_params, 'log_stepsizes', init_log_stepsizes)
parser.put(sampler_params, 'log_noise_sizes', init_log_noise_sizes)
parser.put(sampler_params, 'log_gradient_scales', init_log_gradient_scales)
parser.put(sampler_params, 'invsig_gradient_power', inv_sigmoid(init_gradient_power))
def get_batch_marginal_likelihood_estimate(sampler_params):
rs = np.random.npr.RandomState(0)
samples, loglik_estimates, entropy_estimates = sample_and_run_langevin(sampler_params, rs, num_samples)
marginal_likelihood_estimates = loglik_estimates + entropy_estimates
return np.mean(marginal_likelihood_estimates)
check_grads(get_batch_marginal_likelihood_estimate, sampler_params)
def test_nested_list():
A = [[1.0], 2.0, 1.5]
def fun(x):
return x[1:][0]
check_grads(fun, A)
def check_fun_and_grads(fun, args, kwargs, argnums):
wrt_args = [args[i] for i in argnums]
try:
if isinstance(fun, primitive):
check_equivalent(fun(*args, **kwargs),
fun.fun(*args, **kwargs))
except:
print "Value test failed! Args were", args, kwargs
raise
try:
def scalar_fun(*new_args):
full_args = list(args)
for i, argnum in enumerate(argnums):
full_args[argnum] = new_args[i]
return to_scalar(fun(*full_args, **kwargs))
check_grads(scalar_fun, *wrt_args)
except:
print "First derivative test failed! Args were", args, kwargs
raise
try:
for i in range(len(argnums)):
def d_scalar_fun(*args):
return to_scalar(grad(scalar_fun, argnum=i)(*args))
check_grads(d_scalar_fun, *wrt_args)
except:
print "Second derivative test failed! Args were", args, kwargs
raise
def check_fun_and_grads(fun, args, kwargs, argnums):
wrt_args = [args[i] for i in argnums]
try:
if isinstance(fun, primitive):
wrapped = fun(*args, **kwargs)
unwrapped = fun.fun(*args, **kwargs)
try:
assert wrapped == unwrapped
except:
check_equivalent(wrapped, unwrapped)
except:
print("Value test failed! Args were", args, kwargs)
raise
with warnings.catch_warnings(record=True) as w:
try:
def scalar_fun(*new_args):
full_args = list(args)
for i, argnum in enumerate(argnums):
full_args[argnum] = new_args[i]
return to_scalar(fun(*full_args, **kwargs))
check_grads(scalar_fun, *wrt_args)
except:
print("First derivative test failed! Args were", args, kwargs)
raise
try:
for i in range(len(argnums)):
def d_scalar_fun(*args):
return to_scalar(grad(scalar_fun, argnum=i)(*args))
check_grads(d_scalar_fun, *wrt_args)
except:
print("Second derivative test failed! Args were", args, kwargs)
raise
def opt_traj(func, fdict, T, opt_method = 'SGD', init = None, \
learning_rate = 0.1, seed = 100, momentum = False, noise_level = 0.0):
# do optimization and return the trajectory
params = {'x': 0.0, 'y': 0.0}
domain = fdict['domain']
optimum = fdict['optimum']
loss_and_grad = value_and_grad(func)
#quick_grad_check(func, params)
params = init_params(params, domain, init, seed)
check_grads(func, params)
opt_server = Parameter_Server(opt_method, momentum)
opt_server.init_gradient_storage(params)
x_traj = []
y_traj = []
f_traj = []
print 'optimising function using %s...' % opt_method
for t in xrange(T):
(func_value, func_grad) = loss_and_grad(params)
x_traj.append(params['x'])
y_traj.append(params['y'])
f_traj.append(func_value)
func_grad = inject_noise(func_grad, noise_level)
if opt_method == 'SGD':
norm = np.sqrt(func_grad['x'] ** 2 + func_grad['y'] ** 2)
if norm >= 2.0:
func_grad['x'] /= norm / 2; func_grad['y'] /= norm / 2
params = opt_server.update(params, func_grad, learning_rate)
return np.array(x_traj), np.array(y_traj), np.array(f_traj)
def test_jacobian_higher_order():
fun = lambda x: np.sin(np.outer(x,x)) + np.cos(np.dot(x,x))
assert jacobian(fun)(npr.randn(3)).shape == (3,3,3)
assert jacobian(jacobian(fun))(npr.randn(3)).shape == (3,3,3,3)
# assert jacobian(jacobian(jacobian(fun)))(npr.randn(3)).shape == (3,3,3,3,3)
check_grads(lambda x: np.sum(np.sin(jacobian(fun)(x))), npr.randn(3))
check_grads(lambda x: np.sum(np.sin(jacobian(jacobian(fun))(x))), npr.randn(3))
def test_jacobian_higher_order():
fun = lambda x: np.sin(np.outer(x, x)) + np.cos(np.dot(x, x))
jacobian(fun)(npr.randn(3)).shape == (3, 3, 3)
jacobian(jacobian(fun))(npr.randn(3)).shape == (3, 3, 3, 3)
jacobian(jacobian(jacobian(fun)))(npr.randn(3)).shape == (3, 3, 3, 3, 3)
check_grads(lambda x: np.sum(jacobian(fun)(x)), npr.randn(3))
check_grads(lambda x: np.sum(jacobian(jacobian(fun))(x)), npr.randn(3))
def test_gradient_csr_binary_dot_left():
"""
Checks that the gradient is computed correctly.
"""
def sum_csr_binary_dot_left(feats):
result = csr_binary_dot_left(feats, rows, cols)
return np.sum(result)
# gradfunc = grad(sum_csr_binary_dot_left)
check_grads(sum_csr_binary_dot_left, feats)
def test_gradient_csr_binary_dot_left():
"""
Checks that the gradient is computed correctly.
"""
def sum_csr_binary_dot_left(feats):
result = csr_binary_dot_left(feats, rows, cols)
return np.sum(result)
# gradfunc = grad(sum_csr_binary_dot_left)
check_grads(sum_csr_binary_dot_left, feats)
def test_grads():
def fun(input_list):
A = np.sum(np.sin(input_list[0]))
B = np.sum(np.cos(input_list[1]))
return A + B
def d_fun(input_list):
g = grad(fun)(input_list)
A = np.sum(g[0])
B = np.sum(np.sin(g[0]))
C = np.sum(np.sin(g[1]))
return A + B + C
input_list = [npr.randn(5, 6), npr.randn(4, 3), npr.randn(2, 4)]
check_grads(fun, input_list)
check_grads(d_fun, input_list)
def test_grads():
def fun(input_dict):
A = np.sum(np.sin(input_dict['item_1']))
B = np.sum(np.cos(input_dict['item_2']))
return A + B
def d_fun(input_dict):
g = grad(fun)(input_dict)
A = np.sum(g['item_1'])
B = np.sum(np.sin(g['item_1']))
C = np.sum(np.sin(g['item_2']))
return A + B + C
input_dict = {'item_1' : npr.randn(5, 6),
'item_2' : npr.randn(4, 3),
'item_X' : npr.randn(2, 4)}
check_grads(fun, input_dict)
check_grads(d_fun, input_dict)
def test_grads():
def fun(input_list):
A = np.sum(np.sin(input_list[0]))
B = np.sum(np.cos(input_list[1]))
return A + B
def d_fun(input_list):
g = grad(fun)(input_list)
A = np.sum(g[0])
B = np.sum(np.sin(g[0]))
C = np.sum(np.sin(g[1]))
return A + B + C
input_list = [npr.randn(5, 6),
npr.randn(4, 3),
npr.randn(2, 4)]
check_grads(fun, input_list)
check_grads(d_fun, input_list)
def test_iter():
def fun(input_dict):
for i, k in enumerate(sorted(input_dict)):
A = np.sum(np.sin(input_dict[k])) * (i + 1.0)
B = np.sum(np.cos(input_dict[k]))
return A + B
def d_fun(input_dict):
g = grad(fun)(input_dict)
A = np.sum(g['item_1'])
B = np.sum(np.sin(g['item_1']))
C = np.sum(np.sin(g['item_2']))
return A + B + C
input_dict = {'item_1' : npr.randn(5, 6),
'item_2' : npr.randn(4, 3),
'item_X' : npr.randn(2, 4)}
check_grads(fun, input_dict)
check_grads(d_fun, input_dict)
def test_gradient_csr_binary_dot_right():
def sum_csr_binary_dot_right(feats):
result = csr_binary_dot_right(feats, rows, cols)
return np.sum(result)
check_grads(sum_csr_binary_dot_right, feats)
def test_morg_net_gradient():
loss, parser = morg_fp_func()
weights = npr.randn(len(parser))
check_grads(loss, weights)
def check_symmetric_matrix_grads(fun, *args):
symmetrize = lambda A: symm(np.tril(A))
new_fun = lambda *args: fun(symmetrize(args[0]), *args[1:])
return check_grads(new_fun, *args)
def check_conv_grads(conv_params):
loss, parser = conv_fp_func(conv_params)
weights = npr.randn(len(parser)) * 0.1
check_grads(loss, weights)
def check_conv_grads(conv_params):
loss, parser = conv_fp_func(conv_params)
weights = npr.randn(len(parser)) * 0.1
check_grads(loss, weights)
def test_gradient_csr_binary_dot_right():
def sum_csr_binary_dot_right(feats):
result = csr_binary_dot_right(feats, rows, cols)
return np.sum(result)
check_grads(sum_csr_binary_dot_right, feats)
def test_morg_net_gradient():
loss, parser = morg_fp_func()
weights = npr.randn(len(parser))
check_grads(loss, weights)
def check_fun_and_grads(fun, args, kwargs, argnums, fwd=True):
cover.load()
cover.start()
wrt_args = [args[i] for i in argnums]
rand_vecs = [npr.randn(flatten(arg)[0].size) for arg in wrt_args]
try:
if isinstance(fun, primitive):
wrapped = fun(*args, **kwargs)
unwrapped = fun.fun(*args, **kwargs)
try:
assert wrapped == unwrapped
except:
check_equivalent(wrapped, unwrapped)
except:
print("Value test failed! Args were", args, kwargs)
raise
with warnings.catch_warnings(record=True) as w:
try:
def scalar_fun(*new_args):
full_args = list(args)
for i, argnum in enumerate(argnums):
full_args[argnum] = new_args[i]
return to_scalar(fun(*full_args, **kwargs))
check_grads(scalar_fun, *wrt_args)
except:
print("First derivative test failed! Args were", args, kwargs)
raise
try:
for i in range(len(argnums)):
def d_scalar_fun(*args):
return to_scalar(grad(scalar_fun, argnum=i)(*args))
check_grads(d_scalar_fun, *wrt_args)
except:
print("Second derivative test failed! Args were", args, kwargs)
raise
if fwd:
try:
def scalar_args_fun(*new_args):
full_args = list(args)
for i, argnum in enumerate(argnums):
wrt_flat, unflatten = flatten(wrt_args[i])
full_args[argnum] = unflatten(wrt_flat + new_args[i] * rand_vecs[i])
return to_scalar(fun(*full_args, **kwargs))
check_forward_grads(scalar_args_fun, *anp.zeros(len(wrt_args)))
except:
print("First forward derivative test failed! Args were", args, kwargs)
raise
try:
for i, _ in enumerate(argnums):
def d_scalar_args_fun(*args):
return forward_derivative(scalar_args_fun, argnum=i)(*args)
check_grads(d_scalar_args_fun, *anp.zeros(len(wrt_args)))
check_forward_grads(d_scalar_args_fun, *anp.zeros(len(wrt_args)))
check_forward_grads(grad(scalar_args_fun, argnum=i), *anp.zeros(len(wrt_args)))
except:
print("Second forward derivative test failed! Args were", args, kwargs)
raise
cover.stop()
cover.save()
