def variable_update(self, variable, grad, scale_factor, weight_clip_value):
grad = clip_gradient_value(grad, self.gradient_clip_value)
state = ng.persistent_tensor(axes=grad.axes, initial_value=0.)
updates = ng.sequential([
ng.assign(state, state + ng.square(grad)),
ng.assign(variable,
clip_weight_value(variable - (scale_factor * self.lrate * grad) /
(ng.sqrt(state + self.epsilon)), weight_clip_value))
])
return updates
def test_sequential_side(M):
x1_np = 2
x2_np = 3
b_np = 1
x_np = np.array([1, 2, 3], dtype=np.float32)
x = ng.variable([M], initial_value=x_np)
x1 = ng.persistent_tensor(axes=(), initial_value=x1_np)
x2 = ng.persistent_tensor(axes=(), initial_value=x2_np)
x1_vo = ng.value_of(x1)
x2_vo = ng.value_of(x2)
b = ng.persistent_tensor(axes=(), initial_value=b_np)
y = ng.sequential([
x1_vo, x2_vo,
ng.assign(x1,
ng.sum(x, out_axes=()) + x1 * b + (1 - b)),
ng.assign(x2,
ng.mean(x, out_axes=()) + x2 * b + (1 - b)), x * 2
])
with ExecutorFactory() as ex:
main_effect = ex.executor((y, x1_vo, x2_vo, x1, x2))
current_values = ex.executor((x1, x2))
# Run main path #1
y_val, x1_init_val, x2_init_val, x1_final_val, x2_final_val = main_effect(
)
y_np = x_np * 2
assert np.allclose(y_val, y_np)
assert np.allclose(x1_init_val, x1_np)
assert np.allclose(x2_init_val, x2_np)
x1_np = np.sum(x_np) + x1_np * b_np + (1 - b_np)
x2_np = np.mean(x_np) + x2_np * b_np + (1 - b_np)
assert np.allclose(x1_final_val, x1_np)
assert np.allclose(x2_final_val, x2_np)
x1_val, x2_val = current_values()
assert np.allclose(x1_val, x1_np)
assert np.allclose(x2_val, x2_np)
# Run main path #2 (Should be the same as before)
y_val, x1_init_val, x2_init_val, x1_final_val, x2_final_val = main_effect(
)
y_np = x_np * 2
assert np.allclose(y_val, y_np)
assert np.allclose(x1_init_val, x1_np)
assert np.allclose(x2_init_val, x2_np)
x1_np = np.sum(x_np) + x1_np * b_np + (1 - b_np)
x2_np = np.mean(x_np) + x2_np * b_np + (1 - b_np)
assert np.allclose(x1_final_val, x1_np)
assert np.allclose(x2_final_val, x2_np)
def variable_update(self, variable, grad, scale_factor, weight_clip_value):
m = ng.persistent_tensor(axes=grad.axes, initial_value=0.)
v = ng.persistent_tensor(axes=grad.axes, initial_value=0.)
updates = ng.sequential([
ng.assign(m, m * self.beta_1 + (1 - self.beta_1) * grad),
ng.assign(v, v * self.beta_2 + (1 - self.beta_2) * grad * grad),
ng.assign(variable,
clip_weight_value(variable - (scale_factor * self.ell * m) /
(ng.sqrt(v) + self.epsilon), weight_clip_value))
])
return updates
def variable_update(self, variable, grad, scale_factor, weight_clip_value):
epsilon, decay = (self.epsilon, self.decay_rate)
grad = clip_gradient_value(grad, self.gradient_clip_value)
state = ng.persistent_tensor(axes=variable.axes, initial_value=1.)
velocity = ng.persistent_tensor(axes=variable.axes,
initial_value=0.).named(variable.name + '_vel')
updates = ng.sequential([
ng.assign(state, decay * state + (1.0 - decay) * ng.square(grad)),
ng.assign(velocity, velocity * self.momentum +
(self.lrate * scale_factor * grad / ng.sqrt(state + epsilon)) +
self.lrate * self.wdecay * variable),
ng.assign(variable, clip_weight_value(variable - velocity, weight_clip_value))
])
return updates
def test_modify_state():
with ExecutorFactory() as ex:
N = ng.make_axis(3, name='N')
x_np = np.ones((N.length)) * 4
x = ng.variable([N], initial_value=x_np).named('x')
val = ng.sequential([ng.assign(x, x + x), x])
f = ex.executor(val)
x_val = f()
assert np.allclose(x_np + x_np, x_val)
def test_sequential(N):
x = ng.variable([N], initial_value=0)
x0 = x + x
x1 = x + x
p = ng.sequential([x0, ng.assign(x, 2), x1, x0])
with ExecutorFactory() as ex:
x0_val, x1_val, p_val = ex.executor([x0, x1, p])()
assert x0_val == 0
assert x1_val == 4
assert p_val == 0
def __call__(self, *args, **kwargs):
if len(self.ops) == 0:
self.beta_1 = ng.constant(self.beta_1, dtype=np.float32)
self.beta_2 = ng.constant(self.beta_2, dtype=np.float32)
self.t = ng.persistent_tensor(axes=(), initial_value=0)
self.t = ng.sequential([ng.assign(self.t, self.t + 1), self.t])
self.ell = self.lrate * ng.sqrt(1 - self.beta_2 ** self.t) / (1 - self.beta_1 ** self.t)
return super(Adam, self).__call__(*args, **kwargs)
def variable_update(self, variable, grad, scale_factor, weight_clip_value):
updates = []
"""
for op in ng.Op.ordered_ops([grad]):
op_var = ng.persistent_tensor(axes=op.tensor.axes,
initial_value=0.).named(variable.name + '_' + op.name)
updates.append(ng.assign(op_var, op))
"""
velocity = ng.persistent_tensor(axes=variable.axes,
initial_value=0.).named(variable.name + '_vel')
clip_grad = clip_gradient_value(grad, self.gradient_clip_value)
lr = - self.lrate * (scale_factor * clip_grad + self.wdecay * variable)
updates.append(ng.assign(velocity, velocity * self.momentum_coef + lr))
if self.nesterov:
delta = (self.momentum_coef * velocity + lr)
else:
delta = velocity
updates.append(ng.assign(variable, clip_weight_value(variable + delta, weight_clip_value)))
return ng.sequential(updates)
def __call__(self, in_obj):
if not self.initialized:
w_axis = ng.make_axis()
self.weight = ng.variable(axes=[w_axis],
initial_value=2,
metadata={"label": LABELS["weight"]},
name="W")
self.side_effect = ng.persistent_tensor(axes=[w_axis],
initial_value=0)
return ng.sequential([ng.assign(self.side_effect, self.weight),
self.weight * in_obj])
def test_scope_ops(input_placeholder):
"""
Test scope_ops creates a subgraph with correct attributes
"""
with scope_ops(name="foo") as subgraph:
w = ng.variable(ng.make_axis(), initial_value=1, name="W")
y = w * input_placeholder
z = y + 4
v1 = ng.persistent_tensor(w.axes, initial_value=0, name="effect1")
v2 = ng.persistent_tensor(w.axes, initial_value=0, name="effect2")
ng.sequential([ng.assign(v1, w), ng.assign(v2, w), z.named("output")])
assert len(subgraph.inputs) == 1
assert input_placeholder.unscoped_name in subgraph.inputs
assert len(subgraph.variables) == 1
assert "W" in subgraph.variables
assert len(subgraph.outputs) == 1
assert "output" in subgraph.outputs
assert len(subgraph.side_effects) == 2
def test_logreg():
# xs: (C, N), y: (N,)
xs = np.array([[0.52, 0.88, 0.52, 0.74], [1.12, -1.08, 0.06, -2.49],
[0.77, 0.15, -1.3, 1.39]])
ys = np.array([1, 1, 0, 1])
max_iter = 10
alpha = 0.1
thetas = np.array([0., 0., 0.])
np_logreg = NumpyLogreg(xs, ys, thetas)
C, N = ng.make_axis(length=3), ng.make_axis(length=4)
# input tensors
xs_v = ng.placeholder((C, N))
ys_v = ng.placeholder([N])
alpha_v = ng.placeholder(())
thetas_var = ng.variable([C], initial_value=thetas)
# define ops
ys_pred = ng.sigmoid(ng.dot(thetas_var, xs_v))
log_likelihoods = ng.log(ys_pred) * ys_v + ng.log(1 - ys_pred) * (1 - ys_v)
loss = -ng.sum(log_likelihoods, reduction_axes=[N])
grad_comp = ng.deriv(loss, thetas_var)
weight_update = ng.sequential(
[ng.assign(thetas_var, thetas_var - alpha_v * grad_comp), thetas_var])
# transformer
with ExecutorFactory() as ex:
train_eval_func = ex.executor([grad_comp, loss, weight_update], xs_v,
ys_v, alpha_v)
# evaluate
for i in range(max_iter):
grad_np, loss_np, thetas_np = np_logreg.optimize(alpha)
grad_ng, loss_ng, thetas_ng = train_eval_func(xs, ys, alpha)
ng.testing.assert_allclose(loss_np, loss_ng, rtol=1e-05, atol=1e-05, \
transformer_overwrite=False)
ng.testing.assert_allclose(grad_np, grad_ng, rtol=1e-05, atol=1e-05, \
transformer_overwrite=False)
ng.testing.assert_allclose(thetas_np, thetas_ng, rtol=1e-05, atol=1e-05, \
transformer_overwrite=False)
def minimize(self, cost, variables):
"""
Minimize cost by returning update Ops.
Arguments:
cost: The cost Op to be minimized
variables: TODO
Returns:
A doall op containing setitems to variable ops.
"""
assert cost is not None
assert variables is not None
return ng.doall([
ng.assign(variable,
variable - self.compute_lr_op * ng.deriv(cost, variable))
for variable in variables
])