def cifar10_mean_subtract(x):
bgr_mean = ng.persistent_tensor(axes=[x.axes.channel_axis()],
initial_value=np.array(
[113.9, 123.0, 125.3]))
bgr_std = ng.persistent_tensor(axes=[x.axes.channel_axis()],
initial_value=np.array([66.7, 62.1, 63.0]))
return (x - bgr_mean) / bgr_std
def test_sequential_side(M):
x1_np = 2
x2_np = 3
b_np = 1
x_np = np.array([1, 2, 3], dtype=np.float32)
x = ng.variable([M], initial_value=x_np)
x1 = ng.persistent_tensor(axes=(), initial_value=x1_np)
x2 = ng.persistent_tensor(axes=(), initial_value=x2_np)
x1_vo = ng.value_of(x1)
x2_vo = ng.value_of(x2)
b = ng.persistent_tensor(axes=(), initial_value=b_np)
y = ng.sequential([
x1_vo, x2_vo,
ng.assign(x1,
ng.sum(x, out_axes=()) + x1 * b + (1 - b)),
ng.assign(x2,
ng.mean(x, out_axes=()) + x2 * b + (1 - b)), x * 2
])
with ExecutorFactory() as ex:
main_effect = ex.executor((y, x1_vo, x2_vo, x1, x2))
current_values = ex.executor((x1, x2))
# Run main path #1
y_val, x1_init_val, x2_init_val, x1_final_val, x2_final_val = main_effect(
)
y_np = x_np * 2
assert np.allclose(y_val, y_np)
assert np.allclose(x1_init_val, x1_np)
assert np.allclose(x2_init_val, x2_np)
x1_np = np.sum(x_np) + x1_np * b_np + (1 - b_np)
x2_np = np.mean(x_np) + x2_np * b_np + (1 - b_np)
assert np.allclose(x1_final_val, x1_np)
assert np.allclose(x2_final_val, x2_np)
x1_val, x2_val = current_values()
assert np.allclose(x1_val, x1_np)
assert np.allclose(x2_val, x2_np)
# Run main path #2 (Should be the same as before)
y_val, x1_init_val, x2_init_val, x1_final_val, x2_final_val = main_effect(
)
y_np = x_np * 2
assert np.allclose(y_val, y_np)
assert np.allclose(x1_init_val, x1_np)
assert np.allclose(x2_init_val, x2_np)
x1_np = np.sum(x_np) + x1_np * b_np + (1 - b_np)
x2_np = np.mean(x_np) + x2_np * b_np + (1 - b_np)
assert np.allclose(x1_final_val, x1_np)
assert np.allclose(x2_final_val, x2_np)
def variable_update(self, variable, grad, scale_factor, weight_clip_value):
m = ng.persistent_tensor(axes=grad.axes, initial_value=0.)
v = ng.persistent_tensor(axes=grad.axes, initial_value=0.)
updates = ng.sequential([
ng.assign(m, m * self.beta_1 + (1 - self.beta_1) * grad),
ng.assign(v, v * self.beta_2 + (1 - self.beta_2) * grad * grad),
ng.assign(variable,
clip_weight_value(variable - (scale_factor * self.ell * m) /
(ng.sqrt(v) + self.epsilon), weight_clip_value))
])
return updates
def variable_update(self, variable, grad, scale_factor, weight_clip_value):
epsilon, decay = (self.epsilon, self.decay_rate)
grad = clip_gradient_value(grad, self.gradient_clip_value)
state = ng.persistent_tensor(axes=variable.axes, initial_value=1.)
velocity = ng.persistent_tensor(axes=variable.axes,
initial_value=0.).named(variable.name + '_vel')
updates = ng.sequential([
ng.assign(state, decay * state + (1.0 - decay) * ng.square(grad)),
ng.assign(velocity, velocity * self.momentum +
(self.lrate * scale_factor * grad / ng.sqrt(state + epsilon)) +
self.lrate * self.wdecay * variable),
ng.assign(variable, clip_weight_value(variable - velocity, weight_clip_value))
])
return updates
def test_concatenate():
with ExecutorFactory() as ex:
A = ng.make_axis(name='A', length=3)
B = ng.make_axis(name='B', length=4)
np_shape = (A.length, B.length)
x0_np = -np.ones(np_shape)
x1_np = np.ones(np_shape)
x0_ng = ng.persistent_tensor([A, B], initial_value=x0_np).named('x0')
x1_ng = ng.persistent_tensor([A, B], initial_value=x1_np).named('x1')
j_np = np.concatenate([x0_np, x1_np], axis=0)
j_ng = ng.concat_along_axis([x0_ng, x1_ng], A)
f = ex.executor(j_ng)
j_val = f()
ng.testing.assert_allclose(j_val, j_np)
def variable_update(self, variable, grad, scale_factor, weight_clip_value):
grad = clip_gradient_value(grad, self.gradient_clip_value)
state = ng.persistent_tensor(axes=grad.axes, initial_value=0.)
updates = ng.sequential([
ng.assign(state, state + ng.square(grad)),
ng.assign(variable,
clip_weight_value(variable - (scale_factor * self.lrate * grad) /
(ng.sqrt(state + self.epsilon)), weight_clip_value))
])
return updates
def __call__(self, *args, **kwargs):
if len(self.ops) == 0:
self.beta_1 = ng.constant(self.beta_1, dtype=np.float32)
self.beta_2 = ng.constant(self.beta_2, dtype=np.float32)
self.t = ng.persistent_tensor(axes=(), initial_value=0)
self.t = ng.sequential([ng.assign(self.t, self.t + 1), self.t])
self.ell = self.lrate * ng.sqrt(1 - self.beta_2 ** self.t) / (1 - self.beta_1 ** self.t)
return super(Adam, self).__call__(*args, **kwargs)
def test_write_state():
"""
This reads back a tensor set from an argument. No code is generated.
"""
with ExecutorFactory() as ex:
N = ng.make_axis(3, name='N')
x_np = np.ones((N.length)) * 4
x = ng.persistent_tensor([N]).named('x')
f = ex.executor(x, x)
x_val = f(x_np)
assert np.allclose(x_np, x_val)
def __call__(self, in_obj):
if not self.initialized:
w_axis = ng.make_axis()
self.weight = ng.variable(axes=[w_axis],
initial_value=2,
metadata={"label": LABELS["weight"]},
name="W")
self.side_effect = ng.persistent_tensor(axes=[w_axis],
initial_value=0)
return ng.sequential([ng.assign(self.side_effect, self.weight),
self.weight * in_obj])
def test_scope_ops(input_placeholder):
"""
Test scope_ops creates a subgraph with correct attributes
"""
with scope_ops(name="foo") as subgraph:
w = ng.variable(ng.make_axis(), initial_value=1, name="W")
y = w * input_placeholder
z = y + 4
v1 = ng.persistent_tensor(w.axes, initial_value=0, name="effect1")
v2 = ng.persistent_tensor(w.axes, initial_value=0, name="effect2")
ng.sequential([ng.assign(v1, w), ng.assign(v2, w), z.named("output")])
assert len(subgraph.inputs) == 1
assert input_placeholder.unscoped_name in subgraph.inputs
assert len(subgraph.variables) == 1
assert "W" in subgraph.variables
assert len(subgraph.outputs) == 1
assert "output" in subgraph.outputs
assert len(subgraph.side_effects) == 2
def variable_update(self, variable, grad, scale_factor, weight_clip_value):
updates = []
"""
for op in ng.Op.ordered_ops([grad]):
op_var = ng.persistent_tensor(axes=op.tensor.axes,
initial_value=0.).named(variable.name + '_' + op.name)
updates.append(ng.assign(op_var, op))
"""
velocity = ng.persistent_tensor(axes=variable.axes,
initial_value=0.).named(variable.name + '_vel')
clip_grad = clip_gradient_value(grad, self.gradient_clip_value)
lr = - self.lrate * (scale_factor * clip_grad + self.wdecay * variable)
updates.append(ng.assign(velocity, velocity * self.momentum_coef + lr))
if self.nesterov:
delta = (self.momentum_coef * velocity + lr)
else:
delta = velocity
updates.append(ng.assign(variable, clip_weight_value(variable + delta, weight_clip_value)))
return ng.sequential(updates)
def test_specific_slice_deriv():
#
with ExecutorFactory() as ex:
A = ng.make_axis(name='A', length=3)
B = ng.make_axis(name='B', length=4)
np_shape = (A.length, B.length)
x_np = np.empty(np_shape, dtype=np.float32)
for i in range(A.length):
for j in range(B.length):
x_np[i, j] = 10 * i + j
x_ng = ng.persistent_tensor([A, B], initial_value=x_np)
for i in range(A.length):
for j in range(B.length):
slice = ng.tensor_slice(x_ng, (i, j))
dslice_dx = ng.deriv(slice, x_ng)
dslice_dx_fun = ex.executor(dslice_dx)
dslice_dx_val = dslice_dx_fun()
dslice_dx_np = np.zeros_like(x_np)
dslice_dx_np[i, j] = 1
ng.testing.assert_allclose(dslice_dx_val, dslice_dx_np)
def test_persistent_tensor():
input_axes = ng.make_axes([ng.make_axis(10), ng.make_axis(3)])
bgr = ng.persistent_tensor(axes=input_axes,
initial_value=np.array([113.9, 123.0, 125.3]))
bgr_comp = ng.computation(bgr, "all")
results = dict()
weight_saver = Saver()
with closing(ngt.make_transformer()) as transformer:
bgr_func = transformer.add_computation(bgr_comp)
weight_saver.setup_save(transformer=transformer, computation=bgr_comp)
results['saved'] = bgr_func().copy()
weight_saver.save(filename="test_persistent_tensor")
with closing(ngt.make_transformer()) as restore_transformer:
bgr_refunc = restore_transformer.add_computation(bgr_comp)
weight_saver.setup_restore(transformer=restore_transformer,
computation=bgr_comp,
filename="test_persistent_tensor")
weight_saver.restore()
results['restored'] = bgr_refunc().copy()
os.remove("test_persistent_tensor.npz")
assert np.allclose(results['saved'], results['restored'], atol=0)
def cifar_mean_subtract(x):
bgr_mean = ng.persistent_tensor(
axes=x.axes.find_by_name('C'),
initial_value=np.array([104., 119., 127.]))
return (x - bgr_mean) / 255.
def i1k_mean_subtract(x):
bgr_mean = ng.persistent_tensor(axes=[x.axes.channel_axis()],
initial_value=np.array(
[127.0, 119.0, 104.0]))
return (x - bgr_mean)