def check_sgd_optimizer(optimizer_attr):
init_program = framework.Program()
program = framework.Program()
block = program.global_block()
mul_x = block.create_parameter(dtype="float32",
shape=[5, 10],
lod_level=0,
name="mul.x",
optimize_attr=optimizer_attr)
mul_y = block.create_var(dtype="float32",
shape=[10, 8],
lod_level=0,
name="mul.y")
mul_out = block.create_var(dtype="float32",
shape=[5, 8],
lod_level=0,
name="mul.out")
mean_out = block.create_var(dtype="float32",
shape=[1],
lod_level=0,
name="mean.out")
block.append_op(type="mul",
inputs={
"X": mul_x,
"Y": mul_y
},
outputs={"Out": mul_out},
attrs={"x_num_col_dims": 1})
block.append_op(type="mean",
inputs={"X": mul_out},
outputs={"Out": mean_out})
sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.01)
opts, _ = sgd_optimizer.minimize(mean_out, init_program)
return opts
def test_infer_no_need_buffer_slots(self):
program = framework.Program()
startup_program = framework.Program()
with fluid.program_guard(program, startup_program):
loss = self.net()
sgd = fluid.optimizer.SGD(learning_rate=0.01)
sgd.minimize(loss)
block = program.global_block()
for idx, op in enumerate(block.ops):
op_desc = op.desc
inputs = {}
for input_name in op_desc.input_names():
inputs[input_name] = op_desc.input(input_name)
outputs = {}
for output_name in op_desc.output_names():
outputs[output_name] = op_desc.output(output_name)
attrs = {}
for attr_name in op_desc.attr_names():
attrs[attr_name] = op_desc.attr(attr_name)
if idx == 0:
# elementwise_add op
self.assertEqual(
core.infer_no_need_buffer_slots(op.type, inputs, outputs,
attrs), set([]))
elif idx == 1:
# fill constant op
self.assertEqual(
core.infer_no_need_buffer_slots(op.type, inputs, outputs,
attrs), set([]))
else:
# elementwise_add_grad op
self.assertEqual(
core.infer_no_need_buffer_slots(op.type, inputs, outputs,
attrs), set(['Y', 'X']))
def test_vanilla_momentum_optimizer(self):
init_program = framework.Program()
program = framework.Program()
block = program.global_block()
mul_x = block.create_parameter(dtype="float32",
shape=[5, 10],
lod_level=0,
name="mul.x",
optimize_attr={'learning_rate': 1.1})
mul_y = block.create_var(dtype="float32",
shape=[10, 8],
lod_level=0,
name="mul.y")
mul_out = block.create_var(dtype="float32",
shape=[5, 8],
lod_level=0,
name="mul.out")
block.append_op(type="mul",
inputs={
"X": mul_x,
"Y": mul_y
},
outputs={"Out": mul_out},
attrs={"x_num_col_dims": 1})
learning_rate = 0.01
momentum_optimizer = self.MockMomentum(learning_rate=learning_rate,
momentum=0.2)
mean_out = block.create_var(dtype="float32",
shape=[1],
lod_level=0,
name="mean.out")
block.append_op(type="mean",
inputs={"X": mul_out},
outputs={"Out": mean_out})
params_grads = append_backward(mean_out)
self.assertEqual(len(params_grads), 1)
self.assertEqual(len(momentum_optimizer.get_accumulators()), 0)
opts = momentum_optimizer.create_optimization_pass(
params_grads, mul_out, init_program)
self.assertEqual(len(opts), 3)
sgd_op = opts[-1]
self.assertEqual([op.type for op in opts],
["fill_constant", "elementwise_mul", "momentum"])
self.assertFalse(sgd_op.attr('use_nesterov'))
# Check accumulators
accumulators = momentum_optimizer.get_accumulators()
self.assertEqual(len(accumulators), 1)
self.assertTrue(momentum_optimizer.get_velocity_str() in accumulators)
velocity_acc = accumulators[momentum_optimizer.get_velocity_str()]
self.assertEqual(len(velocity_acc), 1)
self.assertTrue(mul_x.name in velocity_acc)
# Check init_program
init_ops = init_program.global_block().ops
self.assertEqual(len(init_ops), 2)
self.assertEqual(init_ops[0].type, "fill_constant")
self.assertAlmostEqual(init_ops[0].attr('value'), learning_rate)
self.assertEqual(init_ops[1].type, "fill_constant")
self.assertAlmostEqual(init_ops[1].attr('value'), 0.0)
def test_adam_optimizer(self):
init_program = framework.Program()
program = framework.Program()
block = program.global_block()
mul_x = block.create_parameter(dtype="float32",
shape=[5, 10],
lod_level=0,
name="mul.x",
optimize_attr={'learning_rate': 1.1})
mul_y = block.create_var(dtype="float32",
shape=[10, 8],
lod_level=0,
name="mul.y")
mul_out = block.create_var(dtype="float32",
shape=[5, 8],
lod_level=0,
name="mul.out")
block.append_op(type="mul",
inputs={
"X": mul_x,
"Y": mul_y
},
outputs={"Out": mul_out},
attrs={"x_num_col_dims": 1})
mean_out = block.create_var(dtype="float32",
shape=[1],
lod_level=0,
name="mean.out")
block.append_op(type="mean",
inputs={"X": mul_out},
outputs={"Out": mean_out})
learning_rate = 0.01
adam_optimizer = self.MockAdam(learning_rate=learning_rate,
beta1=0.9,
beta2=0.999)
params_grads = append_backward(mean_out)
self.assertEqual(len(params_grads), 1)
self.assertEqual(len(adam_optimizer.get_accumulators()), 0)
with framework.program_guard(program, init_program):
opts = adam_optimizer.apply_gradients(params_grads)
self.assertEqual(len(opts), 2)
self.assertEqual([op.type for op in opts], ["scale", "adam"])
# Check accumulators
accumulators = adam_optimizer.get_accumulators()
self.assertEqual(len(accumulators), 4)
self.assertTrue(adam_optimizer.get_moment1_str() in accumulators)
self.assertTrue(adam_optimizer.get_moment2_str() in accumulators)
moment1_acc = accumulators[adam_optimizer.get_moment1_str()]
moment2_acc = accumulators[adam_optimizer.get_moment2_str()]
self.assertEqual(len(moment1_acc), 1)
self.assertEqual(len(moment2_acc), 1)
self.assertTrue(mul_x.name in moment1_acc)
self.assertTrue(mul_x.name in moment2_acc)
# Check init_program
init_ops = init_program.global_block().ops
self.assertEqual(len(init_ops), 5)
self.assertEqual(init_ops[0].type, "fill_constant")
self.assertAlmostEqual(init_ops[0].attr('value'), learning_rate)
def test_error(self):
main_program = framework.Program()
startup_program = framework.Program()
with framework.program_guard(main_program, startup_program):
cond = layers.fill_constant(shape=[1], dtype='float32', value=0.0)
zero_var = layers.fill_constant(shape=[1],
dtype='float32',
value=0.0)
result = layers.create_global_var(shape=[1],
value=-1.0,
dtype='float32',
persistable=True)
# 1. The type of 'condition' in case must be Variable.
def test_condition_type():
with layers.Switch() as switch:
with switch.case(1):
layers.assign(zero_var, result)
self.assertRaises(TypeError, test_condition_type)
# 2. The dtype of 'condition' in case must be 'bool'.
def test_condition_dtype():
with layers.Switch() as switch:
with switch.case(cond):
layers.assign(zero_var, result)
self.assertRaises(TypeError, test_condition_dtype)
def test_prune_with_multi_optimizers(self):
'''
If there are multiple optimizers in the program, we can run specific one by
pass the return of optimize.minimize() to fetch_list.
'''
exe = fluid.Executor(fluid.CPUPlace())
program = framework.Program()
startup_program = framework.Program()
scope = fluid.Scope()
# do not use_prune
with fluid.scope_guard(scope):
with fluid.program_guard(program, startup_program):
(x, y, label, loss1, loss2, w_param_attrs) = self.net1()
sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.5)
train1, _ = sgd_optimizer.minimize(loss1)
cloned_program = program.clone()
train2, _ = sgd_optimizer.minimize(loss2)
exe.run(startup_program)
x_np = np.random.random(size=(10, 2)).astype('float32')
label_np = np.random.randint(1, size=(10, 1)).astype('int64')
res = exe.run(program,
feed={
'x': x_np,
'label': label_np
},
fetch_list=[loss1.name],
use_prune=False)
weight_without_prune = np.array(
scope.find_var(w_param_attrs.name).get_tensor())
scope = fluid.Scope()
# use_prune
with fluid.scope_guard(scope):
exe.run(startup_program)
res = exe.run(program,
feed={
'x': x_np,
'label': label_np
},
fetch_list=[loss1.name, train1],
use_prune=True)
weight_with_prune = np.array(
scope.find_var(w_param_attrs.name).get_tensor())
# expected
scope = fluid.Scope()
with fluid.scope_guard(scope):
exe.run(startup_program)
exe.run(cloned_program,
feed={
'x': x_np,
'label': label_np
},
fetch_list=[loss1.name],
use_prune=False)
weight_expected = np.array(
scope.find_var(w_param_attrs.name).get_tensor())
self.assertTrue(np.array_equal(weight_with_prune, weight_expected))
self.assertFalse(np.array_equal(weight_without_prune, weight_expected))
def test_decayed_adagrad_optimizer(self):
init_program = framework.Program()
program = framework.Program()
block = program.global_block()
mul_x = block.create_parameter(dtype="float32",
shape=[5, 10],
lod_level=0,
name="mul.x",
optimize_attr={'learning_rate': 1.1})
mul_y = block.create_var(dtype="float32",
shape=[10, 8],
lod_level=0,
name="mul.y")
mul_out = block.create_var(dtype="float32",
shape=[5, 8],
lod_level=0,
name="mul.out")
block.append_op(type="mul",
inputs={
"X": mul_x,
"Y": mul_y
},
outputs={"Out": mul_out},
attrs={"x_num_col_dims": 1})
mean_out = block.create_var(dtype="float32",
shape=[1],
lod_level=0,
name="mean.out")
block.append_op(type="mean",
inputs={"X": mul_out},
outputs={"Out": mean_out})
learning_rate = 0.01
decayed_adagrad_optimizer = self.MockDecayedAdagrad(
learning_rate=learning_rate, decay=0.95, epsilon=1.0e-6)
params_grads = append_backward(mean_out)
self.assertEqual(len(params_grads), 1)
self.assertEqual(len(decayed_adagrad_optimizer.get_accumulators()), 0)
opts = decayed_adagrad_optimizer.create_optimization_pass(
params_grads, mul_out, init_program)
self.assertEqual(len(opts), 3)
self.assertEqual(
[op.type for op in opts],
["fill_constant", "elementwise_mul", "decayed_adagrad"])
# Check accumulators
accumulators = decayed_adagrad_optimizer.get_accumulators()
self.assertEqual(len(accumulators), 1)
self.assertTrue(
decayed_adagrad_optimizer.get_moment_str() in accumulators)
moment_acc = accumulators[decayed_adagrad_optimizer.get_moment_str()]
self.assertEqual(len(moment_acc), 1)
self.assertTrue(mul_x.name in moment_acc)
# Check init_program
init_ops = init_program.global_block().ops
self.assertEqual(len(init_ops), 2)
self.assertEqual(init_ops[0].type, "fill_constant")
self.assertAlmostEqual(init_ops[0].attr('value'), learning_rate)
self.assertEqual(init_ops[1].type, "fill_constant")
self.assertAlmostEqual(init_ops[1].attr('value'), 0.0)
def test_prune_feed_without_optimizer(self):
program = framework.Program()
startup_program = framework.Program()
scope = fluid.Scope()
with fluid.scope_guard(scope):
with fluid.program_guard(program, startup_program):
(x, y, label, loss1, loss2, w_param_attrs) = self.net1()
exe = fluid.Executor(fluid.CPUPlace())
exe.run(startup_program)
weight_init = np.array(
scope.find_var(w_param_attrs.name).get_tensor())
x_np = np.random.random(size=(10, 2)).astype('float32')
label_np = np.random.randint(1, size=(10, 1)).astype('int64')
res = exe.run(program,
feed={
y.name: x_np,
'label': label_np
},
fetch_list=[loss1.name],
use_prune=True)
self.assertIsNotNone(scope.find_var(loss1.name))
self.assertIsNone(scope.find_var(loss2.name))
weight = np.array(
scope.find_var(w_param_attrs.name).get_tensor())
self.assertTrue(np.array_equal(weight_init,
weight)) # weight unchanged
def test_prune_compiled_program(self):
program = framework.Program()
startup_program = framework.Program()
scope = fluid.Scope()
with fluid.scope_guard(scope):
with fluid.program_guard(program, startup_program):
(x, y, label, loss1, loss2, w_param_attrs) = self.net1()
sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.5)
sgd_optimizer.minimize(loss1)
exe = fluid.Executor(fluid.CPUPlace())
exe.run(startup_program)
compiled_prog = fluid.CompiledProgram(
program).with_data_parallel(loss_name=loss1.name,
places=fluid.CPUPlace())
weight_init = np.array(
scope.find_var(w_param_attrs.name).get_tensor())
x_np = np.random.random(size=(10, 2)).astype('float32')
label_np = np.random.randint(1, size=(10, 1)).astype('int64')
res = exe.run(compiled_prog,
feed={
'x': x_np,
'label': label_np
},
fetch_list=[loss1.name],
use_prune=True)
self.assertIsNotNone(scope.find_var(loss1.name))
self.assertIsNone(scope.find_var(loss2.name))
weight = np.array(
scope.find_var(w_param_attrs.name).get_tensor())
self.assertFalse(np.array_equal(weight_init,
weight)) # weight changed
def test_prune_fetches_with_optimizer(self):
"""
Prune operators and operators which are not needed to generate 'fetches'.
In train mode, the operators and operators in backward and optimization should be kept.
"""
program = framework.Program()
startup_program = framework.Program()
scope = fluid.Scope()
with fluid.scope_guard(scope):
with fluid.program_guard(program, startup_program):
(x, y, label, loss1, loss2, w_param_attrs) = self.net1()
sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.5)
sgd_optimizer.minimize(loss1)
exe = fluid.Executor(fluid.CPUPlace())
exe.run(startup_program)
weight_init = np.array(
scope.find_var(w_param_attrs.name).get_tensor())
x_np = np.random.random(size=(10, 2)).astype('float32')
label_np = np.random.randint(1, size=(10, 1)).astype('int64')
res = exe.run(program,
feed={
'x': x_np,
'label': label_np
},
fetch_list=[loss1.name],
use_prune=True)
self.assertIsNotNone(scope.find_var(loss1.name))
self.assertIsNone(scope.find_var(loss2.name)) #loss2 is pruned
weight = np.array(
scope.find_var(w_param_attrs.name).get_tensor())
self.assertFalse(np.array_equal(weight_init,
weight)) # weight changed
def test_override(self):
# compare func to check
compare_fns = [
lambda _a, _b: _a == _b,
lambda _a, _b: _a != _b,
lambda _a, _b: _a < _b,
lambda _a, _b: _a <= _b,
lambda _a, _b: _a > _b,
lambda _a, _b: _a >= _b,
]
# places to check
places = [fluid.CPUPlace()]
if fluid.core.is_compiled_with_cuda():
places.append(fluid.CUDAPlace(0))
# dtypes to check
dtypes = ['int32', 'float32']
for place in places:
for dtype in dtypes:
for compare_fn in compare_fns:
with framework.program_guard(framework.Program(),
framework.Program()):
self.check_result(compare_fn, place, dtype)
def test_set_global_bias_initilizer(self):
"""Test Set Global Bias initilizer with NormalInitializer
"""
main_prog = framework.Program()
startup_prog = framework.Program()
fluid.set_global_initializer(initializer.Uniform(low=-0.5, high=0.5),
bias_init=initializer.Normal(loc=0.0,
scale=2.0))
with fluid.program_guard(main_prog, startup_prog):
x = fluid.data(name="x", shape=[1, 3, 32, 32])
# default initilizer of bias in layers.conv2d is ConstantInitializer
conv = fluid.layers.conv2d(x, 5, 3)
block = startup_prog.global_block()
self.assertEqual(len(block.ops), 2)
# init bias is the first op, and weight is the second
bias_init_op = block.ops[0]
self.assertEqual(bias_init_op.type, 'gaussian_random')
self.assertAlmostEqual(bias_init_op.attr('mean'), 0.0, delta=DELTA)
self.assertAlmostEqual(bias_init_op.attr('std'), 2.0, delta=DELTA)
self.assertEqual(bias_init_op.attr('seed'), 0)
param_init_op = block.ops[1]
self.assertEqual(param_init_op.type, 'uniform_random')
self.assertAlmostEqual(param_init_op.attr('min'), -0.5, delta=DELTA)
self.assertAlmostEqual(param_init_op.attr('max'), 0.5, delta=DELTA)
self.assertEqual(param_init_op.attr('seed'), 0)
fluid.set_global_initializer(None)
def test_switch(self):
test_data = {(-0.1, 0), (0.1, 1), (1.1, 2), (2.1, 3)}
for x, expected_result in test_data:
main_program = framework.Program()
startup_program = framework.Program()
with framework.program_guard(main_program, startup_program):
result = self.check_switch(x)
self.assertEqual(result, expected_result)
def from_func_spec(func_spec, input_spec, class_instance):
"""
Builds the main_program with specialized inputs and returns outputs
of program as fetch_list.
Args:
func_spec(FunctionSpec): A FunctionSpec instance for decorated function.
input_spec(list[InputSpec]):
"""
# Transforms dygraph function into static function and caches it.
dygraph_function = func_spec.dygraph_function
static_func = convert_to_static(dygraph_function)
main_program, startup_program = framework.Program(), framework.Program()
# Note: The random seed should be synchronized into cached program
# if set in `fluid.dygraph_guard` because some ops rely on it, such as
# `fluid.layers.dropout`.
main_program.random_seed = framework.default_main_program().random_seed
startup_program.random_seed = framework.default_startup_program(
).random_seed
with framework.program_guard(main_program, startup_program):
with _switch_declarative_mode_guard_(is_declarative=True):
# 1. Adds `fluid.data` layers for input if needed
inputs = func_spec.to_static_inputs_with_spec(input_spec,
main_program)
if class_instance:
inputs = tuple([class_instance] + list(inputs))
# 2. Gets all ParamBases and buffered VarBases in the function
all_parameters_and_buffers = list(
get_parameters(class_instance).values()) + list(
get_buffers(class_instance).values())
# 3. Builds program only once and returns the output Variables.
with param_guard(get_parameters(
class_instance, False)), param_guard(
get_buffers(class_instance, False)):
try:
outputs = static_func(*inputs)
except BaseException as e:
# NOTE: If e is raised in compile time, e should be attached to ERROR_DATA here.
attach_error_data(e)
raise
if not isinstance(outputs,
(tuple, list)) and outputs is not None:
outputs = [outputs]
main_program = update_op_callstack_with_origin_info(main_program)
return ConcreteProgram(
inputs=inputs,
outputs=outputs,
parameters=all_parameters_and_buffers,
function=dygraph_function,
main_program=main_program,
startup_program=startup_program)
def save_program_desc(network_func):
startup_program = framework.Program()
train_program = framework.Program()
with framework.program_guard(train_program, startup_program):
network_func(with_optimize=False)
with open("startup_program", "w") as f:
f.write(startup_program.desc.serialize_to_string())
with open("main_program", "w") as f:
f.write(train_program.desc.serialize_to_string())
def save_program_desc():
startup_program = framework.Program()
train_program = framework.Program()
with framework.program_guard(train_program, startup_program):
minist_classfication_network()
with open("startup_program", "w") as f:
f.write(startup_program.desc.serialize_to_string())
with open("main_program", "w") as f:
f.write(train_program.desc.serialize_to_string())
def save_program_desc(network_func):
startup_program = framework.Program()
train_program = framework.Program()
with framework.program_guard(train_program, startup_program):
# build graph here
network_func(with_optimize=is_trainable)
with open("startup_program", "wb") as f:
f.write(startup_program.desc.serialize_to_string())
with open("main_program", "wb") as f:
f.write(train_program.desc.serialize_to_string())
def test_decay(self):
common_kwargs_true = {
"learning_rate": 1.0,
"decay_steps": 5,
"decay_rate": 0.5,
"staircase": True
}
common_kwargs_false = copy.deepcopy(common_kwargs_true)
common_kwargs_false["staircase"] = False
decay_fns = [
(exponential_decay, layers.exponential_decay, common_kwargs_true),
(exponential_decay, layers.exponential_decay, common_kwargs_false),
(natural_exp_decay, layers.natural_exp_decay, common_kwargs_true),
(natural_exp_decay, layers.natural_exp_decay, common_kwargs_false),
(inverse_time_decay, layers.inverse_time_decay,
common_kwargs_true),
(inverse_time_decay, layers.inverse_time_decay,
common_kwargs_false),
(polynomial_decay, layers.polynomial_decay, {
"learning_rate": 1.0,
"decay_steps": 5,
"cycle": True
}),
(polynomial_decay, layers.polynomial_decay, {
"learning_rate": 1.0,
"decay_steps": 5,
"cycle": False
}),
(piecewise_decay, layers.piecewise_decay, {
"boundaries": [3, 6, 9],
"values": [0.1, 0.2, 0.3, 0.4]
}),
(cosine_decay, layers.cosine_decay, {
"learning_rate": 0.1,
"step_each_epoch": 100,
"epochs": 120
}),
(noam_decay, layers.noam_decay, {
"d_model": 0.01,
"warmup_steps": 200,
"learning_rate": 2.0
})
]
for py_decay_fn, fluid_decay_fn, kwargs in decay_fns:
print("class=" + self.__class__.__name__ + " decay_fn=" +
py_decay_fn.__name__ + " kwargs=" + str(kwargs))
main_program = framework.Program()
startup_program = framework.Program()
with framework.program_guard(main_program, startup_program):
self.check_decay(py_decay_fn, fluid_decay_fn, kwargs)
def test_prune_with_cache_program2(self):
'''
When use_prune=True, Executor should cache the pruned program.
If the only difference in fetch_list is optimize_ops during multiple runs,
the cache_keys should be different and get different pruned program.
'''
with _mock_guard(mock):
exe = fluid.Executor(fluid.CPUPlace())
exe.prune_called_times = 0
program = framework.Program()
startup_program = framework.Program()
scope = fluid.Scope()
with fluid.scope_guard(scope):
with fluid.program_guard(program, startup_program):
(x1, x2, y1, y2, label, loss1, loss2, w1_param_attrs,
w2_param_attrs) = self.net2()
adam_optimizer1 = fluid.optimizer.AdamOptimizer(
learning_rate=0.5)
train1 = adam_optimizer1.minimize(loss1)
adam_optimizer2 = fluid.optimizer.AdamOptimizer(
learning_rate=0.5)
train2 = adam_optimizer2.minimize(loss2)
exe.run(startup_program)
x_np = np.random.random(size=(10, 2)).astype('float32')
label_np = np.random.randint(1,
size=(10, 1)).astype('int64')
for i in range(10):
if i % 2:
res = exe.run(program,
feed={
'x1': x_np,
'x2': x_np,
'label': label_np
},
fetch_list=[loss1, loss2, train1],
use_prune=True)
else:
res = exe.run(program,
feed={
'x1': x_np,
'x2': x_np,
'label': label_np
},
fetch_list=[loss1, loss2, train2],
use_prune=True)
if i == 0:
self.assertEqual(exe.prune_called_times, 1)
elif i == 1:
self.assertEqual(exe.prune_called_times, 2)
else:
self.assertEqual(exe.prune_called_times, 2)
def test_prune_with_input(self):
program = framework.Program()
startup_program = framework.Program()
block = program.global_block()
with fluid.program_guard(program, startup_program):
(x, y, label, loss) = self.net()
self.assertEqual(len(block.ops), 5)
self.assertEqual(
[op.type for op in block.ops],
["mul", "elementwise_add", "softmax", "cross_entropy2", "mean"])
pruned_program = program._prune_with_input(
feeded_var_names=[y.name, label.name], targets=[loss])
self.assertEqual(len(pruned_program.global_block().ops), 2)
self.assertEqual([op.type for op in pruned_program.global_block().ops],
["cross_entropy2", "mean"])
def test_prune_target_not_list(self):
program = framework.Program()
startup_program = framework.Program()
block = program.global_block()
with fluid.program_guard(program, startup_program):
(x, y, label, loss) = self.net()
self.assertEqual(len(block.ops), 5)
self.assertEqual(
[op.type for op in block.ops],
["mul", "elementwise_add", "softmax", "cross_entropy2", "mean"])
pruned_program = program._prune(targets=loss)
self.assertEqual(len(pruned_program.global_block().ops), 5)
self.assertEqual(
[op.type for op in pruned_program.global_block().ops],
["mul", "elementwise_add", "softmax", "cross_entropy2", "mean"])
def guard(place=None):
train = framework.Program()
startup = framework.Program()
tracer = Tracer(train.current_block().desc)
if place is None:
if core.is_compiled_with_cuda():
place = core.CUDAPlace(0)
else:
place = core.CPUPlace()
with framework.program_guard(train, startup):
with framework.unique_name.guard():
with framework._dygraph_guard(tracer):
with framework._dygraph_place_guard(place):
yield
def test_xavier_normal_initializer_conv(self):
"""Test Xavier initializer with normal distribution on
for convolutions.
"""
paddle.enable_static()
program = framework.Program()
block = program.global_block()
for _ in range(2):
param = block.create_parameter(
dtype="float32",
shape=[5, 10, 15, 20],
lod_level=0,
name="param",
initializer=initializer.XavierNormal())
self.assertEqual(len(block.ops), 1)
init_op = block.ops[0]
self.assertEqual(init_op.type, 'gaussian_random')
receptive_field_size = float(15 * 20)
std = np.sqrt(
2.0 / ((param.shape[0] + param.shape[1]) * receptive_field_size))
self.assertAlmostEqual(init_op.attr('mean'), 0.0, delta=DELTA)
self.assertAlmostEqual(init_op.attr('std'), std, delta=DELTA)
self.assertEqual(init_op.attr('seed'), 0)
paddle.disable_static()
def test_uniform_initializer_two_op(self,
dtype="float32",
seed=123,
min_value=-4.2,
max_vlaue=0.0):
"""Test uniform initializer with supplied attributes
"""
paddle.enable_static()
program = framework.Program()
program.random_seed = seed
block = program.global_block()
for i in range(2):
block.create_parameter(dtype=dtype,
shape=[5, 10],
lod_level=0,
name="param",
initializer=initializer.Uniform(
min_value, float(i)))
num_ops = 2 if dtype == "float16" else 1
self.assertEqual(len(block.ops), num_ops)
init_op0 = block.ops[0]
self.assertEqual(init_op0.type, 'uniform_random')
self.assertAlmostEqual(init_op0.attr('min'), min_value, delta=DELTA)
self.assertAlmostEqual(init_op0.attr('max'), 0.0, delta=DELTA)
self.assertEqual(init_op0.attr("seed"), seed)
paddle.disable_static()
return block
def test_uniform_initializer_default_value(self,
dtype="float32",
seed=0,
min_value=-1.0,
max_vlaue=1.0):
"""Test the uniform initializer with default value
"""
paddle.enable_static()
program = framework.Program()
program.random_seed = seed
block = program.global_block()
for _ in range(2):
block.create_parameter(dtype=dtype,
shape=[5, 10],
lod_level=0,
name="param",
initializer=initializer.Uniform())
num_ops = 2 if dtype == "float16" else 1
self.assertEqual(len(block.ops), num_ops)
init_op = block.ops[0]
self.assertEqual(init_op.type, 'uniform_random')
self.assertAlmostEqual(init_op.attr('min'), min_value, delta=DELTA)
self.assertAlmostEqual(init_op.attr('max'), max_vlaue, delta=DELTA)
self.assertEqual(init_op.attr('seed'), seed)
paddle.disable_static()
return block
def test_prune_target_none(self):
program = framework.Program()
startup_program = framework.Program()
block = program.global_block()
with fluid.program_guard(program, startup_program):
(x, y, label, loss) = self.net()
self.assertEqual(len(block.ops), 5)
self.assertEqual(
[op.type for op in block.ops],
["mul", "elementwise_add", "softmax", "cross_entropy2", "mean"])
try:
pruned_program = program._prune(targets=None)
except ValueError as e:
self.assertEqual(
"All targets of prune() can only be Variable or Operator.",
cpt.get_exception_message(e))
def test_xavier_initializer_supplied_arguments(self,
dtype="float32",
uniform=True):
"""Test the Xavier initializer with supplied arguments
"""
program = framework.Program()
block = program.global_block()
for _ in range(2):
block.create_parameter(
dtype=dtype,
shape=[5, 10],
lod_level=0,
name="param",
initializer=initializer.XavierInitializer(
uniform=uniform, fan_in=12, fan_out=23, seed=134))
num_ops = 2 if (dtype == "float16" or (dtype == "uint16" and
not uniform)) else 1
self.assertEqual(len(block.ops), num_ops)
init_op = block.ops[0]
if uniform:
self.assertEqual(init_op.type, 'uniform_random')
limit = np.sqrt(6.0 / (12 + 23))
self.assertAlmostEqual(init_op.attr('min'), -limit, delta=DELTA)
self.assertAlmostEqual(init_op.attr('max'), limit, delta=DELTA)
else:
self.assertEqual(init_op.type, 'gaussian_random')
self.assertEqual(init_op.attr('seed'), 134)
return block
def test_l2decay_regularizer(self):
program = framework.Program()
block = program.global_block()
mul_x = block.create_parameter(
dtype="float32",
shape=[5, 10],
lod_level=0,
name="mul.x",
regularizer=regularizer.L1DecayRegularizer(0.5))
self.assertTrue(mul_x.regularizer is not None)
self.assertTrue(
isinstance(mul_x.regularizer, regularizer.L1DecayRegularizer))
mul_y = block.create_var(
dtype="float32", shape=[10, 8], lod_level=0, name="mul.y")
mul_out = block.create_var(
dtype="float32", shape=[5, 8], lod_level=0, name="mul.out")
block.append_op(
type="mul",
inputs={"X": mul_x,
"Y": mul_y},
outputs={"Out": mul_out},
attrs={"x_num_col_dims": 1})
mean_out = block.create_var(
dtype="float32", shape=[1], lod_level=0, name="mean.out")
block.append_op(
type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out})
params_grads = append_backward(mean_out)
self.assertEqual(len(params_grads), 1)
count_ops = len(block.ops)
params_grads = optimizer.append_regularization_ops(params_grads)
self.assertEqual(len(params_grads), 1)
self.assertEqual(len(block.ops), count_ops + 3)
self.assertEqual(block.ops[-1].type, 'sum')
self.assertEqual(block.ops[-2].type, 'scale')
self.assertEqual(block.ops[-3].type, 'sign')
def test_select(self):
with framework.program_guard(framework.Program()):
ch1 = fluid.make_channel(dtype=core.VarDesc.VarType.LOD_TENSOR,
capacity=1)
result1 = self._create_tensor('return_value',
core.VarDesc.VarType.LOD_TENSOR,
core.VarDesc.VarType.FP64)
input_value = fill_constant(shape=[1],
dtype=core.VarDesc.VarType.FP64,
value=10)
with fluid.Select() as select:
with select.case(fluid.channel_send, ch1, input_value):
# Execute something.
pass
with select.default():
pass
# This should not block because we are using a buffered channel.
result1, status = fluid.channel_recv(ch1, result1)
fluid.channel_close(ch1)
cpu = core.CPUPlace()
exe = Executor(cpu)
result = exe.run(fetch_list=[result1])
self.assertEqual(result[0][0], 10)
def split_heter_worker_ops_pass(program, config):
"""
split heter worker program from origin-program
1. find heter op (located on different device)
2. find input&output of every heter-block
3. create heter worker program, add listen&serv op
"""
default_deveice = "cpu"
program, heter_ops, _, program_block_ops = find_heter_ops(
program, default_deveice)
if len(heter_ops) == 0:
warnings.warn(
"Currently running in Heter Parameter Server mode, but no OP running on heterogeneous devices, Please check your code."
)
return program
current_device = "gpu"
if current_device not in heter_ops:
raise ValueError(
"Op which run on device {} not exist.".format(current_device))
block_vars_detail = find_block_joints(program, program_block_ops,
heter_ops)
heter_program = framework.Program()
create_heter_program(program, config, heter_program, heter_ops,
block_vars_detail, current_device)
return heter_program
