def test_with_asp_sharding(self):
fleet.init(is_collective=True)
train_prog, startup_prog = fluid.Program(), fluid.Program()
avg_cost, strategy, input_x, input_y = self.net(
train_prog, startup_prog)
with fluid.program_guard(train_prog, startup_prog):
optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01)
optimizer = fleet.distributed_optimizer(optimizer,
strategy=strategy)
optimizer.minimize(avg_cost)
if paddle.fluid.is_compiled_with_cuda():
place = fluid.CUDAPlace(
int(os.environ.get('FLAGS_selected_gpus', 0)))
else:
place = fluid.CPUPlace()
exe = fluid.Executor(place)
feeder = fluid.DataFeeder(feed_list=[input_x, input_y], place=place)
exe.run(startup_prog)
sparsity.prune_model(train_prog)
data = (np.random.randn(64, 32), np.random.randint(2, size=(64, 1)))
exe.run(train_prog, feed=feeder.feed([data]))
for param in train_prog.global_block().all_parameters():
if ASPHelper._is_supported_layer(train_prog, param.name):
mat = np.array(fluid.global_scope().find_var(
param.name).get_tensor())
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
def test_asp_training_with_amp(self):
if core.is_compiled_with_cuda():
place = paddle.CUDAPlace(0)
with fluid.program_guard(self.main_program, self.startup_program):
self.optimizer = fluid.contrib.mixed_precision.decorator.decorate(
self.optimizer)
self.optimizer = sparsity.decorate(self.optimizer)
self.optimizer.minimize(self.loss, self.startup_program)
exe = fluid.Executor(place)
feeder = fluid.DataFeeder(
feed_list=[self.img, self.label], place=place)
exe.run(self.startup_program)
sparsity.prune_model(self.main_program)
data = (np.random.randn(64, 3, 32, 32), np.random.randint(
10, size=(64, 1)))
exe.run(self.main_program, feed=feeder.feed([data]))
for param in self.main_program.global_block().all_parameters():
if ASPHelper._is_supported_layer(self.main_program, param.name):
mat = np.array(fluid.global_scope().find_var(param.name)
.get_tensor())
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, n=2, m=4))
def __pruning_and_checking(self, exe, place, mask_func_name,
check_func_name, with_mask):
exe.run(self.startup_program)
sparsity.prune_model(self.main_program,
mask_algo=mask_func_name,
with_mask=with_mask)
for param in self.main_program.global_block().all_parameters():
if ASPHelper._is_supported_layer(self.main_program, param.name):
mat = np.array(fluid.global_scope().find_var(
param.name).get_tensor())
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, func_name=check_func_name, n=2, m=4))
def test_with_asp_and_pure_fp16(self):
fleet.init(is_collective=True)
train_prog, startup_prog = fluid.Program(), fluid.Program()
with paddle.static.amp.fp16_guard():
avg_cost, strategy, \
input_x, input_y = self.net(train_prog,
startup_prog)
strategy.amp = True
strategy.amp_configs = {'use_pure_fp16': True}
with fluid.program_guard(train_prog, startup_prog):
with paddle.static.amp.fp16_guard():
optimizer = optimizer = paddle.optimizer.Momentum(
learning_rate=0.01, multi_precision=True)
optimizer = fleet.distributed_optimizer(
optimizer, strategy=strategy)
optimizer.minimize(avg_cost)
place = fluid.CUDAPlace(0) if paddle.fluid.is_compiled_with_cuda(
) else fluid.CPUPlace()
exe = fluid.Executor(place)
feeder = fluid.DataFeeder(feed_list=[input_x, input_y], place=place)
exe.run(startup_prog)
optimizer.amp_init(place)
sparsity.prune_model(train_prog)
data = (np.random.randn(64, 32), np.random.randint(2, size=(64, 1)))
exe.run(train_prog, feed=feeder.feed([data]))
for param in train_prog.global_block().all_parameters():
if ASPHelper._is_supported_layer(train_prog, param.name):
mat = np.array(fluid.global_scope().find_var(param.name)
.get_tensor())
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, n=2, m=4))