def _test_static(self, python_func, paddle_api, kwarg, place): scheduler = paddle_api(**kwarg) adam = paddle.optimizer.Adam(learning_rate=scheduler) main_prog = paddle.static.Program() start_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, start_prog): x = paddle.static.data(name='x', shape=[3, 4, 5]) loss = paddle.mean(x) adam.minimize(loss) lr_var = adam._global_learning_rate() test_prog = main_prog.clone() num = 0 exe = paddle.static.Executor(place) exe.run(start_prog) for epoch in range(5): for batch_id in range(2): out = exe.run( main_prog, feed={'x': np.random.randn(3, 4, 5).astype('float32')}, fetch_list=lr_var.name) self.assertEqual(out, np.array(python_func(num, **kwarg))) scheduler.step() num += 1 for epoch in range(5): for batch_id in range(2): out = exe.run( test_prog, feed={'x': np.random.randn(3, 4, 5).astype('float32')}, fetch_list=lr_var.name) self.assertEqual(out, np.array(python_func(num, **kwarg))) scheduler.step() num += 1 if isinstance(place, paddle.CPUPlace): compiled_train_prog = paddle.static.CompiledProgram( main_prog).with_data_parallel(loss_name=loss.name, places=fluid.cpu_places(4)) for epoch in range(5): python_result = python_func(num, **kwarg) for batch_id in range(2): _ = exe.run(compiled_train_prog, feed={ 'x': np.random.randn(12, 4, 5).astype('float32') }, fetch_list=lr_var.name) scopes = compiled_train_prog._executor.local_scopes() out = np.array(scopes[0].var(lr_var.name).get_tensor()) self.assertEqual(out, np.array(python_result)) out = np.array(scopes[1].var(lr_var.name).get_tensor()) self.assertEqual(out, np.array(python_result)) out = np.array(scopes[2].var(lr_var.name).get_tensor()) self.assertEqual(out, np.array(python_result)) out = np.array(scopes[3].var(lr_var.name).get_tensor()) self.assertEqual(out, np.array(python_result)) scheduler.step() num += 1 compiled_test_prog = paddle.static.CompiledProgram( test_prog).with_data_parallel( loss_name=loss.name, share_vars_from=compiled_train_prog, places=fluid.cpu_places(4)) for epoch in range(5): python_result = python_func(num, **kwarg) for batch_id in range(2): _ = exe.run(compiled_test_prog, feed={ 'x': np.random.randn(12, 4, 5).astype('float32') }, fetch_list=lr_var.name) scopes = compiled_test_prog._executor.local_scopes() out = np.array(scopes[0].var(lr_var.name).get_tensor()) self.assertEqual(out, np.array(python_result)) out = np.array(scopes[1].var(lr_var.name).get_tensor()) self.assertEqual(out, np.array(python_result)) out = np.array(scopes[2].var(lr_var.name).get_tensor()) self.assertEqual(out, np.array(python_result)) out = np.array(scopes[3].var(lr_var.name).get_tensor()) self.assertEqual(out, np.array(python_result)) scheduler.step() num += 1
ernie = ErnieForTokenClassification.from_pretrained( "ernie-1.0", num_classes=len(label_vocab)) model = ErnieCrfForTokenClassification(ernie) metric = ChunkEvaluator(label_list=label_vocab.keys(), suffix=True) optimizer = paddle.optimizer.AdamW(learning_rate=2e-5, parameters=model.parameters()) step = 0 for epoch in range(args.epochs): for input_ids, token_type_ids, lengths, labels in train_loader: loss = model(input_ids, token_type_ids, lengths=lengths, labels=labels) avg_loss = paddle.mean(loss) avg_loss.backward() optimizer.step() optimizer.clear_grad() step += 1 print("[TRAIN] Epoch:%d - Step:%d - Loss: %f" % (epoch, step, avg_loss)) evaluate(model, metric, dev_loader) paddle.save(model.state_dict(), os.path.join(args.save_dir, 'model_%d' % step)) preds = predict(model, test_loader, test_ds, label_vocab) file_path = "ernie_crf_results.txt" with open(file_path, "w", encoding="utf8") as fout: fout.write("\n".join(preds))
def test_accuracy(self): image = paddle.static.data(name='image', shape=[None, 1, 28, 28], dtype='float32') label = paddle.static.data(name='label', shape=[None, 1], dtype='int64') model = MobileNet() out = model.net(input=image, class_dim=10) cost = paddle.nn.functional.loss.cross_entropy(input=out, label=label) avg_cost = paddle.mean(x=cost) acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1) acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5) optimizer = paddle.optimizer.Momentum( momentum=0.9, learning_rate=0.01, weight_decay=paddle.regularizer.L2Decay(4e-5)) optimizer.minimize(avg_cost) main_prog = paddle.static.default_main_program() val_prog = main_prog.clone(for_test=True) place = paddle.CUDAPlace( 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) def transform(x): return np.reshape(x, [1, 28, 28]) train_dataset = paddle.vision.datasets.MNIST(mode='train', backend='cv2', transform=transform) test_dataset = paddle.vision.datasets.MNIST(mode='test', backend='cv2', transform=transform) train_loader = paddle.io.DataLoader(train_dataset, places=place, feed_list=[image, label], drop_last=True, batch_size=64, return_list=False) valid_loader = paddle.io.DataLoader(test_dataset, places=place, feed_list=[image, label], batch_size=64, return_list=False) def sample_generator_creator(): def __reader__(): for data in test_dataset: image, label = data yield image, label return __reader__ def train(program): iter = 0 for data in train_loader(): cost, top1, top5 = exe.run( program, feed=data, fetch_list=[avg_cost, acc_top1, acc_top5]) iter += 1 if iter % 100 == 0: print( 'train iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) def test(program, outputs=[avg_cost, acc_top1, acc_top5]): iter = 0 result = [[], [], []] for data in valid_loader(): cost, top1, top5 = exe.run(program, feed=data, fetch_list=outputs) iter += 1 if iter % 100 == 0: print( 'eval iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) result[0].append(cost) result[1].append(top1) result[2].append(top5) print(' avg loss {}, acc_top1 {}, acc_top5 {}'.format( np.mean(result[0]), np.mean(result[1]), np.mean(result[2]))) return np.mean(result[1]), np.mean(result[2]) train(main_prog) top1_1, top5_1 = test(val_prog) paddle.fluid.io.save_inference_model( dirname='./test_quant_post', feeded_var_names=[image.name, label.name], target_vars=[avg_cost, acc_top1, acc_top5], main_program=val_prog, executor=exe, model_filename='model', params_filename='params') quant_post_static(exe, './test_quant_post', './test_quant_post_inference', sample_generator=sample_generator_creator(), model_filename='model', params_filename='params', batch_nums=10) quant_post_prog, feed_target_names, fetch_targets = paddle.fluid.io.load_inference_model( dirname='./test_quant_post_inference', executor=exe, model_filename='__model__', params_filename='__params__') top1_2, top5_2 = test(quant_post_prog, fetch_targets) print("before quantization: top1: {}, top5: {}".format(top1_1, top5_1)) print("after quantization: top1: {}, top5: {}".format(top1_2, top5_2))
def create_loss(self, raw_pred, label): loss = paddle.nn.functional.log_loss(input=raw_pred, label=paddle.cast( label, "float32")) loss = paddle.mean(loss) return loss
# Define the model netword and its loss model = ErnieForTokenClassification.from_pretrained( "ernie-1.0", num_classes=len(label_vocab)) if trainer_num > 1: model = paddle.DataParallel(model) metric = ChunkEvaluator(label_list=label_vocab.keys(), suffix=True) loss_fn = paddle.nn.loss.CrossEntropyLoss(ignore_index=ignore_label) optimizer = paddle.optimizer.AdamW(learning_rate=2e-5, parameters=model.parameters()) step = 0 for epoch in range(args.epochs): for input_ids, token_type_ids, length, labels in train_loader: logits = model(input_ids, token_type_ids) loss = paddle.mean(loss_fn(logits, labels)) loss.backward() optimizer.step() optimizer.clear_grad() step += 1 print("[TRAIN] Epoch:%d - Step:%d - Loss: %f" % (epoch, step, loss)) evaluate(model, metric, dev_loader) model_to_save = model._layers if isinstance( model, paddle.DataParallel) else model model_to_save.save_pretrained( os.path.join(args.save_dir, 'model_%d' % step)) if rank == 0: preds = predict(model, test_loader, test_ds, label_vocab) file_path = "ernie_results.txt"
def forward(self, logits, label): return paddle.mean(F.softmax_with_cross_entropy(logits, label))
import paddle img = paddle.randn(shape=[100]) img.stop_gradient = False img[10] = img[10] + 10.0 out = img * 2 rlt = paddle.mean(out) rlt.backward()
import os import paddle import paddle.static as static paddle.enable_static() os.environ['CPU_NUM'] = str(2) places = static.cpu_places() data = static.data(name="x", shape=[None, 1], dtype="float32") hidden = static.nn.fc(input=data, size=10) loss = paddle.mean(hidden) paddle.optimizer.SGD(learning_rate=0.01).minimize(loss) build_strategy = static.BuildStrategy() build_strategy.enable_inplace = True build_strategy.memory_optimize = True build_strategy.reduce_strategy = static.BuildStrategy.ReduceStrategy.Reduce program = static.CompiledProgram(static.default_main_program()) program = program.with_data_parallel(loss_name=loss.name, build_strategy=build_strategy, places=places)
def adaptation_loss(t_var, s_var): hint_loss = paddle.mean( paddle.nn.functional.square_error_cost(s_var, t_var)) return hint_loss
def train_resnet(self, enable_amp=True, use_data_loader=False): seed = 90 batch_size = train_parameters["batch_size"] batch_num = 1 paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) resnet = ResNet(use_cudnn=True) optimizer = optimizer_setting( train_parameters, parameter_list=resnet.parameters()) np.random.seed(seed) train_reader = paddle.batch( paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size) dy_param_init_value = {} for param in resnet.parameters(): dy_param_init_value[param.name] = param.numpy() program = None scaler = paddle.amp.GradScaler( enable=enable_amp, init_loss_scaling=2.**10) if use_data_loader: train_reader = paddle.batch( reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), batch_size=batch_size, drop_last=True) train_loader = fluid.io.DataLoader.from_generator( capacity=4, use_double_buffer=True, iterable=True, return_list=True) train_loader.set_sample_list_generator(train_reader) train_reader = train_loader for batch_id, data in enumerate(train_reader()): if batch_id >= batch_num: break if use_data_loader: img, label = data else: dy_x_data = np.array( [x[0].reshape(3, 224, 224) for x in data]).astype('float32') if len(np.array([x[1] for x in data]).astype('int64')) != batch_size: continue y_data = np.array([x[1] for x in data]).astype('int64').reshape( -1, 1) img = paddle.to_tensor(dy_x_data) label = paddle.to_tensor(y_data) label.stop_gradient = True with paddle.amp.auto_cast(enable=enable_amp): out = resnet(img) loss = paddle.nn.functional.cross_entropy(input=out, label=label) avg_loss = paddle.mean(x=loss) dy_out = avg_loss.numpy() scaled_loss = scaler.scale(avg_loss) scaled_loss.backward() scaler.minimize(optimizer, scaled_loss) dy_grad_value = {} for param in resnet.parameters(): if param.trainable: np_array = np.array(param._grad_ivar().value().get_tensor()) dy_grad_value[param.name + fluid.core.grad_var_suffix( )] = np_array resnet.clear_gradients() dy_param_value = {} for param in resnet.parameters(): dy_param_value[param.name] = param.numpy() if use_data_loader: train_reader._reset() return dy_out, dy_param_value, dy_grad_value
def skip_gram_word2vec(dict_size, embedding_size, is_sparse=False, neg_num=5): datas = [] input_word = paddle.static.data(name="input_word", shape=[None, 1], dtype='int64') true_word = paddle.static.data(name='true_label', shape=[None, 1], dtype='int64') neg_word = paddle.static.data(name="neg_label", shape=[None, neg_num], dtype='int64') datas.append(input_word) datas.append(true_word) datas.append(neg_word) py_reader = fluid.layers.create_py_reader_by_data(capacity=64, feed_list=datas, name='py_reader', use_double_buffer=True) words = fluid.layers.read_file(py_reader) words[0] = paddle.reshape(words[0], [-1]) words[1] = paddle.reshape(words[1], [-1]) init_width = 0.5 / embedding_size input_emb = paddle.static.nn.embedding( input=words[0], is_sparse=is_sparse, size=[dict_size, embedding_size], param_attr=paddle.ParamAttr(name='emb', initializer=paddle.nn.initializer.Uniform( -init_width, init_width))) true_emb_w = paddle.static.nn.embedding( input=words[1], is_sparse=is_sparse, size=[dict_size, embedding_size], param_attr=paddle.ParamAttr( name='emb_w', initializer=paddle.nn.initializer.Constant(value=0.0))) true_emb_b = paddle.static.nn.embedding( input=words[1], is_sparse=is_sparse, size=[dict_size, 1], param_attr=paddle.ParamAttr( name='emb_b', initializer=paddle.nn.initializer.Constant(value=0.0))) neg_word_reshape = paddle.reshape(words[2], shape=[-1]) neg_word_reshape.stop_gradient = True neg_emb_w = paddle.static.nn.embedding( input=neg_word_reshape, is_sparse=is_sparse, size=[dict_size, embedding_size], param_attr=paddle.ParamAttr(name='emb_w', learning_rate=1.0)) neg_emb_w_re = paddle.reshape(neg_emb_w, shape=[-1, neg_num, embedding_size]) neg_emb_b = paddle.static.nn.embedding( input=neg_word_reshape, is_sparse=is_sparse, size=[dict_size, 1], param_attr=paddle.ParamAttr(name='emb_b', learning_rate=1.0)) neg_emb_b_vec = paddle.reshape(neg_emb_b, shape=[-1, neg_num]) true_logits = paddle.add( paddle.mean(paddle.multiply(input_emb, true_emb_w), axis=1, keepdim=True), true_emb_b) input_emb_re = paddle.reshape(input_emb, shape=[-1, 1, embedding_size]) neg_matmul = fluid.layers.matmul(input_emb_re, neg_emb_w_re, transpose_y=True) neg_matmul_re = paddle.reshape(neg_matmul, shape=[-1, neg_num]) neg_logits = paddle.add(neg_matmul_re, neg_emb_b_vec) #nce loss label_ones = fluid.layers.fill_constant_batch_size_like(true_logits, shape=[-1, 1], value=1.0, dtype='float32') label_zeros = fluid.layers.fill_constant_batch_size_like( true_logits, shape=[-1, neg_num], value=0.0, dtype='float32') true_xent = fluid.layers.sigmoid_cross_entropy_with_logits( true_logits, label_ones) neg_xent = fluid.layers.sigmoid_cross_entropy_with_logits( neg_logits, label_zeros) cost = paddle.add(paddle.sum(true_xent, axis=1), paddle.sum(neg_xent, axis=1)) avg_cost = paddle.mean(cost) return avg_cost, py_reader
def obj_loss(self, sobj, tobj): obj_mask = paddle.cast(tobj > 0., dtype="float32") obj_mask.stop_gradient = True loss = paddle.mean( ops.sigmoid_cross_entropy_with_logits(sobj, obj_mask)) return loss
def obj_weighted_cls(self, scls, tcls, tobj): loss = ops.sigmoid_cross_entropy_with_logits(scls, F.sigmoid(tcls)) weighted_loss = paddle.mean(paddle.multiply(loss, F.sigmoid(tobj))) return weighted_loss
def train(to_static): """ Tests model decorated by `dygraph_to_static_output` in static mode. For users, the model is defined in dygraph mode and trained in static mode. """ paddle.disable_static(place) np.random.seed(SEED) paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) train_reader = paddle.batch(reader_decorator( paddle.dataset.flowers.train(use_xmap=False)), batch_size=batch_size, drop_last=True) data_loader = paddle.io.DataLoader.from_generator(capacity=5, iterable=True) data_loader.set_sample_list_generator(train_reader) resnet = ResNet() optimizer = optimizer_setting(parameter_list=resnet.parameters()) for epoch in range(epoch_num): total_loss = 0.0 total_acc1 = 0.0 total_acc5 = 0.0 total_sample = 0 for batch_id, data in enumerate(data_loader()): start_time = time.time() img, label = data pred = resnet(img) loss = paddle.nn.functional.cross_entropy(input=pred, label=label) avg_loss = paddle.mean(x=loss) acc_top1 = paddle.metric.accuracy(input=pred, label=label, k=1) acc_top5 = paddle.metric.accuracy(input=pred, label=label, k=5) avg_loss.backward() optimizer.minimize(avg_loss) resnet.clear_gradients() total_loss += avg_loss total_acc1 += acc_top1 total_acc5 += acc_top5 total_sample += 1 end_time = time.time() if batch_id % 2 == 0: print( "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" % \ ( epoch, batch_id, total_loss.numpy() / total_sample, \ total_acc1.numpy() / total_sample, total_acc5.numpy() / total_sample, end_time-start_time)) if batch_id == 10: if to_static: paddle.jit.save(resnet, MODEL_SAVE_PREFIX) else: paddle.fluid.dygraph.save_dygraph(resnet.state_dict(), DY_STATE_DICT_SAVE_PATH) # avoid dataloader throw abort signaal data_loader._reset() break paddle.enable_static() return total_loss.numpy()
def create_loss(self, hit_prob): loss = -paddle.sum(paddle.log(hit_prob), axis=-1) avg_cost = paddle.mean(x=loss) return avg_cost
def create_loss(self, pred, label): cost = paddle.nn.functional.cross_entropy(input=pred, label=label) avg_cost = paddle.mean(x=cost) return avg_cost
def compress(args): ############################################################################################################ # 1. quantization configs ############################################################################################################ quant_config = { # weight quantize type, default is 'channel_wise_abs_max' 'weight_quantize_type': 'channel_wise_abs_max', # activation quantize type, default is 'moving_average_abs_max' 'activation_quantize_type': 'moving_average_abs_max', # weight quantize bit num, default is 8 'weight_bits': 8, # activation quantize bit num, default is 8 'activation_bits': 8, # ops of name_scope in not_quant_pattern list, will not be quantized 'not_quant_pattern': ['skip_quant'], # ops of type in quantize_op_types, will be quantized 'quantize_op_types': ['conv2d', 'depthwise_conv2d', 'mul'], # data type after quantization, such as 'uint8', 'int8', etc. default is 'int8' 'dtype': 'int8', # window size for 'range_abs_max' quantization. defaulf is 10000 'window_size': 10000, # The decay coefficient of moving average, default is 0.9 'moving_rate': 0.9, } if args.data == "mnist": train_dataset = paddle.vision.datasets.MNIST(mode='train') val_dataset = paddle.vision.datasets.MNIST(mode='test') class_dim = 10 image_shape = "1,28,28" elif args.data == "imagenet": import imagenet_reader as reader train_dataset = reader.ImageNetDataset(mode='train') val_dataset = reader.ImageNetDataset(mode='val') class_dim = 1000 image_shape = "3,224,224" else: raise ValueError("{} is not supported.".format(args.data)) image_shape = [int(m) for m in image_shape.split(",")] assert args.model in model_list, "{} is not in lists: {}".format( args.model, model_list) image = paddle.static.data(name='image', shape=[None] + image_shape, dtype='float32') label = paddle.static.data(name='label', shape=[None, 1], dtype='int64') # model definition model = models.__dict__[args.model]() out = model.net(input=image, class_dim=class_dim) cost = paddle.nn.functional.loss.cross_entropy(input=out, label=label) avg_cost = paddle.mean(x=cost) acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1) acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5) train_prog = paddle.static.default_main_program() val_program = paddle.static.default_main_program().clone(for_test=True) place = paddle.CUDAPlace(0) if args.use_gpu else paddle.CPUPlace() ############################################################################################################ # 2. quantization transform programs (training aware) # Make some quantization transforms in the graph before training and testing. # According to the weight and activation quantization type, the graph will be added # some fake quantize operators and fake dequantize operators. ############################################################################################################ val_program = quant_aware(val_program, place, quant_config, scope=None, for_test=True) compiled_train_prog = quant_aware(train_prog, place, quant_config, scope=None, for_test=False) opt = create_optimizer(args) opt.minimize(avg_cost) exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) assert os.path.exists( args.pretrained_model), "pretrained_model doesn't exist" if args.pretrained_model: paddle.static.load(train_prog, args.pretrained_model, exe) places = paddle.static.cuda_places( ) if args.use_gpu else paddle.static.cpu_places() train_loader = paddle.io.DataLoader(train_dataset, places=places, feed_list=[image, label], drop_last=True, batch_size=args.batch_size, return_list=False, use_shared_memory=True, shuffle=True, num_workers=4) valid_loader = paddle.io.DataLoader(val_dataset, places=place, feed_list=[image, label], drop_last=False, return_list=False, batch_size=args.batch_size, use_shared_memory=True, shuffle=False) def test(epoch, program): batch_id = 0 acc_top1_ns = [] acc_top5_ns = [] for data in valid_loader(): start_time = time.time() acc_top1_n, acc_top5_n = exe.run( program, feed=data, fetch_list=[acc_top1.name, acc_top5.name]) end_time = time.time() if batch_id % args.log_period == 0: _logger.info( "Eval epoch[{}] batch[{}] - acc_top1: {}; acc_top5: {}; time: {}" .format(epoch, batch_id, np.mean(acc_top1_n), np.mean(acc_top5_n), end_time - start_time)) acc_top1_ns.append(np.mean(acc_top1_n)) acc_top5_ns.append(np.mean(acc_top5_n)) batch_id += 1 _logger.info( "Final eval epoch[{}] - acc_top1: {}; acc_top5: {}".format( epoch, np.mean(np.array(acc_top1_ns)), np.mean(np.array(acc_top5_ns)))) return np.mean(np.array(acc_top1_ns)) def train(epoch, compiled_train_prog): batch_id = 0 for data in train_loader(): start_time = time.time() loss_n, acc_top1_n, acc_top5_n = exe.run( compiled_train_prog, feed=data, fetch_list=[avg_cost.name, acc_top1.name, acc_top5.name]) end_time = time.time() loss_n = np.mean(loss_n) acc_top1_n = np.mean(acc_top1_n) acc_top5_n = np.mean(acc_top5_n) if batch_id % args.log_period == 0: _logger.info( "epoch[{}]-batch[{}] - loss: {}; acc_top1: {}; acc_top5: {}; time: {}" .format(epoch, batch_id, loss_n, acc_top1_n, acc_top5_n, end_time - start_time)) batch_id += 1 build_strategy = paddle.static.BuildStrategy() build_strategy.memory_optimize = False build_strategy.enable_inplace = False build_strategy.fuse_all_reduce_ops = False build_strategy.sync_batch_norm = False exec_strategy = paddle.static.ExecutionStrategy() compiled_train_prog = compiled_train_prog.with_data_parallel( loss_name=avg_cost.name, build_strategy=build_strategy, exec_strategy=exec_strategy) ############################################################################################################ # train loop ############################################################################################################ best_acc1 = 0.0 best_epoch = 0 for i in range(args.num_epochs): train(i, compiled_train_prog) acc1 = test(i, val_program) paddle.static.save(program=val_program, model_path=os.path.join(args.checkpoint_dir, str(i))) if acc1 > best_acc1: best_acc1 = acc1 best_epoch = i paddle.static.save(program=val_program, model_path=os.path.join(args.checkpoint_dir, 'best_model')) if os.path.exists(os.path.join(args.checkpoint_dir, 'best_model')): paddle.static.load(executor=exe, model_path=os.path.join(args.checkpoint_dir, 'best_model'), program=val_program) ############################################################################################################ # 3. Freeze the graph after training by adjusting the quantize # operators' order for the inference. # The dtype of float_program's weights is float32, but in int8 range. ############################################################################################################ float_program, int8_program = convert(val_program, place, quant_config, \ scope=None, \ save_int8=True) print("eval best_model after convert") final_acc1 = test(best_epoch, float_program) ############################################################################################################ # 4. Save inference model ############################################################################################################ model_path = os.path.join( quantization_model_save_dir, args.model, 'act_' + quant_config['activation_quantize_type'] + '_w_' + quant_config['weight_quantize_type']) float_path = os.path.join(model_path, 'float') if not os.path.isdir(model_path): os.makedirs(model_path) paddle.fluid.io.save_inference_model(dirname=float_path, feeded_var_names=[image.name], target_vars=[out], executor=exe, main_program=float_program, model_filename=float_path + '/model', params_filename=float_path + '/params')
def create_loss(predict, label_input): cost = F.square_error_cost(predict, paddle.cast(x=label_input, dtype='float32')) avg_cost = paddle.mean(cost) return avg_cost
def train(model): # 开启0号GPU训练 use_gpu = True paddle.set_device('gpu:0') if use_gpu else paddle.set_device('cpu') print('start training ... ') model.train() epoch_num = 5 opt = paddle.optimizer.Momentum(learning_rate=0.001, momentum=0.9, parameters=model.parameters()) # 使用Paddle自带的数据读取器 train_loader = paddle.batch(paddle.dataset.mnist.train(), batch_size=10) valid_loader = paddle.batch(paddle.dataset.mnist.test(), batch_size=10) for epoch in range(epoch_num): for batch_id, data in enumerate(train_loader()): # 调整输入数据形状和类型 x_data = np.array([item[0] for item in data], dtype='float32').reshape(-1, 1, 28, 28) y_data = np.array([item[1] for item in data], dtype='int64').reshape(-1, 1) # 将numpy.ndarray转化成Tensor img = paddle.to_tensor(x_data) label = paddle.to_tensor(y_data) # 计算模型输出 logits = model(img) # 计算损失函数 loss = F.softmax_with_cross_entropy(logits, label) avg_loss = paddle.mean(loss) if batch_id % 1000 == 0: print("epoch: {}, batch_id: {}, loss is: {}".format( epoch, batch_id, avg_loss.numpy())) avg_loss.backward() opt.step() opt.clear_grad() model.eval() accuracies = [] losses = [] for batch_id, data in enumerate(valid_loader()): # 调整输入数据形状和类型 x_data = np.array([item[0] for item in data], dtype='float32').reshape(-1, 1, 28, 28) y_data = np.array([item[1] for item in data], dtype='int64').reshape(-1, 1) # 将numpy.ndarray转化成Tensor img = paddle.to_tensor(x_data) label = paddle.to_tensor(y_data) # 计算模型输出 logits = model(img) pred = F.softmax(logits) # 计算损失函数 loss = F.softmax_with_cross_entropy(logits, label) acc = paddle.metric.accuracy(pred, label) accuracies.append(acc.numpy()) losses.append(loss.numpy()) print("[validation] accuracy/loss: {}/{}".format( np.mean(accuracies), np.mean(losses))) model.train() # 保存模型参数 paddle.save(model.state_dict(), 'mnist.pdparams')
def train_iter(self, optimizers=None): self.set_requires_grad(self.nets['discriminator'], False) optimizers['optimG'].clear_grad() l_total = 0 self.output = self.nets['generator'](self.lq) self.visual_items['output'] = self.output # pixel loss if self.pixel_criterion: l_pix = self.pixel_criterion(self.output, self.gt) l_total += l_pix self.losses['loss_pix'] = l_pix if self.perceptual_criterion: l_g_percep, l_g_style = self.perceptual_criterion( self.output, self.gt) # l_total += l_pix if l_g_percep is not None: l_total += l_g_percep self.losses['loss_percep'] = l_g_percep if l_g_style is not None: l_total += l_g_style self.losses['loss_style'] = l_g_style # gan loss (relativistic gan) real_d_pred = self.nets['discriminator'](self.gt).detach() fake_g_pred = self.nets['discriminator'](self.output) l_g_real = self.gan_criterion(real_d_pred - paddle.mean(fake_g_pred), False, is_disc=False) l_g_fake = self.gan_criterion(fake_g_pred - paddle.mean(real_d_pred), True, is_disc=False) l_g_gan = (l_g_real + l_g_fake) / 2 l_total += l_g_gan self.losses['l_g_gan'] = l_g_gan l_total.backward() optimizers['optimG'].step() self.set_requires_grad(self.nets['discriminator'], True) optimizers['optimD'].clear_grad() # real fake_d_pred = self.nets['discriminator'](self.output).detach() real_d_pred = self.nets['discriminator'](self.gt) l_d_real = self.gan_criterion( real_d_pred - paddle.mean(fake_d_pred), True, is_disc=True) * 0.5 # fake fake_d_pred = self.nets['discriminator'](self.output.detach()) l_d_fake = self.gan_criterion( fake_d_pred - paddle.mean(real_d_pred.detach()), False, is_disc=True) * 0.5 (l_d_real + l_d_fake).backward() optimizers['optimD'].step() self.losses['l_d_real'] = l_d_real self.losses['l_d_fake'] = l_d_fake self.losses['out_d_real'] = paddle.mean(real_d_pred.detach()) self.losses['out_d_fake'] = paddle.mean(fake_d_pred.detach())
def train(args): # 使用 GPU训练 if paddle.is_compiled_with_cuda(): paddle.set_device("gpu:0") # 创建多进程的游戏环境 envs = MultipleEnvironments(args.world, args.stage, args.action_type, args.num_processes) # 固定初始化状态 paddle.seed(123) # 创建模型 model = Model(envs.num_states, envs.num_actions) # 创建保存模型的文件夹 if not os.path.isdir(args.saved_path): os.makedirs(args.saved_path) paddle.save(model.state_dict(), "{}/model_{}_{}.pdparams".format(args.saved_path, args.world, args.stage)) # 为游戏评估单独开一个进程 mp = _mp.get_context("spawn") process = mp.Process(target=eval, args=(args, envs.num_states, envs.num_actions)) process.start() # 创建优化方法 clip_grad = paddle.nn.ClipGradByNorm(clip_norm=0.5) optimizer = paddle.optimizer.Adam(parameters=model.parameters(), learning_rate=args.lr, grad_clip=clip_grad) # 刚开始给每个进程的游戏执行初始化 [agent_conn.send(("reset", None)) for agent_conn in envs.agent_conns] # 获取游戏初始的界面 curr_states = [agent_conn.recv() for agent_conn in envs.agent_conns] curr_states = paddle.to_tensor(np.concatenate(curr_states, 0), dtype='float32') curr_episode = 0 while curr_episode < 400: curr_episode += 1 old_log_policies, actions, values, states, rewards, dones = [], [], [], [], [], [] for _ in range(args.num_local_steps): states.append(curr_states) # 执行预测 logits, value = model(curr_states) # 计算每个动作的概率值 policy = F.softmax(logits) # 根据每个标签的概率随机生成符合概率的标签 old_m = Categorical(policy) action = old_m.sample([1]).squeeze() # 记录预测数据 actions.append(action) values.append(value.squeeze()) # 计算类别的概率的对数 old_log_policy = old_m.log_prob(paddle.unsqueeze(action, axis=1)) old_log_policy = paddle.squeeze(old_log_policy) old_log_policies.append(old_log_policy) # 向各个进程游戏发送动作 [agent_conn.send(("step", int(act[0]))) for agent_conn, act in zip(envs.agent_conns, action)] # 将多进程的游戏数据打包 state, reward, done, info = zip(*[agent_conn.recv() for agent_conn in envs.agent_conns]) # 进行数据转换 state = paddle.to_tensor(np.concatenate(state, 0), dtype='float32') # 转换为tensor数据 reward = paddle.to_tensor(reward, dtype='float32') done = paddle.to_tensor(done, dtype='float32') # 记录预测数据 rewards.append(reward) dones.append(done) curr_states = state # 根据上面最后的图像预测 _, next_value, = model(curr_states) next_value = next_value.squeeze() old_log_policies = paddle.concat(old_log_policies).detach().squeeze() actions = paddle.concat(actions).squeeze() values = paddle.concat(values).squeeze().detach() states = paddle.concat(states).squeeze() gae = 0.0 R = [] for value, reward, done in list(zip(values, rewards, dones))[::-1]: gae = gae * args.gamma * args.tau gae = gae + reward + args.gamma * next_value.detach() * (1.0 - done) - value.detach() next_value = value R.append(gae + value) R = R[::-1] R = paddle.concat(R).detach() advantages = R - values for i in range(args.num_epochs): indice = paddle.randperm(args.num_local_steps * args.num_processes) for j in range(args.batch_size): batch_indices = indice[ int(j * (args.num_local_steps * args.num_processes / args.batch_size)): int((j + 1) * ( args.num_local_steps * args.num_processes / args.batch_size))] # 根据拿到的图像执行预测 logits, value = model(paddle.gather(states, batch_indices)) # 计算每个动作的概率值 new_policy = F.softmax(logits) # 计算类别的概率的对数 new_m = Categorical(new_policy) new_log_policy = new_m.log_prob(paddle.unsqueeze(paddle.gather(actions, batch_indices), axis=1)) new_log_policy = paddle.squeeze(new_log_policy) # 计算actor损失 ratio = paddle.exp(new_log_policy - paddle.gather(old_log_policies, batch_indices)) advantage = paddle.gather(advantages, batch_indices) actor_loss = paddle.clip(ratio, 1.0 - args.epsilon, 1.0 + args.epsilon) * advantage actor_loss = paddle.concat([paddle.unsqueeze(ratio * advantage, axis=0), paddle.unsqueeze(actor_loss, axis=0)]) actor_loss = -paddle.mean(paddle.min(actor_loss, axis=0)) # 计算critic损失 critic_loss = F.smooth_l1_loss(paddle.gather(R, batch_indices), value.squeeze()) entropy_loss = paddle.mean(new_m.entropy()) # 计算全部损失 total_loss = actor_loss + critic_loss - args.beta * entropy_loss # 计算梯度 total_loss.backward() optimizer.step() optimizer.clear_grad() paddle.save(model.state_dict(), "{}/model_{}_{}.pdparams".format(args.saved_path, args.world, args.stage)) print("Episode: {}. Total loss: {:.4f}".format(curr_episode, total_loss.numpy()[0])) if os.path.exists("{}/model_{}_{}_finish.pdparams".format(args.saved_path, args.world, args.stage)): process.kill() envs.close() print('顺利通关,world=%d, stage=%d' % (args.world, args.stage)) break process.kill() envs.close()
def forward(self, *args, **kwargs): return paddle.mean(self.loss_fn(*args, **kwargs))
fleet.init(is_collective=True) # 3. create layer & optimizer layer = paddle.nn.Linear(10, 10) adam = paddle.optimizer.Adam(learning_rate=0.001, parameters=layer.parameters()) # 4. get data_parallel model using fleet adam = fleet.distributed_optimizer(adam) dp_layer = fleet.distributed_model(layer) # 5. run layer for step in range(1): inputs = paddle.randn([10, 10], 'float32') outputs = dp_layer(inputs) loss = paddle.mean(outputs) print("step:{}\tloss:{}".format(step, loss.numpy())) loss = dp_layer.scale_loss(loss) loss.backward() dp_layer.apply_collective_grads() adam.step() adam.clear_grad() print("origin lr:", adam.get_lr()) lr_list = [0.2, 0.3, 0.4, 0.5, 0.6] for i in range(5): adam.set_lr(lr_list[i]) lr = adam.get_lr() assert (lr == lr_list[i])
def variation_loss(image, ksize=1): dh = image[:, :, :-ksize, :] - image[:, :, ksize:, :] dw = image[:, :, :, :-ksize] - image[:, :, :, ksize:] return (paddle.mean(paddle.abs(dh)) + paddle.mean(paddle.abs(dw)))
def train(args): config = parse_config(args.config) train_config = merge_configs(config, 'train', vars(args)) valid_config = merge_configs(config, 'valid', vars(args)) print_configs(train_config, 'Train') use_data_parallel = args.use_data_parallel paddle.disable_static(paddle.CUDAPlace(0)) place = paddle.CUDAPlace(paddle.distributed.ParallelEnv().dev_id) \ if use_data_parallel else paddle.CUDAPlace(0) if use_data_parallel: paddle.distributed.init_parallel_env() video_model = TSN_ResNet(train_config) if use_data_parallel: video_model = paddle.DataParallel(video_model) pre_state_dict = paddle.load(args.pretrain) #if paddle.distributed.parallel.Env().local_rank == 0: video_model = init_model(video_model, pre_state_dict) optimizer = create_optimizer(train_config.TRAIN, video_model.parameters()) bs_denominator = 1 if args.use_gpu: # check number of GPUs gpus = os.getenv("CUDA_VISIBLE_DEVICES", "") if gpus == "": pass else: gpus = gpus.split(",") num_gpus = len(gpus) bs_denominator = num_gpus bs_train_single = int(train_config.TRAIN.batch_size / bs_denominator) bs_val_single = int(valid_config.VALID.batch_size / bs_denominator) train_dataset = TSN_UCF101_Dataset(train_config, 'train') val_dataset = TSN_UCF101_Dataset(valid_config, 'valid') train_sampler = DistributedBatchSampler( train_dataset, batch_size=bs_train_single, shuffle=train_config.TRAIN.use_shuffle, drop_last=True) train_loader = DataLoader(train_dataset, batch_sampler=train_sampler, places=place, num_workers=train_config.TRAIN.num_workers, return_list=True) val_sampler = DistributedBatchSampler(val_dataset, batch_size=bs_val_single) val_loader = DataLoader(val_dataset, batch_sampler=val_sampler, places=place, num_workers=valid_config.VALID.num_workers, return_list=True) # resume training the model if args.resume is not None: model_state, opt_state = paddle.load(args.resume) video_model.set_dict(model_state) optimizer.set_dict(opt_state) reader_cost_averager = TimeAverager() batch_cost_averager = TimeAverager() for epoch in range(1, train_config.TRAIN.epoch + 1): epoch_start = time.time() video_model.train() total_loss = 0.0 total_acc1 = 0.0 total_acc5 = 0.0 total_sample = 0 batch_start = time.time() for batch_id, data in enumerate(train_loader): reader_cost_averager.record(time.time() - batch_start) imgs = paddle.to_tensor(data[0], place=paddle.CUDAPinnedPlace()) labels = paddle.to_tensor(data[1], place=paddle.CUDAPinnedPlace()) labels.stop_gradient = True outputs = video_model(imgs) loss = F.cross_entropy(input=outputs, label=labels, ignore_index=-1) avg_loss = paddle.mean(loss) acc_top1 = paddle.metric.accuracy(input=outputs, label=labels, k=1) acc_top5 = paddle.metric.accuracy(input=outputs, label=labels, k=5) dy_out = avg_loss.numpy()[0] if use_data_parallel: # (data_parallel step5/6) avg_loss = video_model.scale_loss(avg_loss) avg_loss.backward() video_model.apply_collective_grads() else: avg_loss.backward() optimizer.minimize(avg_loss) optimizer.step() optimizer.clear_grad() total_loss += dy_out total_acc1 += acc_top1.numpy()[0] total_acc5 += acc_top5.numpy()[0] total_sample += 1 batch_cost_averager.record(time.time() - batch_start, num_samples=bs_train_single) if batch_id % args.log_interval == 0: print( 'TRAIN Epoch: {}, iter: {}, loss={:.6f}, acc1 {:.6f}, acc5 {:.6f}, batch_cost: {:.5f} sec, reader_cost: {:.5f} sec, ips: {:.5f} samples/sec' .format(epoch, batch_id, total_loss / total_sample, total_acc1 / total_sample, total_acc5 / total_sample, batch_cost_averager.get_average(), reader_cost_averager.get_average(), batch_cost_averager.get_ips_average())) batch_cost_averager.reset() reader_cost_averager.reset() batch_start = time.time() train_epoch_cost = time.time() - epoch_start print( 'TRAIN End, Epoch {}, avg_loss= {:.6f}, avg_acc1= {:.6f}, avg_acc5= {:.6f}, epoch_cost: {:.5f} sec' .format(epoch, total_loss / total_sample, total_acc1 / total_sample, total_acc5 / total_sample, train_epoch_cost)) # save model's and optimizer's parameters which used for resuming the training stage save_parameters = (not use_data_parallel) or ( use_data_parallel and paddle.distributed.ParallelEnv().local_rank == 0) if save_parameters: model_path_pre = "_tsn" if not os.path.isdir(args.checkpoint): os.makedirs(args.checkpoint) model_path = os.path.join( args.checkpoint, "_" + model_path_pre + "_epoch{}".format(epoch)) paddle.save(video_model.state_dict(), model_path) paddle.save(optimizer.state_dict(), model_path) if args.validate: video_model.eval() val_acc = val(epoch, video_model, val_loader, valid_config, args) # save the best parameters in trainging stage if epoch == 1: best_acc = val_acc else: if val_acc > best_acc: best_acc = val_acc if paddle.distributed.ParallelEnv().local_rank == 0: if not os.path.isdir(args.weights): os.makedirs(args.weights) paddle.save(video_model.state_dict(), args.weights + "/final") else: if paddle.distributed.parallel.Env().local_rank == 0: if not os.path.isdir(args.weights): os.makedirs(args.weights) paddle.save(video_model.state_dict(), args.weights + "/final") logger.info('[TRAIN] training finished')
def get_bert_encoding(bert_config, model_bert, tokenizer, input_sequence, input_schema, bert_input_version='v1', max_seq_length=512, num_out_layers_n=1, num_out_layers_h=1): if bert_input_version == 'v1': nlu_t, hds = prepare_input(tokenizer, input_sequence, input_schema, max_seq_length) elif bert_input_version == 'v2': nlu_t, hds, max_seq_length = prepare_input_v2(tokenizer, input_sequence, input_schema) wemb_n, wemb_h, l_n, l_hpu, l_hs, nlu_tt, t_to_tt_idx, tt_to_t_idx, t_to_tt_idx_hds = get_wemb_bert( bert_config, model_bert, tokenizer, nlu_t, hds, max_seq_length, num_out_layers_n, num_out_layers_h) t_to_tt_idx = t_to_tt_idx[0] assert len(t_to_tt_idx) == len(input_sequence) assert sum(len(t_to_tt_idx_hds1) for t_to_tt_idx_hds1 in t_to_tt_idx_hds) == len( input_schema.column_names_embedder_input) assert list(wemb_h.shape)[0] == len( input_schema.column_names_embedder_input) utterance_states = [] for i in range(len(t_to_tt_idx)): start = t_to_tt_idx[i] if i == len(t_to_tt_idx) - 1: end = l_n[0] else: end = t_to_tt_idx[i + 1] utterance_states.append( paddle.mean(wemb_n[:, start:end, :], axis=[0, 1])) assert len(utterance_states) == len(input_sequence) schema_token_states = [] cnt = -1 for t_to_tt_idx_hds1 in t_to_tt_idx_hds: for t_to_tt_idx_hds11 in t_to_tt_idx_hds1: cnt += 1 schema_token_states1 = [] for i in range(len(t_to_tt_idx_hds11)): start = t_to_tt_idx_hds11[i] if i == len(t_to_tt_idx_hds11) - 1: end = l_hpu[cnt] else: end = t_to_tt_idx_hds11[i + 1] schema_token_states1.append( paddle.mean(wemb_h[cnt, start:end, :], axis=0)) assert len(schema_token_states1) == len( input_schema.column_names_embedder_input[cnt].split()) schema_token_states.append(schema_token_states1) assert len(schema_token_states) == len( input_schema.column_names_embedder_input) return utterance_states, schema_token_states
def test_accuracy(self): image = paddle.static.data( name='image', shape=[None, 1, 28, 28], dtype='float32') image.stop_gradient = False label = paddle.static.data(name='label', shape=[None, 1], dtype='int64') model = MobileNet() out = model.net(input=image, class_dim=10) cost = paddle.nn.functional.loss.cross_entropy(input=out, label=label) avg_cost = paddle.mean(x=cost) acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1) acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5) optimizer = paddle.optimizer.Momentum( momentum=0.9, learning_rate=0.01, weight_decay=paddle.regularizer.L2Decay(4e-5)) optimizer.minimize(avg_cost) main_prog = paddle.static.default_main_program() val_prog = main_prog.clone(for_test=True) place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda( ) else paddle.CPUPlace() exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) def transform(x): return np.reshape(x, [1, 28, 28]) train_dataset = paddle.vision.datasets.MNIST( mode='train', backend='cv2', transform=transform) test_dataset = paddle.vision.datasets.MNIST( mode='test', backend='cv2', transform=transform) train_loader = paddle.io.DataLoader( train_dataset, places=place, feed_list=[image, label], drop_last=True, return_list=False, batch_size=64) valid_loader = paddle.io.DataLoader( test_dataset, places=place, feed_list=[image, label], batch_size=64, return_list=False) def train(program): iter = 0 for data in train_loader(): cost, top1, top5 = exe.run( program, feed=data, fetch_list=[avg_cost, acc_top1, acc_top5]) iter += 1 if iter % 100 == 0: print( 'train iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) def test(program): iter = 0 result = [[], [], []] for data in valid_loader(): cost, top1, top5 = exe.run( program, feed=data, fetch_list=[avg_cost, acc_top1, acc_top5]) iter += 1 if iter % 100 == 0: print('eval iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) result[0].append(cost) result[1].append(top1) result[2].append(top5) print(' avg loss {}, acc_top1 {}, acc_top5 {}'.format( np.mean(result[0]), np.mean(result[1]), np.mean(result[2]))) return np.mean(result[1]), np.mean(result[2]) train(main_prog) top1_1, top5_1 = test(main_prog) config = { 'weight_quantize_type': 'channel_wise_abs_max', 'activation_quantize_type': 'moving_average_abs_max', 'quantize_op_types': ['depthwise_conv2d', 'mul', 'conv2d'], } quant_train_prog_pact = quant_aware( main_prog, place, config, for_test=False, act_preprocess_func=pact, optimizer_func=get_optimizer, executor=exe) quant_eval_prog = quant_aware(val_prog, place, config, for_test=True) train(quant_train_prog_pact) quant_eval_prog, int8_prog = convert( quant_eval_prog, place, config, save_int8=True) top1_2, top5_2 = test(quant_eval_prog) # values before quantization and after quantization should be close print("before quantization: top1: {}, top5: {}".format(top1_1, top5_1)) print("after quantization: top1: {}, top5: {}".format(top1_2, top5_2))
def forward(self, inputs, label=None): inputs = paddle.reshape(inputs, [-1, inputs.shape[2], inputs.shape[3], inputs.shape[4]]) googLeNet_part1 = self.googLeNet_part1(inputs) googleNet_b3d, before3d = self.before3d(googLeNet_part1) if len(self.out_3d) == self.seg_num: self.out_3d[:self.seg_num - 1] = self.out_3d[1:] self.out_3d[self.seg_num - 1] = before3d self.out_3d[self.seg_num - 2].stop_gradient = True # for input_old in self.out_3d[:self.seg_num - 1]: # input_old.stop_gradient = True else: while len(self.out_3d) < self.seg_num: self.out_3d.append(before3d) y_out_3d = self.out_3d[0] for i in range(len(self.out_3d) - 1): y_out_3d = paddle.concat(x=[y_out_3d,self.out_3d[i+1]], axis=0) y_out_3d = paddle.reshape(y_out_3d, [-1, self.seg_num, y_out_3d.shape[1], y_out_3d.shape[2], y_out_3d.shape[3]]) y_out_3d = paddle.reshape(y_out_3d, [y_out_3d.shape[0], y_out_3d.shape[2], y_out_3d.shape[1], y_out_3d.shape[3], y_out_3d.shape[4]]) out_final_3d = self.res3d(y_out_3d) out_final_3d = paddle.reshape(out_final_3d, [-1, out_final_3d.shape[1]]) out_final_3d = paddle.nn.functional.dropout(out_final_3d, p=0.5) out_final_3d = paddle.reshape(out_final_3d, [-1, self.seg_num, out_final_3d.shape[1]]) out_final_3d = paddle.mean(out_final_3d, axis=1) googLeNet_part2 = self.googLeNet_part2(googleNet_b3d) googLeNet_part3 = self.googLeNet_part3(googLeNet_part2) googLeNet_part3 = paddle.nn.functional.dropout(googLeNet_part3, p=0.6) out_final_2d = paddle.reshape(googLeNet_part3, [-1, googLeNet_part3.shape[1]]) out_final_2d = paddle.reshape(out_final_2d, [-1, self.seg_num, out_final_2d.shape[1]]) out_final_2d = paddle.mean(out_final_2d, axis=1) out_final = paddle.concat(x=[out_final_2d,out_final_3d], axis=1) out_final = self.out(out_final) out_final = paddle.nn.functional.softmax(out_final) if label is not None: acc = paddle.metric.accuracy(input=out_final, label=label) return out_final, acc else: return out_final
def create_loss(self, pred, label): cost = paddle.nn.functional.log_loss(input=pred, label=paddle.cast( label, dtype="float32")) avg_cost = paddle.mean(x=cost) return avg_cost
def forward(self, x): mean = paddle.mean(x, axis=1, keepdim=True) var = paddle.mean(paddle.square(x - mean), axis=1, keepdim=True) tmp = (x - mean) / paddle.sqrt(var + self.eps) return tmp