def create_parameter(self, dtype, in_dim, param_attr=None, use_bias=True, bias_attr=None, transpose_weight=False): if param_attr is None: stddev = math.sqrt(2.0 / (in_dim + self.nclasses)) param_attr = ParamAttr(initializer=Normal(scale=stddev)) weight_shape = [self.shard_dim, in_dim ] if transpose_weight else [in_dim, self.shard_dim] weight = self._layer_helper.create_parameter(shape=weight_shape, dtype=dtype, attr=param_attr, is_bias=False) # avoid allreducing gradients for distributed parameters weight.is_distributed = True # avoid broadcasting distributed parameters in startup program default_startup_program().global_block().vars[ weight.name].is_distributed = True bias = None if use_bias: bias = self._layer_helper.create_parameter(shape=[self.shard_dim], attr=bias_attr, dtype=dtype, is_bias=True) bias.is_distributed = True default_startup_program().global_block().vars[ bias.name].is_distributed = True return weight, bias
def infer(): args = parse_args() num_layers = args.num_layers src_vocab_size = args.vocab_size tar_vocab_size = args.vocab_size batch_size = args.batch_size init_scale = args.init_scale max_grad_norm = args.max_grad_norm hidden_size = args.hidden_size attr_init = args.attr_init latent_size = 32 if args.enable_ce: fluid.default_main_program().random_seed = 102 framework.default_startup_program().random_seed = 102 model = VAE(hidden_size, latent_size, src_vocab_size, tar_vocab_size, batch_size, num_layers=num_layers, init_scale=init_scale, attr_init=attr_init) beam_size = args.beam_size trans_res = model.build_graph(mode='sampling', beam_size=beam_size) # clone from default main program and use it as the validation program main_program = fluid.default_main_program() place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() exe = Executor(place) exe.run(framework.default_startup_program()) dir_name = args.reload_model print("dir name", dir_name) dir_name = os.path.join(dir_name, "checkpoint") fluid.load(main_program, dir_name, exe) vocab, tar_id2vocab = get_vocab(args.dataset_prefix) infer_output = np.ones((batch_size, 1), dtype='int64') * BOS_ID fetch_outs = exe.run(feed={'tar': infer_output}, fetch_list=[trans_res.name], use_program_cache=False) with io.open(args.infer_output_file, 'w', encoding='utf-8') as out_file: for line in fetch_outs[0]: end_id = -1 if EOS_ID in line: end_id = np.where(line == EOS_ID)[0][0] new_line = [tar_id2vocab[e[0]] for e in line[1:end_id]] out_file.write(space_tok.join(new_line)) out_file.write(line_tok)
def test_forward(self): data = layers.data(name='X', shape=[1], dtype='float32') data.stop_gradient = False cond = ConditionalBlock(inputs=[data]) out = layers.create_tensor(dtype='float32') with cond.block(): hidden = layers.fc(input=data, size=10) layers.assign(hidden, out) cpu = core.CPUPlace() exe = Executor(cpu) exe.run(default_startup_program()) x = numpy.random.random(size=(10, 1)).astype('float32') outs = exe.run(feed={'X': x}, fetch_list=[out])[0] print(outs) loss = layers.mean(out) append_backward(loss=loss) outs = exe.run(feed={'X': x}, fetch_list=[ default_main_program().block(0).var(data.name + "@GRAD") ])[0] print(outs)
def train_loop(main_program): exe.run(framework.default_startup_program()) PASS_NUM = 100 for pass_id in range(PASS_NUM): for batch_id, data in enumerate(train_reader()): # train a mini-batch outs = exe.run(program=main_program, feed=func_feed(feeding, data), fetch_list=[avg_cost]) out = np.array(outs[0]) if (batch_id + 1) % 10 == 0: avg_cost_set = [] for test_data in test_reader(): avg_cost_np = exe.run( program=test_program, feed=func_feed(feeding, test_data), fetch_list=[avg_cost]) avg_cost_set.append(avg_cost_np[0]) break # test only 1 segment for speeding up CI # get test avg_cost test_avg_cost = np.array(avg_cost_set).mean() if test_avg_cost < 6.0: # if avg_cost less than 6.0, we think our code is good. if save_dirname is not None: fluid.io.save_inference_model(save_dirname, [ "user_id", "gender_id", "age_id", "job_id", "movie_id", "category_id", "movie_title" ], [scale_infer], exe) return if math.isnan(float(out[0])): sys.exit("got NaN loss, training failed.")
def test_forward(self): data = layers.data(name='X', shape=[1], dtype='float32') data.stop_gradient = False cond = layers.ConditionalBlock(inputs=[data]) out = layers.create_tensor(dtype='float32') with cond.block(): hidden = layers.fc(input=data, size=10) layers.assign(hidden, out) cpu = core.CPUPlace() exe = Executor(cpu) exe.run(default_startup_program()) x = numpy.random.random(size=(10, 1)).astype('float32') outs = exe.run(feed={'X': x}, fetch_list=[out])[0] print outs loss = layers.mean(out) append_backward(loss=loss) outs = exe.run( feed={'X': x}, fetch_list=[ default_main_program().block(0).var(data.name + "@GRAD") ])[0] print outs
def check_switch(self, value): x = layers.fill_constant(shape=[1], dtype='float32', value=value) zero_var = layers.fill_constant(shape=[1], dtype='float32', value=0.0) one_var = layers.fill_constant(shape=[1], dtype='float32', value=1.0) two_var = layers.fill_constant(shape=[1], dtype='float32', value=2.0) three_var = layers.fill_constant(shape=[1], dtype='float32', value=3.0) result = layers.create_global_var(shape=[1], value=-1.0, dtype='float32', persistable=True) with layers.Switch() as switch: with switch.case(layers.less_than(x, zero_var)): layers.assign(zero_var, result) with switch.case(layers.less_than(x, one_var)): layers.assign(one_var, result) with switch.case(layers.less_than(x, two_var)): layers.assign(two_var, result) with switch.default(): layers.assign(three_var, result) cpu = core.CPUPlace() exe = Executor(cpu) exe.run(default_startup_program()) out = exe.run(feed={}, fetch_list=[result])[0][0] return out
def load(): """ load model for predict """ config = model_config() config.vocab_size = len(open(config.vocab_path).readlines()) final_score, final_ids, final_index = knowledge_seq2seq(config) final_score.persistable = True final_ids.persistable = True final_index.persistable = True main_program = fluid.default_main_program() if config.use_gpu: place = fluid.CUDAPlace(0) else: place = fluid.CPUPlace() exe = Executor(place) exe.run(framework.default_startup_program()) fluid.io.load_params(executor=exe, dirname=config.model_path, main_program=main_program) processors = KnowledgeCorpus( data_dir=config.data_dir, data_prefix=config.data_prefix, vocab_path=config.vocab_path, min_len=config.min_len, max_len=config.max_len) # load dict id_dict_array = load_id2str_dict(config.vocab_path) model_handle = [exe, place, final_score, final_ids, final_index, processors, id_dict_array] return model_handle
def test_network_gradient(self): static_input_grad, loss = self.build_graph() self.exe.run(framework.default_startup_program()) actual_gradients, actual_lod = self.fetch_value(static_input_grad) static_input_shape = self.static_input_tensor.get_dims() numeric_gradients = np.zeros( shape=static_input_shape).astype('float32') # calculate numeric gradients tensor_size = np.product(static_input_shape) for i in xrange(tensor_size): origin = self.static_input_tensor.get_float_element(i) x_pos = origin + self._delta self.static_input_tensor.set_float_element(i, x_pos) y_pos = self.fetch_value(loss)[0][0] x_neg = origin - self._delta self.static_input_tensor.set_float_element(i, x_neg) y_neg = self.fetch_value(loss)[0][0] self.static_input_tensor.set_float_element(i, origin) numeric_gradients.ravel()[i] = (y_pos - y_neg) / self._delta / 2 self.assertTrue(np.allclose(actual_gradients, numeric_gradients, 0.001)) self.assertTrue(np.allclose(actual_lod, self.static_input_tensor.lod()))
def train(config): """ model training """ config.vocab_size = len(open(config.vocab_path).readlines()) bow_loss, kl_loss, nll_loss, final_loss= knowledge_seq2seq(config) bow_loss.persistable = True kl_loss.persistable = True nll_loss.persistable = True final_loss.persistable = True main_program = fluid.default_main_program() inference_program = fluid.default_main_program().clone(for_test=True) fluid.clip.set_gradient_clip( clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=config.grad_clip)) optimizer = fluid.optimizer.Adam(learning_rate=config.lr) if config.stage == 0: print("stage 0") optimizer.minimize(bow_loss) else: print("stage 1") optimizer.minimize(final_loss) opt_var_name_list = optimizer.get_opti_var_name_list() if config.use_gpu: place = fluid.CUDAPlace(0) else: place = fluid.CPUPlace() exe = Executor(place) exe.run(framework.default_startup_program()) param_list = main_program.block(0).all_parameters() param_name_list = [p.name for p in param_list] init_model(config, param_name_list, place) processors = KnowledgeCorpus( data_dir=config.data_dir, data_prefix=config.data_prefix, vocab_path=config.vocab_path, min_len=config.min_len, max_len=config.max_len) train_generator = processors.data_generator( batch_size=config.batch_size, phase="train", shuffle=True) valid_generator = processors.data_generator( batch_size=config.batch_size, phase="dev", shuffle=False) model_handle = [exe, place, bow_loss, kl_loss, nll_loss, final_loss] train_loop(config, train_generator, valid_generator, main_program, inference_program, model_handle, param_name_list, opt_var_name_list)
def minimize(self, losses, scopes=None, startup_programs=None, parameter_list=None, no_grad_set=None): if isinstance(losses, list): raise ValueError("need implement later") self._optimizer.minimize(losses, startup_programs, parameter_list, no_grad_set) fleet._origin_main_program = default_main_program().clone( for_test=False) fleet._origin_startup_program = default_startup_program().clone( for_test=False) compiled_config = public.CompileTimeStrategy( fleet._origin_main_program, fleet._origin_startup_program, self._strategy, fleet._role_maker) fleet.compiled_config = compiled_config fleet.main_program, fleet.startup_program = \ self._build_trainer_programs(compiled_config) if fleet.is_worker() \ else self._build_pserver_programs(compiled_config)
def check_switch(self, value): x = layers.fill_constant(shape=[1], dtype='float32', value=value) zero_var = layers.fill_constant(shape=[1], dtype='float32', value=0.0) one_var = layers.fill_constant(shape=[1], dtype='float32', value=1.0) two_var = layers.fill_constant(shape=[1], dtype='float32', value=2.0) three_var = layers.fill_constant(shape=[1], dtype='float32', value=3.0) result = layers.create_global_var( shape=[1], value=-1.0, dtype='float32', persistable=True) with layers.Switch() as switch: with switch.case(layers.less_than(x, zero_var)): layers.assign(zero_var, result) with switch.case(layers.less_than(x, one_var)): layers.assign(one_var, result) with switch.case(layers.less_than(x, two_var)): layers.assign(two_var, result) with switch.default(): layers.assign(three_var, result) cpu = core.CPUPlace() exe = Executor(cpu) exe.run(default_startup_program()) out = exe.run(feed={}, fetch_list=[result])[0][0] return out
def test_step_out(self): static_step_outs = self.build_graph(only_forward=True) self.exe.run(framework.default_startup_program()) expected_outs, expected_lods = self.get_expected_static_step_outs() for i in xrange(self._max_sequence_len): step_out, lod = self.fetch_value(static_step_outs[i]) self.assertTrue(np.allclose(step_out, expected_outs[i])) self.assertTrue(np.allclose(lod, expected_lods[i]))
def test_step_out(self): static_step_outs = self.build_graph(only_forward=True) self.exe.run(framework.default_startup_program()) expected_outs, expected_lods = self.get_expected_static_step_outs() for i in xrange(self._max_sequence_len): step_out, lod = self.fetch_value(static_step_outs[i]) self.assertTrue(np.allclose(step_out, expected_outs[i])) self.assertTrue(np.allclose(lod, expected_lods[i]))
def from_func_spec(func_spec, input_spec, class_instance): """ Builds the main_program with specialized inputs and returns outputs of program as fetch_list. Args: func_spec(FunctionSpec): A FunctionSpec instance for decorated function. input_spec(list[InputSpec]): """ # Transforms dygraph function into static function and caches it. dygraph_function = func_spec.dygraph_function static_func = convert_to_static(dygraph_function) main_program, startup_program = framework.Program(), framework.Program() # Note: The random seed should be synchronized into cached program # if set in `fluid.dygraph_guard` because some ops rely on it, such as # `fluid.layers.dropout`. main_program.random_seed = framework.default_main_program().random_seed startup_program.random_seed = framework.default_startup_program( ).random_seed with framework.program_guard(main_program, startup_program): with _switch_declarative_mode_guard_(is_declarative=True): # 1. Adds `fluid.data` layers for input if needed inputs = func_spec.to_static_inputs_with_spec(input_spec, main_program) if class_instance: inputs = tuple([class_instance] + list(inputs)) # 2. Gets all ParamBases and buffered VarBases in the function all_parameters_and_buffers = list( get_parameters(class_instance).values()) + list( get_buffers(class_instance).values()) # 3. Builds program only once and returns the output Variables. with param_guard(get_parameters( class_instance, False)), param_guard( get_buffers(class_instance, False)): try: outputs = static_func(*inputs) except BaseException as e: # NOTE: If e is raised in compile time, e should be attached to ERROR_DATA here. attach_error_data(e) raise if not isinstance(outputs, (tuple, list)) and outputs is not None: outputs = [outputs] main_program = update_op_callstack_with_origin_info(main_program) return ConcreteProgram( inputs=inputs, outputs=outputs, parameters=all_parameters_and_buffers, function=dygraph_function, main_program=main_program, startup_program=startup_program)
def test_manual_seed(self): local_program = Program() local_main_prog = default_main_program() local_start_prog = default_startup_program() self.assertEqual(0, local_program.random_seed) self.assertEqual(0, local_main_prog.random_seed) self.assertEqual(0, local_start_prog.random_seed) manual_seed(102) global_program1 = Program() global_program2 = Program() global_main_prog = default_main_program() global_start_prog = default_startup_program() self.assertEqual(102, global_program1.random_seed) self.assertEqual(102, global_program2.random_seed) self.assertEqual(102, global_main_prog.random_seed) self.assertEqual(102, global_start_prog.random_seed)
def test_run(self): inputs = fluid.data(name='inputs', shape=[None, self.input_size], dtype='float32') pre_hidden = fluid.data(name='pre_hidden', shape=[None, self.hidden_size], dtype='float32') pre_cell = fluid.data(name='pre_cell', shape=[None, self.hidden_size], dtype='float32') cell = LSTMCell(self.hidden_size) lstm_hidden_new, lstm_states_new = cell(inputs, [pre_hidden, pre_cell]) lstm_unit = contrib.layers.rnn_impl.BasicLSTMUnit( "basicLSTM", self.hidden_size, None, None, None, None, 1.0, "float32") lstm_hidden, lstm_cell = lstm_unit(inputs, pre_hidden, pre_cell) if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) else: place = core.CPUPlace() exe = Executor(place) exe.run(framework.default_startup_program()) inputs_np = np.random.uniform( -0.1, 0.1, (self.batch_size, self.input_size)).astype('float32') pre_hidden_np = np.random.uniform( -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') pre_cell_np = np.random.uniform( -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') param_names = [[ "LSTMCell/BasicLSTMUnit_0.w_0", "basicLSTM/BasicLSTMUnit_0.w_0" ], ["LSTMCell/BasicLSTMUnit_0.b_0", "basicLSTM/BasicLSTMUnit_0.b_0"]] for names in param_names: param = np.array(fluid.global_scope().find_var( names[0]).get_tensor()) param = np.random.uniform(-0.1, 0.1, size=param.shape).astype('float32') fluid.global_scope().find_var(names[0]).get_tensor().set( param, place) fluid.global_scope().find_var(names[1]).get_tensor().set( param, place) out = exe.run(feed={ 'inputs': inputs_np, 'pre_hidden': pre_hidden_np, 'pre_cell': pre_cell_np }, fetch_list=[lstm_hidden_new, lstm_hidden]) self.assertTrue(np.allclose(out[0], out[1], rtol=1e-4, atol=0))
def main(args): task_name = args.task_name.lower() processor = reader.MatchProcessor(data_dir=args.data_dir, task_name=task_name, vocab_path=args.vocab_path, max_seq_len=args.max_seq_len, do_lower_case=args.do_lower_case) num_labels = len(processor.get_labels()) infer_data_generator = processor.data_generator(batch_size=args.batch_size, phase='test', epoch=1, shuffle=False) num_test_examples = processor.get_num_examples(phase='test') main_program = fluid.default_main_program() feed_order, loss, probs, accuracy, num_seqs = create_model( args, num_labels=num_labels, is_prediction=True) if args.use_cuda: place = fluid.CUDAPlace(0) dev_count = fluid.core.get_cuda_device_count() else: place = fluid.CPUPlace() dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) exe = fluid.Executor(place) exe.run(framework.default_startup_program()) if args.init_checkpoint: init_pretraining_params(exe, args.init_checkpoint, main_program) feed_list = [ main_program.global_block().var(var_name) for var_name in feed_order ] feeder = fluid.DataFeeder(feed_list, place) out_scores = open(args.output, 'w') for batch_id, data in enumerate(infer_data_generator()): results = exe.run(fetch_list=[probs], feed=feeder.feed(data), return_numpy=True) for elem in results[0]: out_scores.write(str(elem[1]) + '\n') out_scores.close() if args.save_inference_model_path: model_path = args.save_inference_model_path fluid.io.save_inference_model(model_path, feed_order, probs, exe, main_program=main_program)
def main(): rnn_out = encoder_decoder() label = layers.data(name="target_language_next_word", shape=[1], dtype='int64', lod_level=1) cost = layers.cross_entropy(input=rnn_out, label=label) avg_cost = fluid.layers.mean(cost) optimizer = fluid.optimizer.Adagrad(learning_rate=1e-4) optimizer.minimize(avg_cost) # fluid.memory_optimize(fluid.default_main_program()) fluid.release_memory(fluid.default_main_program()) # fix the order of training data train_data = paddle.batch(paddle.dataset.wmt14.train(dict_size), batch_size=batch_size) # train_data = paddle.batch( # paddle.reader.shuffle( # paddle.dataset.wmt14.train(dict_size), buf_size=1000), # batch_size=batch_size) place = core.CPUPlace() exe = Executor(place) exe.run(framework.default_startup_program()) feed_order = [ 'src_word_id', 'target_language_word', 'target_language_next_word' ] feed_list = [ fluid.default_main_program().global_block().var(var_name) for var_name in feed_order ] feeder = fluid.DataFeeder(feed_list, place) batch_id = 0 for pass_id in xrange(10): for data in train_data(): outs = exe.run(fluid.default_main_program(), feed=feeder.feed(data), fetch_list=[avg_cost]) avg_cost_val = np.array(outs[0]) print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) + " avg_cost=" + str(avg_cost_val)) if batch_id > 2: exit(0) if math.isnan(float(avg_cost_val)): sys.exit("got NaN loss, training failed.") batch_id += 1
def decode_main(use_cuda, is_sparse): if use_cuda and not fluid.core.is_compiled_with_cuda(): return place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() context = encoder(is_sparse) translation_ids, translation_scores = decoder_decode(context, is_sparse) exe = Executor(place) exe.run(framework.default_startup_program()) init_ids_data = np.array([1 for _ in range(batch_size)], dtype='int64') init_scores_data = np.array([1. for _ in range(batch_size)], dtype='float32') init_ids_data = init_ids_data.reshape((batch_size, 1)) init_scores_data = init_scores_data.reshape((batch_size, 1)) init_recursive_seq_lens = [1] * batch_size init_recursive_seq_lens = [ init_recursive_seq_lens, init_recursive_seq_lens ] init_ids = fluid.create_lod_tensor(init_ids_data, init_recursive_seq_lens, place) init_scores = fluid.create_lod_tensor(init_scores_data, init_recursive_seq_lens, place) train_data = paddle.batch(paddle.reader.shuffle( paddle.dataset.wmt14.train(dict_size), buf_size=1000), batch_size=batch_size) feed_order = ['src_word_id'] feed_list = [ framework.default_main_program().global_block().var(var_name) for var_name in feed_order ] feeder = fluid.DataFeeder(feed_list, place) for data in train_data(): feed_dict = feeder.feed([[x[0]] for x in data]) feed_dict['init_ids'] = init_ids feed_dict['init_scores'] = init_scores result_ids, result_scores = exe.run( framework.default_main_program(), feed=feed_dict, fetch_list=[translation_ids, translation_scores], return_numpy=False) print(result_ids.recursive_sequence_lengths()) break
def main(args): task_name = args.task_name.lower() processors = { 'match': reader.MatchProcessor, } processor = processors[task_name](data_dir=args.data_dir, vocab_path=args.vocab_path, max_seq_len=args.max_seq_len, do_lower_case=args.do_lower_case) num_labels = len(processor.get_labels()) infer_data_generator = processor.data_generator( batch_size=args.batch_size, phase='dev', epoch=args.epoch, shuffle=False) main_program = fluid.default_main_program() feed_order, loss, probs, accuracy, num_seqs = create_model( args, num_labels=num_labels) if args.use_cuda: place = fluid.CUDAPlace(0) else: place = fluid.CPUPlace() exe = fluid.Executor(place) exe.run(framework.default_startup_program()) if args.init_checkpoint: init_pretraining_params(exe, args.init_checkpoint, main_program) feed_list = [ main_program.global_block().var(var_name) for var_name in feed_order ] feeder = fluid.DataFeeder(feed_list, place) label_list = [] for batch_id, data in enumerate(infer_data_generator()): results = exe.run( fetch_list=[probs], feed=feeder.feed(data), return_numpy=True) for elem in results[0]: label_list.append(str(elem[1])) return label_list
def load_model(): """ load model function """ main_program = fluid.default_main_program() #place = fluid.CPUPlace() place = fluid.CUDAPlace(0) exe = fluid.Executor(place) exe.run(framework.default_startup_program()) path = "./models/inference_model" [inference_program, feed_dict, fetch_targets] = \ fluid.io.load_inference_model(dirname=path, executor=exe) model_handle = [exe, inference_program, feed_dict, fetch_targets, place] return model_handle
def train(): startup_program = fluid.default_startup_program() main_program = fluid.default_main_program() raw_data = reader.raw_data('fra.txt', num_samples=num_samples) train_data = raw_data[0] data_vars = raw_data[1] model = BaseModel(hidden_size=latent_dim, src_vocab_size=data_vars['num_encoder_tokens'], tar_vocab_size=data_vars['num_decoder_tokens'], batch_size=batch_size, batch_first=True) loss = model.build_graph() optimizer = fluid.optimizer.Adam(learning_rate=0.001) optimizer.minimize(loss) place = fluid.CUDAPlace(0) exe = fluid.Executor(place) exe.run(framework.default_startup_program()) ce_ppl = [] for epoch_id in range(num_epochs): print("epoch ", epoch_id) train_data_iter = reader.get_data_iter(train_data, batch_size) total_loss = 0 word_count = 0.0 for batch_id, batch in enumerate(train_data_iter): input_data_feed, word_num = prepare_input(batch, epoch_id=epoch_id) fetch_outs = exe.run(feed=input_data_feed, fetch_list=[loss.name], use_program_cache=True) cost_train = np.array(fetch_outs[0]) total_loss += cost_train * batch_size word_count += word_num if batch_id > 0 and batch_id % batch_size == 0: print(" ppl", batch_id, np.exp(total_loss / word_count)) ce_ppl.append(np.exp(total_loss / word_count)) total_loss = 0.0 word_count = 0.0
def main(): rnn_out = encoder_decoder() label = layers.data( name="target_language_next_word", shape=[1], dtype='int64', lod_level=1) cost = layers.cross_entropy(input=rnn_out, label=label) avg_cost = fluid.layers.mean(cost) optimizer = fluid.optimizer.Adagrad(learning_rate=1e-4) optimizer.minimize(avg_cost) # fluid.memory_optimize(fluid.default_main_program()) fluid.release_memory(fluid.default_main_program()) # fix the order of training data train_data = paddle.batch( paddle.dataset.wmt14.train(dict_size), batch_size=batch_size) # train_data = paddle.batch( # paddle.reader.shuffle( # paddle.dataset.wmt14.train(dict_size), buf_size=1000), # batch_size=batch_size) place = core.CPUPlace() exe = Executor(place) exe.run(framework.default_startup_program()) batch_id = 0 for pass_id in xrange(10): for data in train_data(): word_data = to_lodtensor(map(lambda x: x[0], data), place) trg_word = to_lodtensor(map(lambda x: x[1], data), place) trg_word_next = to_lodtensor(map(lambda x: x[2], data), place) outs = exe.run(fluid.default_main_program(), feed={ 'src_word_id': word_data, 'target_language_word': trg_word, 'target_language_next_word': trg_word_next }, fetch_list=[avg_cost]) avg_cost_val = np.array(outs[0]) print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) + " avg_cost=" + str(avg_cost_val)) if batch_id > 2: exit(0) if math.isnan(float(avg_cost_val)): sys.exit("got NaN loss, training failed.") batch_id += 1
def __init__(self): self._inputs = [] self._outputs = [] # Always set program to default_main_program. Because once `__call__` is called, # it means layers(or Ops) are added into default_main_program switched by outer # `with` statement. self._main_program = framework.default_main_program() self._startup_program = framework.default_startup_program() self._func_cache = FunctionCache() # Stores the entry function of Net or Model. self._forward_func = None self._feed_name_to_idx = {} self._is_repeated = False # Indicates whether the function call is still building program. # Because user can call recursively when `Net` has sub class in # `forward()`. self._in_build_process = True
def decode_main(use_cuda, is_sparse): if use_cuda and not fluid.core.is_compiled_with_cuda(): return place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() context = encoder(is_sparse) translation_ids, translation_scores = decode(context, is_sparse) exe = Executor(place) exe.run(framework.default_startup_program()) init_ids_data = np.array([1 for _ in range(batch_size)], dtype='int64') init_scores_data = np.array( [1. for _ in range(batch_size)], dtype='float32') init_ids_data = init_ids_data.reshape((batch_size, 1)) init_scores_data = init_scores_data.reshape((batch_size, 1)) init_lod = [1] * batch_size init_lod = [init_lod, init_lod] init_ids = fluid.create_lod_tensor(init_ids_data, init_lod, place) init_scores = fluid.create_lod_tensor(init_scores_data, init_lod, place) train_data = paddle.batch( paddle.reader.shuffle( paddle.dataset.wmt14.train(dict_size), buf_size=1000), batch_size=batch_size) feed_order = ['src_word_id'] feed_list = [ framework.default_main_program().global_block().var(var_name) for var_name in feed_order ] feeder = fluid.DataFeeder(feed_list, place) for data in train_data(): feed_dict = feeder.feed(map(lambda x: [x[0]], data)) feed_dict['init_ids'] = init_ids feed_dict['init_scores'] = init_scores result_ids, result_scores = exe.run( framework.default_main_program(), feed=feed_dict, fetch_list=[translation_ids, translation_scores], return_numpy=False) print result_ids.lod() break
def train_loop(main_program): exe.run(framework.default_startup_program()) feed_list = [ main_program.global_block().var(var_name) for var_name in feed_order ] feeder = fluid.DataFeeder(feed_list, place) for pass_id in range(1): for batch_id, data in enumerate(train_reader()): outs = exe.run(main_program, feed=feeder.feed(data), fetch_list=[avg_cost]) avg_cost_val = np.array(outs[0]) print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) + " avg_cost=" + str(avg_cost_val)) if batch_id > 3: break
def _transpile(self, config): if not isinstance(config, DistributeTranspilerConfig): raise TypeError( "config must be an instance of DistributeTranspilerConfig") if not config.sync_mode: config.runtime_split_send_recv = True # _origin_program is a deep copy for default_main_program, for inference self._origin_program = default_main_program().clone(for_test=False) self._transpile_config = config if config.geo_sgd_mode: self._transpiler = GeoSgdTranspiler(config) else: self._transpiler = OriginTranspiler(config) if self.is_worker(): self._transpiler.transpile( trainer_id=fleet.worker_index(), pservers=fleet.server_endpoints(to_string=True), trainers=fleet.worker_num(), sync_mode=config.sync_mode) if isinstance(self._role_maker, MPISymetricRoleMaker): config.wait_port = False self.main_program = self._transpiler.get_trainer_program( wait_port=config.wait_port) self.startup_program = default_startup_program() if self._transpile_config.geo_sgd_mode: self.vars_info = self._transpiler._get_vars_info() self.startup_program = self._transpiler.trainer_startup_program else: self._transpiler.transpile( trainer_id=fleet.worker_index(), pservers=fleet.server_endpoints(to_string=True), trainers=fleet.worker_num(), sync_mode=config.sync_mode, current_endpoint=self.server_endpoints()[self.server_index()]) self.main_program, self.startup_program = \ self._transpiler.get_pserver_programs( self.server_endpoints()[self.server_index()])
def train(use_cuda, save_dirname=None): [avg_cost, prediction] = seq_to_seq_net() optimizer = fluid.optimizer.Adagrad(learning_rate=1e-4) optimizer.minimize(avg_cost) train_data = paddle.batch( paddle.reader.shuffle( paddle.dataset.wmt14.train(dict_size), buf_size=1000), batch_size=batch_size) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = Executor(place) exe.run(framework.default_startup_program()) feed_order = ['source_sequence', 'target_sequence', 'label_sequence'] feed_list = [ framework.default_main_program().global_block().var(var_name) for var_name in feed_order ] feeder = fluid.DataFeeder(feed_list, place) batch_id = 0 for pass_id in xrange(2): for data in train_data(): outs = exe.run(framework.default_main_program(), feed=feeder.feed(data), fetch_list=[avg_cost]) avg_cost_val = np.array(outs[0]) print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) + " avg_cost=" + str(avg_cost_val)) if math.isnan(float(avg_cost_val[0])): sys.exit("got NaN loss, training failed.") if batch_id > 3: if save_dirname is not None: fluid.io.save_inference_model( save_dirname, ['source_sequence', 'target_sequence'], [prediction], exe) return batch_id += 1
def train(use_cuda, save_dirname=None): [avg_cost, prediction] = seq_to_seq_net() optimizer = fluid.optimizer.Adagrad(learning_rate=1e-4) optimizer.minimize(avg_cost) train_data = paddle.batch( paddle.reader.shuffle( paddle.dataset.wmt14.train(dict_size), buf_size=1000), batch_size=batch_size) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = Executor(place) exe.run(framework.default_startup_program()) feed_order = ['source_sequence', 'target_sequence', 'label_sequence'] feed_list = [ framework.default_main_program().global_block().var(var_name) for var_name in feed_order ] feeder = fluid.DataFeeder(feed_list, place) batch_id = 0 for pass_id in xrange(2): for data in train_data(): outs = exe.run(framework.default_main_program(), feed=feeder.feed(data), fetch_list=[avg_cost]) avg_cost_val = np.array(outs[0]) print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) + " avg_cost=" + str(avg_cost_val)) if math.isnan(float(avg_cost_val[0])): sys.exit("got NaN loss, training failed.") if batch_id > 3: if save_dirname is not None: fluid.io.save_inference_model( save_dirname, ['source_sequence', 'target_sequence'], [prediction], exe) return batch_id += 1
def test_network_gradient(self): static_input_grad, loss = self.build_graph() self.exe.run(framework.default_startup_program()) actual_gradients, actual_lod = self.fetch_value(static_input_grad) static_input_shape = self.static_input_tensor.get_dims() numeric_gradients = np.zeros(shape=static_input_shape).astype('float32') # calculate numeric gradients tensor_size = np.product(static_input_shape) for i in xrange(tensor_size): origin = self.static_input_tensor.get_float_element(i) x_pos = origin + self._delta self.static_input_tensor.set_float_element(i, x_pos) y_pos = self.fetch_value(loss)[0][0] x_neg = origin - self._delta self.static_input_tensor.set_float_element(i, x_neg) y_neg = self.fetch_value(loss)[0][0] self.static_input_tensor.set_float_element(i, origin) numeric_gradients.ravel()[i] = (y_pos - y_neg) / self._delta / 2 self.assertTrue(np.allclose(actual_gradients, numeric_gradients, 0.001)) self.assertTrue(np.allclose(actual_lod, self.static_input_tensor.lod()))
def _transpile(self, config): if not isinstance(config, DistributeTranspilerConfig): raise ValueError( "config must be an instance of DistributeTranspilerConfig") if not config.sync_mode: config.runtime_split_send_recv = True self._transpile_config = config self._transpiler = OriginTranspiler(config) self._transpiler.transpile( trainer_id=fleet.worker_index(), pservers=fleet.server_endpoints(to_string=True), trainers=fleet.worker_num(), sync_mode=config.sync_mode) if self.is_worker(): self.main_program = self._transpiler.get_trainer_program() self.startup_program = default_startup_program() else: self.main_program, self.startup_program = \ self._transpiler.get_pserver_programs(self.server_endpoints()[self.server_index()])
def decode_main(use_cuda, is_sparse): if use_cuda and not fluid.core.is_compiled_with_cuda(): return place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() context = encoder(is_sparse) translation_ids, translation_scores = decoder_decode(context, is_sparse) exe = Executor(place) exe.run(framework.default_startup_program()) init_ids_data = np.array([1 for _ in range(batch_size)], dtype='int64') init_scores_data = np.array([1. for _ in range(batch_size)], dtype='float32') init_ids_data = init_ids_data.reshape((batch_size, 1)) init_scores_data = init_scores_data.reshape((batch_size, 1)) init_lod = [i for i in range(batch_size)] + [batch_size] init_lod = [init_lod, init_lod] train_data = paddle.batch(paddle.reader.shuffle( paddle.dataset.wmt14.train(dict_size), buf_size=1000), batch_size=batch_size) for _, data in enumerate(train_data()): init_ids = set_init_lod(init_ids_data, init_lod, place) init_scores = set_init_lod(init_scores_data, init_lod, place) src_word_data = to_lodtensor(map(lambda x: x[0], data), place) result_ids, result_scores = exe.run( framework.default_main_program(), feed={ 'src_word_id': src_word_data, 'init_ids': init_ids, 'init_scores': init_scores }, fetch_list=[translation_ids, translation_scores], return_numpy=False) print result_ids.lod() break
def train_loop(main_program): exe.run(framework.default_startup_program()) batch_id = 0 for pass_id in xrange(1): for data in train_data(): word_data = to_lodtensor(map(lambda x: x[0], data), place) trg_word = to_lodtensor(map(lambda x: x[1], data), place) trg_word_next = to_lodtensor(map(lambda x: x[2], data), place) outs = exe.run(main_program, feed={ 'src_word_id': word_data, 'target_language_word': trg_word, 'target_language_next_word': trg_word_next }, fetch_list=[avg_cost]) avg_cost_val = np.array(outs[0]) print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) + " avg_cost=" + str(avg_cost_val)) if batch_id > 3: break batch_id += 1
def train_loop(main_program): exe.run(framework.default_startup_program()) feed_list = [ main_program.global_block().var(var_name) for var_name in feed_order ] feeder = fluid.DataFeeder(feed_list, place) PASS_NUM = 100 for pass_id in range(PASS_NUM): for batch_id, data in enumerate(train_reader()): # train a mini-batch outs = exe.run(program=main_program, feed=feeder.feed(data), fetch_list=[avg_cost]) out = np.array(outs[0]) if (batch_id + 1) % 10 == 0: avg_cost_set = [] for test_data in test_reader(): avg_cost_np = exe.run(program=test_program, feed=feeder.feed(test_data), fetch_list=[avg_cost]) avg_cost_set.append(avg_cost_np[0]) break # test only 1 segment for speeding up CI # get test avg_cost test_avg_cost = np.array(avg_cost_set).mean() if test_avg_cost < 6.0: # if avg_cost less than 6.0, we think our code is good. if save_dirname is not None: fluid.io.save_inference_model(save_dirname, [ "user_id", "gender_id", "age_id", "job_id", "movie_id", "category_id", "movie_title" ], [scale_infer], exe) return if math.isnan(float(out[0])): sys.exit("got NaN loss, training failed.")
def from_func_spec(func_spec): """ Builds the main_program with specialized inputs and returns outputs of program as fetch_list. """ # Transforms dygraph function into static function and caches it. dygaph_function = func_spec.dyfunc static_func = convert_function_with_cache(dygaph_function) main_program, startup_program = framework.Program(), framework.Program( ) # Note: The random seed should be synchronized into cached program # if set in `fluid.dygrap_guard` because some ops rely on it, such as # `fluid.layers.dropout`. main_program.random_seed = framework.default_main_program().random_seed startup_program.random_seed = framework.default_startup_program( ).random_seed with framework.program_guard(main_program, startup_program): # 1. Adds `fluid.data` layers for input if needed inputs = func_spec.to_static_inputs(main_program) # 2. Gets all ParamBases in the function all_parameters = list(func_spec.parameters().values()) # 3. Builds program only once and returns the output Variables. with param_guard(func_spec.parameters(False)): outputs = static_func(*inputs) if not isinstance(outputs, (tuple, list)): outputs = [outputs] if outputs else [] return ConcreteProgram(inputs=inputs, outputs=outputs, parameters=all_parameters, func=dygaph_function, main_program=main_program, startup_program=startup_program)
def test_ptb_rnn_cpu_bfloat16(self): seed = 90 hidden_size = 10 vocab_size = 500 num_layers = 1 num_steps = 3 init_scale = 0.1 batch_size = 4 batch_num = 100 with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed ptb_model = PtbModel("ptb_model", hidden_size=hidden_size, vocab_size=vocab_size, num_layers=num_layers, num_steps=num_steps, init_scale=init_scale) place = self.set_place() exe = fluid.Executor(place) sgd = SGDOptimizer(learning_rate=1e-3) x = fluid.layers.data(name="x", shape=[-1, num_steps], dtype='int64') y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') init_hidden = fluid.layers.data(name="init_hidden", shape=[1], dtype='float32') init_cell = fluid.layers.data(name="init_cell", shape=[1], dtype='float32') static_loss, static_last_hidden, static_last_cell = ptb_model( x, y, init_hidden, init_cell) sgd = paddle.static.amp.bf16.decorate_bf16( sgd, amp_lists=paddle.static.amp.bf16.AutoMixedPrecisionListsBF16( custom_fp32_list={'transpose2', 'concat'}), use_bf16_guard=False, use_pure_bf16=True) sgd.minimize(static_loss, framework.default_startup_program()) out = exe.run(framework.default_startup_program()) for i in range(batch_num): x_data = np.arange(12).reshape(4, 3).astype('int64') y_data = np.arange(1, 13).reshape(4, 3).astype('int64') x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) #TODO investigate initializing model with "float32" instead of "uint16" as it was before # slice_op PR(datatypes in model graph are different than datatypes during runtime because of that) init_hidden_data = np.zeros( (num_layers, batch_size, hidden_size), dtype='uint16') init_cell_data = np.zeros( (num_layers, batch_size, hidden_size), dtype='uint16') fetch_list = [ static_loss, static_last_hidden, static_last_cell ] out = exe.run(fluid.default_main_program(), feed={ "x": x_data, "y": y_data, "init_hidden": init_hidden_data, "init_cell": init_cell_data }, fetch_list=fetch_list) # get value before save main_program = framework.default_main_program() base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: t = np.array(fluid.global_scope().find_var( var.name).get_tensor()) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t fluid.save(main_program, "./test_1") # set var to zero for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) new_t = np.array(fluid.global_scope().find_var( var.name).get_tensor()) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) fluid.load(main_program, "./test_1.pdparams", exe) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: new_t = np.array(fluid.global_scope().find_var( var.name).get_tensor()) base_t = base_map[var.name] self.assertTrue(np.array_equal(new_t, base_t))
def main(): args = parse_args() print(args) num_layers = args.num_layers src_vocab_size = args.vocab_size tar_vocab_size = args.vocab_size batch_size = args.batch_size init_scale = args.init_scale max_grad_norm = args.max_grad_norm hidden_size = args.hidden_size attr_init = args.attr_init latent_size = 32 main_program = fluid.Program() startup_program = fluid.Program() if args.enable_ce: fluid.default_main_program().random_seed = 123 framework.default_startup_program().random_seed = 123 # Training process with fluid.program_guard(main_program, startup_program): with fluid.unique_name.guard(): model = VAE(hidden_size, latent_size, src_vocab_size, tar_vocab_size, batch_size, num_layers=num_layers, init_scale=init_scale, attr_init=attr_init) loss, kl_loss, rec_loss = model.build_graph() # clone from default main program and use it as the validation program main_program = fluid.default_main_program() inference_program = fluid.default_main_program().clone( for_test=True) clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=max_grad_norm) learning_rate = fluid.layers.create_global_var( name="learning_rate", shape=[1], value=float(args.learning_rate), dtype="float32", persistable=True) opt_type = args.optimizer if opt_type == "sgd": optimizer = fluid.optimizer.SGD(learning_rate, grad_clip=clip) elif opt_type == "adam": optimizer = fluid.optimizer.Adam(learning_rate, grad_clip=clip) else: print("only support [sgd|adam]") raise Exception("opt type not support") optimizer.minimize(loss) place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() exe = Executor(place) exe.run(startup_program) train_program = fluid.compiler.CompiledProgram(main_program) dataset_prefix = args.dataset_prefix print("begin to load data") raw_data = reader.raw_data(dataset_prefix, args.max_len) print("finished load data") train_data, valid_data, test_data, _ = raw_data anneal_r = 1.0 / (args.warm_up * len(train_data) / args.batch_size) def prepare_input(batch, kl_weight=1.0, lr=None): src_ids, src_mask = batch res = {} src_ids = src_ids.reshape((src_ids.shape[0], src_ids.shape[1])) in_tar = src_ids[:, :-1] label_tar = src_ids[:, 1:] in_tar = in_tar.reshape((in_tar.shape[0], in_tar.shape[1])) label_tar = label_tar.reshape( (label_tar.shape[0], label_tar.shape[1], 1)) res['src'] = src_ids res['tar'] = in_tar res['label'] = label_tar res['src_sequence_length'] = src_mask res['tar_sequence_length'] = src_mask - 1 res['kl_weight'] = np.array([kl_weight]).astype(np.float32) if lr is not None: res['learning_rate'] = np.array([lr]).astype(np.float32) return res, np.sum(src_mask), np.sum(src_mask - 1) # get train epoch size def eval(data): eval_data_iter = reader.get_data_iter(data, batch_size, mode='eval') total_loss = 0.0 word_count = 0.0 batch_count = 0.0 for batch_id, batch in enumerate(eval_data_iter): input_data_feed, src_word_num, dec_word_sum = prepare_input(batch) fetch_outs = exe.run(inference_program, feed=input_data_feed, fetch_list=[loss.name], use_program_cache=False) cost_train = np.array(fetch_outs[0]) total_loss += cost_train * batch_size word_count += dec_word_sum batch_count += batch_size nll = total_loss / batch_count ppl = np.exp(total_loss / word_count) return nll, ppl def train(): ce_time = [] ce_ppl = [] max_epoch = args.max_epoch kl_w = args.kl_start lr_w = args.learning_rate best_valid_nll = 1e100 # +inf best_epoch_id = -1 decay_cnt = 0 max_decay = args.max_decay decay_factor = 0.5 decay_ts = 2 steps_not_improved = 0 for epoch_id in range(max_epoch): start_time = time.time() if args.enable_ce: train_data_iter = reader.get_data_iter(train_data, batch_size, args.sort_cache, args.cache_num, enable_ce=True) else: train_data_iter = reader.get_data_iter(train_data, batch_size, args.sort_cache, args.cache_num) total_loss = 0 total_rec_loss = 0 total_kl_loss = 0 word_count = 0.0 batch_count = 0.0 batch_times = [] for batch_id, batch in enumerate(train_data_iter): batch_start_time = time.time() kl_w = min(1.0, kl_w + anneal_r) kl_weight = kl_w input_data_feed, src_word_num, dec_word_sum = prepare_input( batch, kl_weight, lr_w) fetch_outs = exe.run( program=train_program, feed=input_data_feed, fetch_list=[loss.name, kl_loss.name, rec_loss.name], use_program_cache=False) cost_train = np.array(fetch_outs[0]) kl_cost_train = np.array(fetch_outs[1]) rec_cost_train = np.array(fetch_outs[2]) total_loss += cost_train * batch_size total_rec_loss += rec_cost_train * batch_size total_kl_loss += kl_cost_train * batch_size word_count += dec_word_sum batch_count += batch_size batch_end_time = time.time() batch_time = batch_end_time - batch_start_time batch_times.append(batch_time) if batch_id > 0 and batch_id % 200 == 0: print("-- Epoch:[%d]; Batch:[%d]; Time: %.4f s; " "kl_weight: %.4f; kl_loss: %.4f; rec_loss: %.4f; " "nll: %.4f; ppl: %.4f" % (epoch_id, batch_id, batch_time, kl_w, total_kl_loss / batch_count, total_rec_loss / batch_count, total_loss / batch_count, np.exp(total_loss / word_count))) ce_ppl.append(np.exp(total_loss / word_count)) end_time = time.time() epoch_time = end_time - start_time ce_time.append(epoch_time) print( "\nTrain epoch:[%d]; Epoch Time: %.4f; avg_time: %.4f s/step\n" % (epoch_id, epoch_time, sum(batch_times) / len(batch_times))) val_nll, val_ppl = eval(valid_data) print("dev ppl", val_ppl) test_nll, test_ppl = eval(test_data) print("test ppl", test_ppl) if val_nll < best_valid_nll: best_valid_nll = val_nll steps_not_improved = 0 best_nll = test_nll best_ppl = test_ppl best_epoch_id = epoch_id save_path = os.path.join(args.model_path, "epoch_" + str(best_epoch_id), "checkpoint") print("save model {}".format(save_path)) fluid.save(main_program, save_path) else: steps_not_improved += 1 if steps_not_improved == decay_ts: old_lr = lr_w lr_w *= decay_factor steps_not_improved = 0 new_lr = lr_w print('-----\nchange lr, old lr: %f, new lr: %f\n-----' % (old_lr, new_lr)) dir_name = args.model_path + "/epoch_" + str(best_epoch_id) fluid.load(main_program, dir_name, exe) decay_cnt += 1 if decay_cnt == max_decay: break print('\nbest testing nll: %.4f, best testing ppl %.4f\n' % (best_nll, best_ppl)) if args.enable_ce: card_num = get_cards() _ppl = 0 _time = 0 try: _time = ce_time[-1] _ppl = ce_ppl[-1] except: print("ce info error") print("kpis\ttrain_duration_card%s\t%s" % (card_num, _time)) print("kpis\ttrain_ppl_card%s\t%f" % (card_num, _ppl)) with profile_context(args.profile): train()
def testLoadStaticModel(self): # static mode a = fluid.data(name="a", shape=[10, 10]) conv_in = fluid.data(name="conv_in", shape=[None, 10, 10, 10]) fc_out1 = fluid.layers.fc(a, 10) fc_out2 = fluid.layers.fc(a, 20) conv_out_1 = fluid.layers.conv2d(conv_in, num_filters=10, filter_size=5, act="relu") conv_out_2 = fluid.layers.conv2d(conv_in, num_filters=10, filter_size=5, act="relu") conv3d_in = fluid.data(name='conv3d_in', shape=[None, 3, 12, 32, 32], dtype='float32') conv3d_out_1 = fluid.layers.conv3d(input=conv3d_in, num_filters=2, filter_size=3, act="relu") conv3d_out_2 = fluid.layers.conv3d(input=conv3d_in, num_filters=2, filter_size=3, act="relu") batchnorm_in = fluid.data(name="batchnorm_in", shape=[None, 10], dtype='float32') batchnorm_out_1 = fluid.layers.batch_norm(batchnorm_in) batchnorm_out_2 = fluid.layers.batch_norm(batchnorm_in) emb_in = fluid.data(name='emb_in', shape=[None, 10], dtype='int64') emb_out_1 = fluid.embedding(emb_in, [1000, 100]) emb_out_2 = fluid.embedding(emb_in, [2000, 200]) layernorm = fluid.data(name="ln", shape=[None, 10], dtype='float32') layernorm_1 = fluid.layers.layer_norm(layernorm) layernorm_2 = fluid.layers.layer_norm(layernorm) nce_in = fluid.data(name="nce_in", shape=[None, 100], dtype='float32') nce_label = fluid.data(name="nce_label", shape=[None, 10], dtype='int64') nce_out_1 = fluid.layers.nce(nce_in, nce_label, 10000) nce_out_2 = fluid.layers.nce(nce_in, nce_label, 10000) prelu_in = fluid.data(name="prelu_in", shape=[None, 5, 10, 10], dtype='float32') prelu_out_1 = fluid.layers.prelu(prelu_in, "channel") prelu_out_2 = fluid.layers.prelu(prelu_in, "channel") bilinear_tensor_pro_x = fluid.data("t1", shape=[None, 5], dtype="float32") bilinear_tensor_pro_y = fluid.data("t2", shape=[None, 4], dtype="float32") bilinear_tensor_pro_out_1 = fluid.layers.bilinear_tensor_product( x=bilinear_tensor_pro_x, y=bilinear_tensor_pro_y, size=1000) bilinear_tensor_pro_out_2 = fluid.layers.bilinear_tensor_product( x=bilinear_tensor_pro_x, y=bilinear_tensor_pro_y, size=1000) conv2d_trans_in = fluid.data(name="conv2d_trans_in", shape=[None, 10, 10, 10]) conv2d_trans_out_1 = fluid.layers.conv2d_transpose(conv2d_trans_in, num_filters=10, filter_size=5, act="relu") conv2d_trans_out_2 = fluid.layers.conv2d_transpose(conv2d_trans_in, num_filters=10, filter_size=5, act="relu") conv3d_trans_in = fluid.data(name='conv3d_trans_in', shape=[None, 3, 12, 32, 32], dtype='float32') conv3d_trans_out_1 = fluid.layers.conv3d_transpose( input=conv3d_trans_in, num_filters=2, filter_size=3, act="relu") conv3d_trans_out_2 = fluid.layers.conv3d_transpose( input=conv3d_trans_in, num_filters=2, filter_size=3, act="relu") groupnorm_in = fluid.data(name='groupnorm_in', shape=[None, 8, 32, 32], dtype='float32') groupnorm_out1 = fluid.layers.group_norm(input=groupnorm_in, groups=4) groupnorm_out2 = fluid.layers.group_norm(input=groupnorm_in, groups=4) ''' spec_norm = fluid.data(name='spec_norm', shape=[2, 8, 32, 32], dtype='float32') spe_norm_out_1 = fluid.layers.spectral_norm(weight=spec_norm, dim=1, power_iters=2) spe_norm_out_2 = fluid.layers.spectral_norm(weight=spec_norm, dim=1, power_iters=2) ''' nodes_vector = fluid.data(name='vectors', shape=[None, 10, 5], dtype='float32') edge_set = fluid.data(name='edge_set', shape=[None, 10, 2], dtype='float32') tree_conv_out1 = fluid.contrib.layers.tree_conv( nodes_vector, edge_set, 6, 1, 2) tree_conv_out2 = fluid.contrib.layers.tree_conv( nodes_vector, edge_set, 6, 1, 2) para1 = fluid.layers.create_parameter([100, 100], 'float32', name="weight_test_1") para2 = fluid.layers.create_parameter([20, 200], 'float32', name="weight_test_2") para_list = fluid.default_main_program().list_vars() exe = fluid.Executor(fluid.CPUPlace( ) if not fluid.is_compiled_with_cuda() else fluid.CUDAPlace(0)) out = exe.run(framework.default_startup_program()) fluid.save(framework.default_main_program(), "./test_1") para_dict = fluid.load_program_state("./test_1") new_dict = {} for k, v in para_dict.items(): #print( k, v.shape ) if k.startswith("fc"): name = k.replace("fc", "linear", 1) new_dict[name] = v else: new_dict[k] = v with fluid.dygraph.guard(): class MyTest(fluid.dygraph.Layer): def __init__(self): super(MyTest, self).__init__() self.linear1 = Linear(10, 10) self.lienar2 = Linear(10, 20) self.conv2d_1 = Conv2D(num_channels=10, num_filters=10, filter_size=5, act="relu") self.conv2d_2 = Conv2D(num_channels=10, num_filters=10, filter_size=5, act="relu") self.conv3d_1 = Conv3D(num_channels=3, num_filters=2, filter_size=3, act="relu") self.conv3d_2 = Conv3D(num_channels=3, num_filters=2, filter_size=3, act="relu") self.batch_norm_1 = BatchNorm(10) self.batch_norm_2 = BatchNorm(10) self.emb1 = Embedding([1000, 100]) self.emb2 = Embedding([2000, 200]) self.layer_norm_1 = LayerNorm([10]) self.layer_norm_2 = LayerNorm(10) self.nce1 = NCE(10000, 100) self.nce2 = NCE(10000, 100) self.prelu1 = PRelu("channel", channel=5) self.prelu2 = PRelu("channel", channel=5) self.group_norm1 = GroupNorm(8, 4) self.gourp_norm2 = GroupNorm(8, 4) self.w_1 = self.create_parameter([100, 100], dtype='float32', attr="weight_test_1") self.w_2 = self.create_parameter([20, 200], dtype='float32', attr="weight_test_2") my_test = MyTest() my_test.set_dict(new_dict, use_structured_name=False) for k, v in my_test.state_dict().items(): self.assertTrue(np.array_equal(v.numpy(), new_dict[v.name]))
# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest import paddle.fluid.core as core from paddle.fluid.framework import Program, default_startup_program main_program = default_startup_program() class TestOperator(unittest.TestCase): def test_error_type(self): block = main_program.create_block() try: block.append_op() self.assertFail() except ValueError as v_err: self.assertEqual( v_err.message, "`type` to initilized an Operator can not be None.") try: block.append_op(type="no_such_op") self.assertFail()