def train(train_file_list_path, test_file_list_path, label_dict_path, model_save_dir): # 获取训练列表 train_file_list = get_file_list(train_file_list_path) # 获取测试列表 test_file_list = get_file_list(test_file_list_path) # 使用训练数据生成标记字典 if not os.path.exists(label_dict_path): print(("Label dictionary is not given, the dictionary " "is automatically built from the training data.")) build_label_dict(train_file_list, label_dict_path) # 获取标签字典 char_dict = load_dict(label_dict_path) # 获取字典大小 dict_size = len(char_dict) # 定义网络拓扑 model = Model(dict_size, IMAGE_SHAPE, is_infer=False) # 初始化PaddlePaddle paddle.init(use_gpu=True, trainer_count=1) # 创建优化方法 optimizer = paddle.optimizer.Momentum( momentum=0.9, regularization=paddle.optimizer.L2Regularization(rate=0.0005 * 128), learning_rate=0.001 / 128, learning_rate_decay_a=0.1, learning_rate_decay_b=128000 * 35, learning_rate_schedule="discexp", ) # 创建训练参数 params = paddle.parameters.create(model.cost) # 定义训练器 trainer = paddle.trainer.SGD(cost=model.cost, parameters=params, update_equation=optimizer, extra_layers=model.eval) # 获取reader my_reader = Reader(char_dict=char_dict, image_shape=IMAGE_SHAPE) # 说明数据层之间的关系 feeding = {'image': 0, 'label': 1} # 训练事件 def event_handler(event): if isinstance(event, paddle.event.EndIteration): if event.batch_id % 100 == 0: print("Pass %d, batch %d, Samples %d, Cost %f" % (event.pass_id, event.batch_id, event.batch_id * BATCH_SIZE, event.cost)) if isinstance(event, paddle.event.EndPass): # 这里由于训练和测试数据共享相同的格式 # 我们仍然使用reader.train_reader来读取测试数据 test_reader = paddle.batch( my_reader.train_reader(test_file_list), batch_size=BATCH_SIZE) result = trainer.test(reader=test_reader, feeding=feeding) print("Test %d, Cost %f" % (event.pass_id, result.cost)) # 检查保存model的路径是否存在,如果不存在就创建 if not os.path.exists(model_save_dir): os.mkdir(model_save_dir) with gzip.open( os.path.join(model_save_dir, "params_pass.tar.gz"), "w") as f: trainer.save_parameter_to_tar(f) # 获取训练数据的reader train_reader = paddle.batch( paddle.reader.shuffle( my_reader.train_reader(train_file_list), buf_size=1000), batch_size=BATCH_SIZE) # 开始训练 trainer.train(reader=train_reader, feeding=feeding, event_handler=event_handler, num_passes=1000)
def train(train_file_list_path, test_file_list_path, label_dict_path, model_save_dir): if not os.path.exists(model_save_dir): os.mkdir(model_save_dir) train_file_list = get_file_list(train_file_list_path) test_file_list = get_file_list(test_file_list_path) if not os.path.exists(label_dict_path): print(("Label dictionary is not given, the dictionary " "is automatically built from the training data.")) build_label_dict(train_file_list, label_dict_path) char_dict = load_dict(label_dict_path) dict_size = len(char_dict) data_generator = DataGenerator(char_dict=char_dict, image_shape=conf.image_shape) paddle.init(use_gpu=conf.use_gpu, trainer_count=conf.trainer_count) # Create optimizer. optimizer = paddle.optimizer.Momentum(momentum=conf.momentum) # Define network topology. model = Model(dict_size, conf.image_shape, is_infer=False) # Create all the trainable parameters. params = paddle.parameters.create(model.cost) trainer = paddle.trainer.SGD(cost=model.cost, parameters=params, update_equation=optimizer, extra_layers=model.eval) # Feeding dictionary. feeding = {'image': 0, 'label': 1} def event_handler(event): if isinstance(event, paddle.event.EndIteration): if event.batch_id % conf.log_period == 0: print("Pass %d, batch %d, Samples %d, Cost %f, Eval %s" % (event.pass_id, event.batch_id, event.batch_id * conf.batch_size, event.cost, event.metrics)) if isinstance(event, paddle.event.EndPass): # Here, because training and testing data share a same format, # we still use the reader.train_reader to read the testing data. result = trainer.test(reader=paddle.batch( data_generator.train_reader(test_file_list), batch_size=conf.batch_size), feeding=feeding) print("Test %d, Cost %f, Eval %s" % (event.pass_id, result.cost, result.metrics)) with gzip.open( os.path.join(model_save_dir, "params_pass_%05d.tar.gz" % event.pass_id), "w") as f: trainer.save_parameter_to_tar(f) trainer.train(reader=paddle.batch(paddle.reader.shuffle( data_generator.train_reader(train_file_list), buf_size=conf.buf_size), batch_size=conf.batch_size), feeding=feeding, event_handler=event_handler, num_passes=conf.num_passes)
def train(train_data_dir, test_data_dir, word_dict_path, label_dict_path, model_save_dir): """ :params train_data_path: The path of training data, if this parameter is not specified, imdb dataset will be used to run this example :type train_data_path: str :params test_data_path: The path of testing data, if this parameter is not specified, imdb dataset will be used to run this example :type test_data_path: str :params word_dict_path: The path of word dictionary, if this parameter is not specified, imdb dataset will be used to run this example :type word_dict_path: str :params label_dict_path: The path of label dictionary, if this parameter is not specified, imdb dataset will be used to run this example :type label_dict_path: str :params model_save_dir: dir where models saved :type model_save_dir: str """ if train_data_dir is not None: assert word_dict_path and label_dict_path, ( "The parameter train_data_dir, word_dict_path, label_dict_path " "should be set at the same time.") if not os.path.exists(model_save_dir): os.mkdir(model_save_dir) use_default_data = (train_data_dir is None) if use_default_data: logger.info(("No training data are porivided, " "use imdb to train the model.")) logger.info("Please wait to build the word dictionary ...") word_dict = reader.imdb_word_dict() train_reader = paddle.batch(paddle.reader.shuffle( lambda: reader.imdb_train(word_dict), buf_size=1000), batch_size=100) test_reader = paddle.batch(lambda: reader.imdb_test(word_dict), batch_size=100) class_num = 2 else: if word_dict_path is None or not os.path.exists(word_dict_path): logger.info(("Word dictionary is not given, the dictionary " "is automatically built from the training data.")) # build the word dictionary to map the original string-typed # words into integer-typed index build_word_dict(data_dir=train_data_dir, save_path=word_dict_path, use_col=1, cutoff_fre=0) if not os.path.exists(label_dict_path): logger.info(("Label dictionary is not given, the dictionary " "is automatically built from the training data.")) # build the label dictionary to map the original string-typed # label into integer-typed index build_label_dict(data_dir=train_data_dir, save_path=label_dict_path, use_col=0) word_dict = load_dict(word_dict_path) label_dict = load_dict(label_dict_path) class_num = len(label_dict) logger.info("Class number is : %d." % class_num) train_reader = paddle.batch(paddle.reader.shuffle( reader.train_reader(train_data_dir, word_dict, label_dict), buf_size=conf.buf_size), batch_size=conf.batch_size) if test_data_dir is not None: # here, because training and testing data share a same format, # we still use the reader.train_reader to read the testing data. test_reader = paddle.batch(paddle.reader.shuffle( reader.train_reader(test_data_dir, word_dict, label_dict), buf_size=conf.buf_size), batch_size=conf.batch_size) else: test_reader = None dict_dim = len(word_dict) logger.info("Length of word dictionary is : %d." % (dict_dim)) paddle.init(use_gpu=conf.use_gpu, trainer_count=conf.trainer_count) # create optimizer adam_optimizer = paddle.optimizer.Adam( learning_rate=conf.learning_rate, regularization=paddle.optimizer.L2Regularization( rate=conf.l2_learning_rate), model_average=paddle.optimizer.ModelAverage( average_window=conf.average_window)) # define network topology. cost, prob, label = nested_net(dict_dim, class_num, is_infer=False) # create all the trainable parameters. parameters = paddle.parameters.create(cost) # create the trainer instance. trainer = paddle.trainer.SGD(cost=cost, extra_layers=paddle.evaluator.auc( input=prob, label=label), parameters=parameters, update_equation=adam_optimizer) # feeding dictionary feeding = {"word": 0, "label": 1} def _event_handler(event): """ Define the end batch and the end pass event handler. """ if isinstance(event, paddle.event.EndIteration): if event.batch_id % conf.log_period == 0: logger.info( "Pass %d, Batch %d, Cost %f, %s\n" % (event.pass_id, event.batch_id, event.cost, event.metrics)) if isinstance(event, paddle.event.EndPass): if test_reader is not None: result = trainer.test(reader=test_reader, feeding=feeding) logger.info("Test at Pass %d, %s \n" % (event.pass_id, result.metrics)) with gzip.open( os.path.join(model_save_dir, "params_pass_%05d.tar.gz" % event.pass_id), "w") as f: trainer.save_parameter_to_tar(f) # begin training network trainer.train(reader=train_reader, event_handler=_event_handler, feeding=feeding, num_passes=conf.num_passes) logger.info("Training has finished.")
def train(train_file_list_path, test_file_list_path, label_dict_path, model_save_dir): # 检查保存model的路径是否存在,如果不存在就创建 if not os.path.exists(model_save_dir): os.mkdir(model_save_dir) # 获取训练列表 train_file_list = get_file_list(train_file_list_path) # 获取测试列表 test_file_list = get_file_list(test_file_list_path) # 使用训练数据生成标记字典 if not os.path.exists(label_dict_path): print(("Label dictionary is not given, the dictionary " "is automatically built from the training data.")) build_label_dict(train_file_list, label_dict_path) # 获取标签字典 char_dict = load_dict(label_dict_path) # 获取字典大小 dict_size = len(char_dict) # 获取reader data_generator = DataGenerator(char_dict=char_dict, image_shape=conf.image_shape) # 初始化PaddlePaddle paddle.init(use_gpu=conf.use_gpu, trainer_count=conf.trainer_count) # 创建训练参数 optimizer = paddle.optimizer.Momentum(momentum=conf.momentum) # 定义网络拓扑 model = Model(dict_size, conf.image_shape, is_infer=False) # 创建训练参数 params = paddle.parameters.create(model.cost) trainer = paddle.trainer.SGD(cost=model.cost, parameters=params, update_equation=optimizer, extra_layers=model.eval) # 说明数据层之间的关系 feeding = {'image': 0, 'label': 1} # 训练事件 def event_handler(event): if isinstance(event, paddle.event.EndIteration): if event.batch_id % conf.log_period == 0: print("Pass %d, batch %d, Samples %d, Cost %f, Eval %s" % (event.pass_id, event.batch_id, event.batch_id * conf.batch_size, event.cost, event.metrics)) if isinstance(event, paddle.event.EndPass): # 这里由于训练和测试数据共享相同的格式 # 我们仍然使用reader.train_reader来读取测试数据 result = trainer.test(reader=paddle.batch( data_generator.train_reader(test_file_list), batch_size=conf.batch_size), feeding=feeding) print("Test %d, Cost %f, Eval %s" % (event.pass_id, result.cost, result.metrics)) with gzip.open(os.path.join(model_save_dir, "params_pass.tar.gz"), "w") as f: trainer.save_parameter_to_tar(f) # 开始训练 trainer.train(reader=paddle.batch(paddle.reader.shuffle( data_generator.train_reader(train_file_list), buf_size=conf.buf_size), batch_size=conf.batch_size), feeding=feeding, event_handler=event_handler, num_passes=conf.num_passes)