def __init__(self, data_types, feeding=None): self.input_names = [] input_types = [] if feeding is None: feeding = default_feeding_map(data_types) self.feeding = feeding for each in data_types: self.input_names.append(each[0]) if not isinstance(each[1], pydp2.InputType): raise TypeError("second item in each data_type should be an " "InputType") input_types.append(each[1]) DataProviderConverter.__init__(self, input_types)
def __init__(self, train_conf, dict_file, model_dir, label_file): """ train_conf: trainer configure. dict_file: word dictionary file name. model_dir: directory of model. """ self.dict = {} self.labels = {} self.labels_reverse = {} self.load_dict_label(dict_file, label_file) len_dict = len(self.dict) len_label = len(self.labels) conf = parse_config( train_conf, 'dict_len=' + str(len_dict) + ',label_len=' + str(len_label) + ',is_predict=True') self.network = swig_paddle.GradientMachine.createFromConfigProto( conf.model_config) self.network.loadParameters(model_dir) slots = [ integer_value_sequence(len_dict), integer_value_sequence(len_dict), integer_value_sequence(len_dict), integer_value_sequence(len_dict), integer_value_sequence(len_dict), integer_value_sequence(2) ] self.converter = DataProviderConverter(slots)
def paddle_predict_main(q, result_q): api.initPaddle("--use_gpu=false") gm = api.GradientMachine.loadFromConfigFile("./output/model/pass-00000/trainer_config.py") assert isinstance(gm, api.GradientMachine) converter = DataProviderConverter(input_types=[dense_vector(28 * 28)]) while True: features = q.get() val = gm.forwardTest(converter([[features]]))[0]['value'][0] result_q.put(val)
def main(): conf = parse_config("./mnist_model/trainer_config.py", "") print conf.data_config.load_data_args network = swig_paddle.GradientMachine.createFromConfigProto( conf.model_config) assert isinstance(network, swig_paddle.GradientMachine) # For code hint. network.loadParameters("./mnist_model/") converter = DataProviderConverter([dense_vector(784)]) inArg = converter(TEST_DATA) print network.forwardTest(inArg)
def __init__(self, train_conf, use_gpu=True, model_dir=None, resize_dim=None, crop_dim=None, mean_file=None, oversample=False, is_color=False): """ train_conf: 网络配置文件 model_dir: 模型路径 resize_dim: 设为原图大小 crop_dim: 图像裁剪大小,一般设为原图大小 oversample: bool, oversample表示多次裁剪,这里禁用 """ self.train_conf = train_conf self.model_dir = model_dir if model_dir is None: self.model_dir = os.path.dirname(train_conf) self.resize_dim = resize_dim self.crop_dims = [crop_dim, crop_dim] self.oversample = oversample self.is_color = is_color self.transformer = image_util.ImageTransformer(is_color = is_color) self.transformer.set_transpose((2,0,1)) self.mean_file = mean_file mean = np.load(self.mean_file)['data_mean'] mean = mean.reshape(1, self.crop_dims[0], self.crop_dims[1]) self.transformer.set_mean(mean) # mean pixel gpu = 1 if use_gpu else 0 conf_args = "is_test=1,use_gpu=%d,is_predict=1" % (gpu) #使用 parse_config() 解析训练时的配置文件 conf = parse_config(train_conf, conf_args) #PaddlePaddle目前使用Swig对其常用的预测接口进行了封装,使在Python环境下的预测接口更加简单 #使用 swig_paddle.initPaddle() 传入命令行参数初始化 PaddlePaddle swig_paddle.initPaddle("--use_gpu=%d" % (int(use_gpu))) #使用 swig_paddle.GradientMachine.createFromConfigproto() 根据上一步解析好的配置创建神经网络 self.network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) assert isinstance(self.network, swig_paddle.GradientMachine) #从模型文件加载参数 self.network.loadParameters(self.model_dir) data_size = 1 * self.crop_dims[0] * self.crop_dims[1] slots = [dense_vector(data_size)] ''' 创建一个 DataProviderConverter 对象converter。 swig_paddle接受的原始数据是C++的Matrix,也就是直接写内存的float数组。 这个接口并不用户友好。所以,我们提供了一个工具类DataProviderConverter。 这个工具类接收和PyDataProvider2一样的输入数据 ''' self.converter = DataProviderConverter(slots)
def __init__(self, data_types, feeding=None): self.input_names = [] input_types = [] if feeding is None: feeding = default_feeding_map(data_types) elif isinstance(feeding, collections.Sequence): feed_list = feeding feeding = dict() for i, name in enumerate(feed_list): feeding[name] = i elif not isinstance(feeding, dict): raise TypeError("Feeding should be dict or sequence or None.") self.feeding = feeding for each in data_types: self.input_names.append(each[0]) if not isinstance(each[1], pydp2.InputType): raise TypeError("second item in each data_type should be an " "InputType") input_types.append(each[1]) DataProviderConverter.__init__(self, input_types)
def main(): options = parse_arguments() api.initPaddle("--use_gpu=%s" % options.use_gpu, "--trainer_count=%s" % options.trainer_count) word_dict = load_dict(options.dict_file) train_dataset = list(load_data(options.train_data, word_dict)) if options.test_data: test_dataset = list(load_data(options.test_data, word_dict)) else: test_dataset = None trainer_config = parse_config(options.config, "dict_file=%s" % options.dict_file) # No need to have data provider for trainer trainer_config.ClearField('data_config') trainer_config.ClearField('test_data_config') # create a GradientMachine from the model configuratin model = api.GradientMachine.createFromConfigProto( trainer_config.model_config) # create a trainer for the gradient machine trainer = api.Trainer.create(trainer_config, model) # create a data converter which converts data to PaddlePaddle # internal format input_types = [ integer_value_sequence(len(word_dict)) if options.seq else sparse_binary_vector(len(word_dict)), integer_value(2) ] converter = DataProviderConverter(input_types) batch_size = trainer_config.opt_config.batch_size trainer.startTrain() for train_pass in xrange(options.num_passes): trainer.startTrainPass() random.shuffle(train_dataset) for pos in xrange(0, len(train_dataset), batch_size): batch = itertools.islice(train_dataset, pos, pos + batch_size) size = min(batch_size, len(train_dataset) - pos) trainer.trainOneDataBatch(size, converter(batch)) trainer.finishTrainPass() if test_dataset: trainer.startTestPeriod() for pos in xrange(0, len(test_dataset), batch_size): batch = itertools.islice(test_dataset, pos, pos + batch_size) size = min(batch_size, len(test_dataset) - pos) trainer.testOneDataBatch(size, converter(batch)) trainer.finishTestPeriod() trainer.finishTrain()
def __init__(self, train_conf, use_gpu=True, model_dir=None, resize_dim=None, crop_dim=None, mean_file=None, oversample=False, is_color=True): """ train_conf: network configure. model_dir: string, directory of model. resize_dim: int, resized image size. crop_dim: int, crop size. mean_file: string, image mean file. oversample: bool, oversample means multiple crops, namely five patches (the four corner patches and the center patch) as well as their horizontal reflections, ten crops in all. """ self.train_conf = train_conf self.model_dir = model_dir if model_dir is None: self.model_dir = os.path.dirname(train_conf) self.resize_dim = resize_dim self.crop_dims = [crop_dim, crop_dim] self.oversample = oversample self.is_color = is_color self.transformer = image_util.ImageTransformer(is_color=is_color) self.transformer.set_transpose((2, 0, 1)) self.mean_file = mean_file mean = np.load(self.mean_file)['data_mean'] mean = mean.reshape(3, self.crop_dims[0], self.crop_dims[1]) self.transformer.set_mean(mean) # mean pixel gpu = 1 if use_gpu else 0 conf_args = "is_test=1,use_gpu=%d,is_predict=1" % (gpu) conf = parse_config(train_conf, conf_args) swig_paddle.initPaddle("--use_gpu=%d" % (gpu)) self.network = swig_paddle.GradientMachine.createFromConfigProto( conf.model_config) assert isinstance(self.network, swig_paddle.GradientMachine) self.network.loadParameters(self.model_dir) data_size = 3 * self.crop_dims[0] * self.crop_dims[1] slots = [dense_vector(data_size)] self.converter = DataProviderConverter(slots)
def predict(data): path = os.path.split(os.path.realpath(__file__))[0] conf = parse_config(path + "/trainer_config_age.py", "is_predict=1") print conf.data_config.load_data_args network = swig_paddle.GradientMachine.createFromConfigProto( conf.model_config) network.loadParameters(path + "/output_age/pass-00099") converter = DataProviderConverter([dense_vector(26)]) inArg = converter(data) network.forwardTest(inArg) output = network.getLayerOutputs("__fc_layer_0__") #print output prob = output["__fc_layer_0__"][0] #print prob lab = np.argsort(-prob) #print lab return lab[0]
def convert(self, dat, argument=None): """ :param dat: A list of mini-batch data. Each sample is a list or tuple one feature or multiple features. :type dat: list :param argument: An Arguments object contains this mini-batch data with one or multiple features. The Arguments definition is in the API. :type argument: py_paddle.swig_paddle.Arguments """ def reorder_data(data): retv = [] for each in data: reorder = [] for name in self.input_names: reorder.append(each[self.feeding[name]]) retv.append(reorder) return retv return DataProviderConverter.convert(self, reorder_data(dat), argument)
def __init__(self, train_conf, dict_file, model_dir=None, label_file = None): """ train_conf: trainer configure. dict_file: word dictionary file name. model_dir: directory of model. """ self.train_conf = train_conf self.dict_file = dict_file self.word_dict = {} self.dict_dim = self.load_dict() self.model_dir = model_dir if model_dir is None: self.model_dir = os.path.dirname(train_conf) self.label = None if label_file is not None: self.load_label(label_file) conf = parse_config(train_conf, "is_predict=1") self.network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) self.network.loadParameters(self.model_dir) input_types = [integer_value_sequence(self.dict_dim)] self.converter = DataProviderConverter(input_types)
def train(self, train_data_reader, topology, parameters, num_passes=1, test_data_reader=None, event_handler=None, batch_size=32, data_types=None): """ Training method. Will train num_passes of input data. :param train_data_reader: :param topology: Network Topology, a protobuf ModelConfig message. :param parameters: The parameter pools. :param num_passes: The total train passes. :param test_data_reader: :param event_handler: Event handler. A method will be invoked when event occurred. :type event_handler: (BaseEvent) => None :param batch_size: Not important, will be removed after data refactor. :param data_types: Not important, will be removed after data refactor. :return: """ if event_handler is None: event_handler = default_event_handler __check_train_args__(**locals()) gm = api.GradientMachine.createFromConfigProto( topology, api.CREATE_MODE_NORMAL, self.__optimizer__.enable_types()) assert isinstance(gm, api.GradientMachine) parameters.append_gradient_machine(gm) updater = self.__optimizer__.create_local_updater() updater.init(gm) gm.start() out_args = api.Arguments.createArguments(0) data_types_lists = [] for each in topology.input_layer_names: if each not in data_types: raise ValueError() data_types_lists.append(data_types[each]) converter = DataProviderConverter(input_types=data_types_lists) for pass_id in xrange(num_passes): updater.startPass() for batch_id, data_batch in enumerate( __data_reader_to_batch__(train_data_reader, batch_size, topology)): pass_type = updater.startBatch(len(data_batch)) gm.forwardBackward(converter(data_batch), out_args, pass_type) for each_param in gm.getParameters(): updater.update(each_param) # Get cost. We use numpy to calculate total cost for this batch. cost_vec = out_args.getSlotValue(0) cost_vec = cost_vec.copyToNumpyMat() cost = cost_vec.sum() / len(data_batch) updater.finishBatch(cost) event_handler( v2_event.EndIteration(pass_id=pass_id, batch_id=batch_id, cost=cost)) updater.finishPass() gm.finish()
def __init__(self, train_conf, model_dir=None, resize_dim=256, crop_dim=224, use_gpu=True, mean_file=None, output_layer=None, oversample=False, is_color=True): """ train_conf: network configure. model_dir: string, directory of model. resize_dim: int, resized image size. crop_dim: int, crop size. mean_file: string, image mean file. oversample: bool, oversample means multiple crops, namely five patches (the four corner patches and the center patch) as well as their horizontal reflections, ten crops in all. """ self.train_conf = train_conf self.model_dir = model_dir if model_dir is None: self.model_dir = os.path.dirname(train_conf) self.resize_dim = resize_dim self.crop_dims = [crop_dim, crop_dim] self.oversample = oversample self.is_color = is_color self.output_layer = output_layer if self.output_layer: assert isinstance(self.output_layer, basestring) self.output_layer = self.output_layer.split(",") self.transformer = image_util.ImageTransformer(is_color=is_color) self.transformer.set_transpose((2, 0, 1)) self.transformer.set_channel_swap((2, 1, 0)) self.mean_file = mean_file if self.mean_file is not None: mean = np.load(self.mean_file)['data_mean'] mean = mean.reshape(3, self.crop_dims[0], self.crop_dims[1]) self.transformer.set_mean(mean) # mean pixel else: # if you use three mean value, set like: # this three mean value is calculated from ImageNet. self.transformer.set_mean(np.array([103.939, 116.779, 123.68])) conf_args = "is_test=1,use_gpu=%d,is_predict=1" % (int(use_gpu)) conf = parse_config(train_conf, conf_args) swig_paddle.initPaddle("--use_gpu=%d" % (int(use_gpu))) self.network = swig_paddle.GradientMachine.createFromConfigProto( conf.model_config) assert isinstance(self.network, swig_paddle.GradientMachine) self.network.loadParameters(self.model_dir) data_size = 3 * self.crop_dims[0] * self.crop_dims[1] slots = [dense_vector(data_size)] self.converter = DataProviderConverter(slots)
def main(): api.initPaddle("-use_gpu=false", "-trainer_count=4") # use 4 cpu cores optimizer = paddle_v2.optimizer.Adam( learning_rate=1e-4, batch_size=1000, model_average=ModelAverage(average_window=0.5), regularization=L2Regularization(rate=0.5)) # Create Local Updater. Local means not run in cluster. # For a cluster training, here we can change to createRemoteUpdater # in future. updater = optimizer.create_local_updater() assert isinstance(updater, api.ParameterUpdater) # define network images = paddle_v2.layer.data(name='pixel', type=paddle_v2.data_type.dense_vector(784)) label = paddle_v2.layer.data(name='label', type=paddle_v2.data_type.integer_value(10)) hidden1 = paddle_v2.layer.fc(input=images, size=200) hidden2 = paddle_v2.layer.fc(input=hidden1, size=200) inference = paddle_v2.layer.fc(input=hidden2, size=10, act=paddle_v2.activation.Softmax()) cost = paddle_v2.layer.classification_cost(input=inference, label=label) # Create Simple Gradient Machine. model_config = paddle_v2.layer.parse_network(cost) m = api.GradientMachine.createFromConfigProto(model_config, api.CREATE_MODE_NORMAL, optimizer.enable_types()) # This type check is not useful. Only enable type hint in IDE. # Such as PyCharm assert isinstance(m, api.GradientMachine) # Initialize Parameter by numpy. init_parameter(network=m) # Initialize ParameterUpdater. updater.init(m) # DataProvider Converter is a utility convert Python Object to Paddle C++ # Input. The input format is as same as Paddle's DataProvider. converter = DataProviderConverter(input_types=[images.type, label.type]) train_file = './data/raw_data/train' test_file = './data/raw_data/t10k' # start gradient machine. # the gradient machine must be started before invoke forward/backward. # not just for training, but also for inference. m.start() # evaluator can print error rate, etc. It is a C++ class. batch_evaluator = m.makeEvaluator() test_evaluator = m.makeEvaluator() # Get Train Data. # TrainData will stored in a data pool. Currently implementation is not care # about memory, speed. Just a very naive implementation. train_data_generator = input_order_converter(read_from_mnist(train_file)) train_data = BatchPool(train_data_generator, 512) # outArgs is Neural Network forward result. Here is not useful, just passed # to gradient_machine.forward outArgs = api.Arguments.createArguments(0) for pass_id in xrange(2): # we train 2 passes. updater.startPass() for batch_id, data_batch in enumerate(train_data()): # data_batch is input images. # here, for online learning, we could get data_batch from network. # Start update one batch. pass_type = updater.startBatch(len(data_batch)) # Start BatchEvaluator. # batch_evaluator can be used between start/finish. batch_evaluator.start() # forwardBackward is a shortcut for forward and backward. # It is sometimes faster than invoke forward/backward separately, # because in GradientMachine, it may be async. m.forwardBackward(converter(data_batch), outArgs, pass_type) for each_param in m.getParameters(): updater.update(each_param) # Get cost. We use numpy to calculate total cost for this batch. cost_vec = outArgs.getSlotValue(0) cost_vec = cost_vec.copyToNumpyMat() cost = cost_vec.sum() / len(data_batch) # Make evaluator works. m.eval(batch_evaluator) # Print logs. print 'Pass id', pass_id, 'Batch id', batch_id, 'with cost=', \ cost, batch_evaluator batch_evaluator.finish() # Finish batch. # * will clear gradient. # * ensure all values should be updated. updater.finishBatch(cost) # testing stage. use test data set to test current network. updater.apply() test_evaluator.start() test_data_generator = input_order_converter(read_from_mnist(test_file)) for data_batch in generator_to_batch(test_data_generator, 512): # in testing stage, only forward is needed. m.forward(converter(data_batch), outArgs, api.PASS_TEST) m.eval(test_evaluator) # print error rate for test data set print 'Pass', pass_id, ' test evaluator: ', test_evaluator test_evaluator.finish() updater.restore() updater.catchUpWith() params = m.getParameters() for each_param in params: assert isinstance(each_param, api.Parameter) value = each_param.getBuf(api.PARAMETER_VALUE) value = value.copyToNumpyArray() # Here, we could save parameter to every where you want print each_param.getName(), value updater.finishPass() m.finish()
import cPickle as pickle except ImportError: import pickle import sys if __name__ == '__main__': model_path = sys.argv[1] swig_paddle.initPaddle('--use_gpu=0') conf = parse_config("trainer_config.py", "is_predict=1") network = swig_paddle.GradientMachine.createFromConfigProto( conf.model_config) assert isinstance(network, swig_paddle.GradientMachine) network.loadParameters(model_path) with open('./data/meta.bin', 'rb') as f: meta = pickle.load(f) headers = list(meta_to_header(meta, 'movie')) headers.extend(list(meta_to_header(meta, 'user'))) cvt = DataProviderConverter(headers) while True: movie_id = int(raw_input("Input movie_id: ")) user_id = int(raw_input("Input user_id: ")) movie_meta = meta['movie'][movie_id] # Query Data From Meta. user_meta = meta['user'][user_id] data = [movie_id - 1] data.extend(movie_meta) data.append(user_id - 1) data.extend(user_meta) print "Prediction Score is %.2f" % ( (network.forwardTest(cvt.convert([data]))[0]['value'][0][0] + 5) / 2)
import cPickle as pickle except ImportError: import pickle import sys if __name__ == '__main__': model_path = sys.argv[1] swig_paddle.initPaddle('--use_gpu=0') conf = parse_config("trainer_config.py", "is_predict=1") network = swig_paddle.GradientMachine.createFromConfigProto( conf.model_config) assert isinstance(network, swig_paddle.GradientMachine) network.loadParameters(model_path) with open('./data/meta.bin', 'rb') as f: meta = pickle.load(f) headers = [h[1] for h in meta_to_header(meta, 'movie')] headers.extend([h[1] for h in meta_to_header(meta, 'user')]) cvt = DataProviderConverter(headers) while True: movie_id = int(raw_input("Input movie_id: ")) user_id = int(raw_input("Input user_id: ")) movie_meta = meta['movie'][movie_id] # Query Data From Meta. user_meta = meta['user'][user_id] data = [movie_id - 1] data.extend(movie_meta) data.append(user_id - 1) data.extend(user_meta) print "Prediction Score is %.2f" % ( (network.forwardTest(cvt.convert([data]))[0]['value'][0][0] + 5) / 2)
try: import cPickle as pickle except ImportError: import pickle import sys if __name__ == '__main__': model_path = sys.argv[1] swig_paddle.initPaddle('--use_gpu=0') conf = parse_config("trainer_config.py", "is_predict=1") network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) assert isinstance(network, swig_paddle.GradientMachine) network.loadParameters(model_path) with open('./data/meta.bin', 'rb') as f: meta = pickle.load(f) headers = list(meta_to_header(meta, 'movie')) headers.extend(list(meta_to_header(meta, 'user'))) cvt = DataProviderConverter(headers) while True: movie_id = int(raw_input("Input movie_id: ")) user_id = int(raw_input("Input user_id: ")) movie_meta = meta['movie'][movie_id] # Query Data From Meta. user_meta = meta['user'][user_id] data = [movie_id - 1] data.extend(movie_meta) data.append(user_id - 1) data.extend(user_meta) print "Prediction Score is %.2f" % ((network.forwardTest( cvt.convert([data]))[0]['value'][0][0] + 5) / 2)