def _initialize_dataloaders(self, collate_fn):
        self.dataloaders = {}
        batch_size = self.config["training"]["batch_size"]
        num_workers = self.config["training"]["num_workers"]
        batch_size_multiplier = self.config["training"].get(
            "batch_size_multiplier", 1.0)

        if self.action == "training":
            for set_name, set_info in self.config["data"].items():
                if set_name not in ["train", "val", "test"]:
                    continue

                if set_name == "train":
                    shuffle = True
                    bs = batch_size
                else:
                    shuffle = False if set_name == "test" else True
                    bs = round(batch_size * batch_size_multiplier)

                dataset = CustomDataset(self.config,
                                        tokenizer=self.tokenizer,
                                        paths=set_info["paths"])
                self.dataloaders[set_name] = DataLoader(
                    dataset,
                    batch_size=bs,
                    shuffle=shuffle,
                    collate_fn=collate_fn,
                    num_workers=num_workers)

        elif self.action == "evaluation":
            if self.config["data_path"] is None and not (
                    "val" in self.config["data"]):
                raise ValueError(
                    "Either argument `data_path` or `val` value in the config file must be specified."
                )

            if self.config["data_path"] is None:
                data_path = self.config["data"]["val"]["paths"]
            else:
                data_path = self.config["data_path"]
            dataset = CustomDataset(self.config,
                                    tokenizer=self.tokenizer,
                                    paths=data_path)
            self.dataloaders["eval"] = DataLoader(
                dataset,
                batch_size=round(batch_size * batch_size_multiplier),
                shuffle=False,
                collate_fn=collate_fn,
                num_workers=num_workers)

        elif self.action == "predict":
            pass

        else:
            raise ValueError(f"Unrecognized action: {self.action}")
    def predict(self, text):
        # Tokenize
        token_idxs = CustomDataset.process_input(self.tokenizer, text)
        token_idxs = torch.Tensor(token_idxs).long().unsqueeze(0).to(
            self.device)
        attention_mask = torch.ones_like(token_idxs)

        tensorOutput = self.model(token_idxs,
                                  attention_mask=attention_mask,
                                  is_training=False)

        output = {
            "food_score_preds":
            tensorOutput["food_score_preds"].item(),
            "food_existence_preds":
            tensorOutput["food_existence_preds"].item(),
            "service_score_preds":
            tensorOutput["service_score_preds"].item(),
            "service_existence_preds":
            tensorOutput["service_existence_preds"].item(),
            "price_score_preds":
            tensorOutput["price_score_preds"].item(),
            "price_existence_preds":
            tensorOutput["price_existence_preds"].item(),
        }

        print("Review:", text)
        print("Predict:", output)
        print("=" * 40)

        return output
Exemple #3
0
def train(model, radio_landmark_loss, epoch_num, data_path, name, shape):
    # 获取数据
    train_dataset = CustomDataset(data_path)
    train_loader = DataLoader(dataset=train_dataset,
                              batch_size=batch_size,
                              shuffle=True)

    # 设置优化方法
    scheduler = paddle.optimizer.lr.PiecewiseDecay(
        boundaries=[6, 14, 20],
        values=[0.001, 0.0001, 0.00001, 0.000001],
        verbose=True)
    optimizer = paddle.optimizer.Adam(
        parameters=model.parameters(),
        learning_rate=scheduler,
        weight_decay=paddle.regularizer.L2Decay(1e-4))

    # 获取损失函数
    class_loss = ClassLoss()
    bbox_loss = BBoxLoss()
    landmark_loss = LandmarkLoss()
    for epoch in range(epoch_num):
        for batch_id, (img, label, bbox,
                       landmark) in enumerate(train_loader()):
            class_out, bbox_out, landmark_out = model(img)
            cls_loss = class_loss(class_out, label)
            box_loss = bbox_loss(bbox_out, bbox, label)
            landmarks_loss = landmark_loss(landmark_out, landmark, label)
            total_loss = radio_cls_loss * cls_loss + radio_bbox_loss * box_loss + radio_landmark_loss * landmarks_loss
            total_loss.backward()
            optimizer.step()
            optimizer.clear_grad()
            if batch_id % 100 == 0:
                acc = accuracy(class_out, label)
                print(
                    '[%s] Train epoch %d, batch %d, total_loss: %f, cls_loss: %f, box_loss: %f, landmarks_loss: %f, '
                    'accuracy:%f' %
                    (datetime.now(), epoch, batch_id, total_loss, cls_loss,
                     box_loss, landmarks_loss, acc))
        scheduler.step()

        # 保存模型
        if not os.path.exists(model_path):
            os.makedirs(model_path)
        paddle.jit.save(layer=model,
                        path=os.path.join(model_path, name),
                        input_spec=[InputSpec(shape=shape, dtype='float32')])
radio_bbox_loss = 0.5
radio_landmark_loss = 1.0

# 训练参数值
data_path = '../dataset/48/all_data'
batch_size = 384
learning_rate = 1e-3
epoch_num = 22
model_path = '../infer_models'

# 获取P模型
model = ONet()
paddle.summary(model, input_size=(batch_size, 3, 48, 48))

# 获取数据
train_dataset = CustomDataset(data_path)
train_loader = DataLoader(dataset=train_dataset,
                          batch_size=batch_size,
                          shuffle=True)

# 设置优化方法
scheduler = paddle.optimizer.lr.PiecewiseDecay(
    boundaries=[6, 14, 20],
    values=[0.001, 0.0001, 0.00001, 0.000001],
    verbose=True)
optimizer = paddle.optimizer.Adam(
    parameters=model.parameters(),
    learning_rate=scheduler,
    weight_decay=paddle.regularizer.L2Decay(1e-4))

# 获取损失函数