def _initialize_dataloaders(self, collate_fn):
self.dataloaders = {}
batch_size = self.config["training"]["batch_size"]
num_workers = self.config["training"]["num_workers"]
batch_size_multiplier = self.config["training"].get(
"batch_size_multiplier", 1.0)
if self.action == "training":
for set_name, set_info in self.config["data"].items():
if set_name not in ["train", "val", "test"]:
continue
if set_name == "train":
shuffle = True
bs = batch_size
else:
shuffle = False if set_name == "test" else True
bs = round(batch_size * batch_size_multiplier)
dataset = CustomDataset(self.config,
tokenizer=self.tokenizer,
paths=set_info["paths"])
self.dataloaders[set_name] = DataLoader(
dataset,
batch_size=bs,
shuffle=shuffle,
collate_fn=collate_fn,
num_workers=num_workers)
elif self.action == "evaluation":
if self.config["data_path"] is None and not (
"val" in self.config["data"]):
raise ValueError(
"Either argument `data_path` or `val` value in the config file must be specified."
)
if self.config["data_path"] is None:
data_path = self.config["data"]["val"]["paths"]
else:
data_path = self.config["data_path"]
dataset = CustomDataset(self.config,
tokenizer=self.tokenizer,
paths=data_path)
self.dataloaders["eval"] = DataLoader(
dataset,
batch_size=round(batch_size * batch_size_multiplier),
shuffle=False,
collate_fn=collate_fn,
num_workers=num_workers)
elif self.action == "predict":
pass
else:
raise ValueError(f"Unrecognized action: {self.action}")
def predict(self, text):
# Tokenize
token_idxs = CustomDataset.process_input(self.tokenizer, text)
token_idxs = torch.Tensor(token_idxs).long().unsqueeze(0).to(
self.device)
attention_mask = torch.ones_like(token_idxs)
tensorOutput = self.model(token_idxs,
attention_mask=attention_mask,
is_training=False)
output = {
"food_score_preds":
tensorOutput["food_score_preds"].item(),
"food_existence_preds":
tensorOutput["food_existence_preds"].item(),
"service_score_preds":
tensorOutput["service_score_preds"].item(),
"service_existence_preds":
tensorOutput["service_existence_preds"].item(),
"price_score_preds":
tensorOutput["price_score_preds"].item(),
"price_existence_preds":
tensorOutput["price_existence_preds"].item(),
}
print("Review:", text)
print("Predict:", output)
print("=" * 40)
return output
def train(model, radio_landmark_loss, epoch_num, data_path, name, shape):
# 获取数据
train_dataset = CustomDataset(data_path)
train_loader = DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
# 设置优化方法
scheduler = paddle.optimizer.lr.PiecewiseDecay(
boundaries=[6, 14, 20],
values=[0.001, 0.0001, 0.00001, 0.000001],
verbose=True)
optimizer = paddle.optimizer.Adam(
parameters=model.parameters(),
learning_rate=scheduler,
weight_decay=paddle.regularizer.L2Decay(1e-4))
# 获取损失函数
class_loss = ClassLoss()
bbox_loss = BBoxLoss()
landmark_loss = LandmarkLoss()
for epoch in range(epoch_num):
for batch_id, (img, label, bbox,
landmark) in enumerate(train_loader()):
class_out, bbox_out, landmark_out = model(img)
cls_loss = class_loss(class_out, label)
box_loss = bbox_loss(bbox_out, bbox, label)
landmarks_loss = landmark_loss(landmark_out, landmark, label)
total_loss = radio_cls_loss * cls_loss + radio_bbox_loss * box_loss + radio_landmark_loss * landmarks_loss
total_loss.backward()
optimizer.step()
optimizer.clear_grad()
if batch_id % 100 == 0:
acc = accuracy(class_out, label)
print(
'[%s] Train epoch %d, batch %d, total_loss: %f, cls_loss: %f, box_loss: %f, landmarks_loss: %f, '
'accuracy：%f' %
(datetime.now(), epoch, batch_id, total_loss, cls_loss,
box_loss, landmarks_loss, acc))
scheduler.step()
# 保存模型
if not os.path.exists(model_path):
os.makedirs(model_path)
paddle.jit.save(layer=model,
path=os.path.join(model_path, name),
input_spec=[InputSpec(shape=shape, dtype='float32')])
radio_bbox_loss = 0.5
radio_landmark_loss = 1.0
# 训练参数值
data_path = '../dataset/48/all_data'
batch_size = 384
learning_rate = 1e-3
epoch_num = 22
model_path = '../infer_models'
# 获取P模型
model = ONet()
paddle.summary(model, input_size=(batch_size, 3, 48, 48))
# 获取数据
train_dataset = CustomDataset(data_path)
train_loader = DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
# 设置优化方法
scheduler = paddle.optimizer.lr.PiecewiseDecay(
boundaries=[6, 14, 20],
values=[0.001, 0.0001, 0.00001, 0.000001],
verbose=True)
optimizer = paddle.optimizer.Adam(
parameters=model.parameters(),
learning_rate=scheduler,
weight_decay=paddle.regularizer.L2Decay(1e-4))
# 获取损失函数