def __prepare__(self, label_json_path):
"""
Args:
label_json_path: str, label_json_path文件的路径
Returns:
model: 加载训练好权重的模型
label_dict: dict，类标名称与类标之间的对应关系
"""
prepare_model = PrepareModel()
model = prepare_model.create_model(self.model_type,
self.classes_num,
self.drop_rate,
pretrained=False,
bn_to_gn=self.bn_to_gn)
model.load_state_dict(torch.load(self.weight_path)['state_dict'])
print('Successfully Loaded from %s' % self.weight_path)
model = model.cuda()
model.eval()
# 得到类标到真实标注的映射
with open(label_json_path, 'r') as f:
label_dict = json.load(f)
return model, label_dict
def prepare(config, train_labels_number):
"""
Args:
config: 配置参数
train_labels_number: list, 某一折的[number_class0, number__class1, ...]
Returns:
optimizer: 优化器
model: 模型
criterion: 损失函数
"""
# 加载模型
prepare_model = PrepareModel()
model = prepare_model.create_model(
model_type=config.model_type,
classes_num=config.num_classes,
drop_rate=config.drop_rate,
pretrained=True,
bn_to_gn=config.bn_to_gn
)
if torch.cuda.is_available():
model = torch.nn.DataParallel(model)
model = model.cuda()
# 加载优化器
optimizer = prepare_model.create_optimizer(config.model_type, model, config)
# 加载损失函数
criterion = Loss(config.model_type, config.loss_name, config.num_classes, train_labels_number, config.beta_CB,
config.gamma)
return optimizer, model, criterion
def __prepare__(self, label_json_path):
prepare_model = PrepareModel()
model = prepare_model.create_model(self.model_type, self.classes_num, 0, pretrained=False)
model.load_state_dict(torch.load(self.weight_path)['state_dict'])
model = model.cuda()
model.eval()
# 得到类标到真实标注的映射
with open(label_json_path, 'r') as f:
label_dict = json.load(f)
return model, label_dict
def __prepare(self):
"""准备模型
"""
prepare_model = PrepareModel()
model = prepare_model.create_model('se_resnext101_32x4d',
self.classes_num,
drop_rate=0,
pretrained=False)
print('Using CPU for inference')
checkpoint = torch.load(self.model_path, map_location='cpu')
model.load_state_dict(checkpoint['state_dict'])
return model
def __init__(self, config, fold):
"""
Args:
config: 配置参数
fold: 当前为第几折
"""
self.config = config
self.fold = fold
self.epoch = config.epoch
self.num_classes = config.num_classes
self.lr_scheduler = config.lr_scheduler
print('USE LOSS: {}'.format(config.loss_name))
# 加载模型
prepare_model = PrepareModel()
self.model = prepare_model.create_local_attention_model(
model_type=config.model_type,
classes_num=self.num_classes,
last_stride=2,
droprate=0)
# 得到最新产生的权重文件
weight_path = os.path.join('checkpoints', config.model_type)
lists = os.listdir(weight_path) # 获得文件夹内所有文件
lists.sort(
key=lambda fn: os.path.getmtime(weight_path + '/' + fn)) # 排序
weight_path = os.path.join(weight_path, lists[-1], 'model_best.pth')
# 加载之前训练的权重
pretrained_dict = torch.load(weight_path)['state_dict']
model_dict = self.model.state_dict()
pretrained_dict = {
k: v
for k, v in pretrained_dict.items() if k in model_dict
} # filter out unnecessary keys
model_dict.update(pretrained_dict)
self.model.load_state_dict(model_dict)
print('Successfully Loaded from %s' % weight_path)
if torch.cuda.is_available():
self.model = torch.nn.DataParallel(self.model)
self.model = self.model.cuda()
# 加载优化器
self.optimizer = prepare_model.create_optimizer(
config.model_type, self.model, config)
# 加载衰减策略
self.exp_lr_scheduler = prepare_model.create_lr_scheduler(
self.lr_scheduler,
self.optimizer,
step_size=config.lr_step_size,
restart_step=config.restart_step,
multi_step=config.multi_step)
# 加载损失函数
self.criterion = Loss(config.model_type, config.loss_name,
self.num_classes)
# 实例化实现各种子函数的 solver 类
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.solver = Solver(self.model, self.device)
# log初始化
self.writer, self.time_stamp = self.init_log()
self.model_path = os.path.join(self.config.save_path,
self.config.model_type, self.time_stamp)
# 初始化分类度量准则类
with open("online-service/model/label_id_name.json",
'r',
encoding='utf-8') as json_file:
self.class_names = list(json.load(json_file).values())
self.classification_metric = ClassificationMetric(
self.class_names, self.model_path)
self.max_accuracy_valid = 0
def __init__(self, config, fold):
"""
Args:
config: 配置参数
fold: 当前为第几折
"""
self.config = config
self.fold = fold
self.epoch = config.epoch
self.num_classes = config.num_classes
self.lr_scheduler = config.lr_scheduler
self.save_interval = 10
self.cut_mix = config.cut_mix
self.beta = config.beta
self.cutmix_prob = config.cutmix_prob
self.auto_aug = config.auto_aug
# 多尺度
self.image_size = config.image_size
self.multi_scale = config.multi_scale
self.val_multi_scale = config.val_multi_scale
self.multi_scale_size = config.multi_scale_size
self.multi_scale_interval = config.multi_scale_interval
# 稀疏训练
self.sparsity = config.sparsity
self.sparsity_scale = config.sparsity_scale
self.penalty_type = config.penalty_type
self.selected_labels = config.selected_labels
if self.auto_aug:
print('@ Using AutoAugment.')
if self.cut_mix:
print('@ Using cut mix.')
if self.multi_scale:
print('@ Using multi scale training.')
print('@ Using LOSS: {}'.format(config.loss_name))
# 加载模型
prepare_model = PrepareModel()
self.model = prepare_model.create_model(model_type=config.model_type,
classes_num=self.num_classes,
drop_rate=config.drop_rate,
pretrained=True,
bn_to_gn=config.bn_to_gn)
if config.weight_path:
self.model = prepare_model.load_chekpoint(self.model,
config.weight_path)
# 稀疏训练
self.sparsity_train = None
if config.sparsity:
print('@ Using sparsity training.')
self.sparsity_train = Sparsity(self.model,
sparsity_scale=self.sparsity_scale,
penalty_type=self.penalty_type)
# l1正则化
self.l1_regular = config.l1_regular
self.l1_decay = config.l1_decay
if self.l1_regular:
print('@ Using l1_regular')
self.l1_reg_loss = Regularization(self.model,
weight_decay=self.l1_decay,
p=1)
if torch.cuda.is_available():
self.model = torch.nn.DataParallel(self.model)
self.model = self.model.cuda()
# 加载优化器
self.optimizer = prepare_model.create_optimizer(
config.model_type, self.model, config)
# 加载衰减策略
self.exp_lr_scheduler = prepare_model.create_lr_scheduler(
self.lr_scheduler,
self.optimizer,
step_size=config.lr_step_size,
restart_step=config.restart_step,
multi_step=config.multi_step,
warmup=config.warmup,
multiplier=config.multiplier,
warmup_epoch=config.warmup_epoch,
delay_epoch=config.delay_epoch)
# 加载损失函数
self.criterion = Loss(config.model_type, config.loss_name,
self.num_classes)
# 实例化实现各种子函数的 solver 类
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.solver = Solver(self.model, self.device)
# log初始化
self.writer, self.time_stamp = self.init_log()
self.model_path = os.path.join(self.config.save_path,
self.config.model_type, self.time_stamp)
# 初始化分类度量准则类
with open("online-service/model/label_id_name.json",
'r',
encoding='utf-8') as json_file:
self.class_names = list(json.load(json_file).values())
self.classification_metric = ClassificationMetric(
self.class_names, self.model_path)
self.max_accuracy_valid = 0
def __init__(self, config, fold, train_labels_number):
"""
Args:
config: 配置参数
fold: int, 当前为第几折
train_labels_number: list, 某一折的[number_class0, number__class1, ...]
"""
self.config = config
self.fold = fold
self.epoch = config.epoch
self.num_classes = config.num_classes
self.lr_scheduler = config.lr_scheduler
self.save_interval = 100
self.cut_mix = config.cut_mix
self.beta = config.beta
self.cutmix_prob = config.cutmix_prob
self.image_size = config.image_size
self.multi_scale = config.multi_scale
self.multi_scale_size = config.multi_scale_size
self.multi_scale_interval = config.multi_scale_interval
if self.cut_mix:
print('Using cut mix.')
if self.multi_scale:
print('Using multi scale training.')
print('USE LOSS: {}'.format(config.loss_name))
# 加载模型
prepare_model = PrepareModel()
self.model = prepare_model.create_model(model_type=config.model_type,
classes_num=self.num_classes,
drop_rate=config.drop_rate,
pretrained=True,
bn_to_gn=config.bn_to_gn)
if torch.cuda.is_available():
self.model = torch.nn.DataParallel(self.model)
self.model = self.model.cuda()
# 加载优化器
self.optimizer = prepare_model.create_optimizer(
config.model_type, self.model, config)
# 加载衰减策略
self.exp_lr_scheduler = prepare_model.create_lr_scheduler(
self.lr_scheduler,
self.optimizer,
step_size=config.lr_step_size,
restart_step=config.restart_step,
multi_step=config.multi_step)
# 加载损失函数
self.criterion = Loss(config.model_type, config.loss_name,
self.num_classes, train_labels_number,
config.beta_CB, config.gamma)
# 实例化实现各种子函数的 solver 类
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.solver = Solver(self.model, self.device)
if config.restore:
weight_path = os.path.join('checkpoints', config.model_type)
if config.restore == 'last':
lists = os.listdir(weight_path) # 获得文件夹内所有文件
lists.sort(key=lambda fn: os.path.getmtime(weight_path + '/' +
fn)) # 按照最近修改时间排序
weight_path = os.path.join(weight_path, lists[-1],
'model_best.pth')
else:
weight_path = os.path.join(weight_path, config.restore,
'model_best.pth')
self.solver.load_checkpoint(weight_path)
# log初始化
self.writer, self.time_stamp = self.init_log()
self.model_path = os.path.join(self.config.train_url,
self.config.model_type, self.time_stamp)
# 初始化分类度量准则类
with open("online-service/model/label_id_name.json",
'r',
encoding='utf-8') as json_file:
self.class_names = list(json.load(json_file).values())
self.classification_metric = ClassificationMetric(self.class_names,
self.model_path,
text_flag=0)
self.max_accuracy_valid = 0
def __init__(self, config, fold, train_labels_number):
"""
Args:
config: 配置参数
fold: int, 当前为第几折
train_labels_number: list, 某一折的[number_class0, number__class1, ...]
"""
self.config = config
self.fold = fold
self.epoch = config.epoch
self.num_classes = config.num_classes
self.lr_scheduler = config.lr_scheduler
self.cut_mix = config.cut_mix
self.beta = config.beta
self.cutmix_prob = config.cutmix_prob
self.train_url = config.train_url
self.bucket_name = config.bucket_name
self.image_size = config.image_size
self.multi_scale = config.multi_scale
self.multi_scale_size = config.multi_scale_size
self.multi_scale_interval = config.multi_scale_interval
if self.cut_mix:
print('Using cut mix.')
if self.multi_scale:
print('Using multi scale training.')
print('USE LOSS: {}'.format(config.loss_name))
# 拷贝预训练权重
print("=> using pre-trained model '{}'".format(config.model_type))
if not mox.file.exists(
'/home/work/.cache/torch/checkpoints/se_resnext101_32x4d-3b2fe3d8.pth'
):
mox.file.copy(
os.path.join(self.bucket_name,
'model_zoo/se_resnext101_32x4d-3b2fe3d8.pth'),
'/home/work/.cache/torch/checkpoints/se_resnext101_32x4d-3b2fe3d8.pth'
)
print(
'copy pre-trained model from OBS to: %s success' %
(os.path.abspath(
'/home/work/.cache/torch/checkpoints/se_resnext101_32x4d-3b2fe3d8.pth'
)))
else:
print('use exist pre-trained model at: %s' % (os.path.abspath(
'/home/work/.cache/torch/checkpoints/se_resnext101_32x4d-3b2fe3d8.pth'
)))
# 拷贝预训练权重
print("=> using pre-trained model '{}'".format(config.model_type))
if not mox.file.exists(
'/home/work/.cache/torch/checkpoints/efficientnet-b5-b6417697.pth'
):
mox.file.copy(
os.path.join(self.bucket_name,
'model_zoo/efficientnet-b5-b6417697.pth'),
'/home/work/.cache/torch/checkpoints/efficientnet-b5-b6417697.pth'
)
print(
'copy pre-trained model from OBS to: %s success' %
(os.path.abspath(
'/home/work/.cache/torch/checkpoints/efficientnet-b5-b6417697.pth'
)))
else:
print('use exist pre-trained model at: %s' % (os.path.abspath(
'/home/work/.cache/torch/checkpoints/efficientnet-b5-b6417697.pth'
)))
# 加载模型
prepare_model = PrepareModel()
self.model = prepare_model.create_model(model_type=config.model_type,
classes_num=self.num_classes,
drop_rate=config.drop_rate,
pretrained=True,
bn_to_gn=config.bn_to_gn)
self.model = torch.nn.DataParallel(self.model).cuda()
# 加载优化器
self.optimizer = prepare_model.create_optimizer(
config.model_type, self.model, config)
# 加载衰减策略
self.exp_lr_scheduler = prepare_model.create_lr_scheduler(
self.lr_scheduler,
self.optimizer,
step_size=config.lr_step_size,
restart_step=config.restart_step,
multi_step=config.multi_step)
# 加载损失函数
self.criterion = Loss(config.model_type, config.loss_name,
self.num_classes, train_labels_number,
config.beta_CB, config.gamma)
# 实例化实现各种子函数的 solver 类
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.solver = Solver(self.model, self.device)
if config.restore:
weight_path = os.path.join('checkpoints', config.model_type)
if config.restore == 'last':
lists = os.listdir(weight_path) # 获得文件夹内所有文件
lists.sort(key=lambda fn: os.path.getmtime(weight_path + '/' +
fn)) # 按照最近修改时间排序
weight_path = os.path.join(weight_path, lists[-1],
'model_best.pth')
else:
weight_path = os.path.join(weight_path, config.restore,
'model_best.pth')
self.solver.load_checkpoint(weight_path)
# log初始化
self.writer, self.time_stamp = self.init_log()
self.model_path = os.path.join(self.config.train_local,
self.config.model_type, self.time_stamp)
# 初始化分类度量准则类
with open(config.local_data_root + 'label_id_name.json',
'r',
encoding='utf-8') as json_file:
self.class_names = list(json.load(json_file).values())
self.classification_metric = ClassificationMetric(self.class_names,
self.model_path,
text_flag=0)
self.max_accuracy_valid = 0