def get_target_data(name, split_id, data_dir, height, width,
batch_size, workers, combine_trainval):
if len(name) <= 1:
root = osp.join(data_dir, name[0])
dataset = datasets.create(name[0], root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
# normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
# std=[1, 1, 1])
train_set = dataset.trainval if combine_trainval else dataset.train
num_classes = (dataset.num_trainval_ids if combine_trainval
else dataset.num_train_ids)
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(
Preprocessor(train_set, root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=True, pin_memory=True, drop_last=True)
val_loader = DataLoader(
Preprocessor(dataset.val, root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
test_loader = DataLoader(
Preprocessor(list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir, transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
return dataset, num_classes, train_loader, val_loader, test_loader
def get_data(name, data_dir, height, width, batch_size, workers):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
test_transformer = T.Compose([
T.Resize((height, width), interpolation=3),
T.ToTensor(),
normalizer
])
test_loader = DataLoader(
Preprocessor(list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir, transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
return dataset, test_loader
def get_data(args):
(name, split_id, data_dir, height, width, batch_size, num_instances,
workers, combine_trainval) = (
args.dataset,
args.split,
args.data_dir,
args.height,
args.width,
args.batch_size,
args.num_instances,
args.workers,
args.combine_trainval,
)
pin_memory = args.pin_mem
name_val = args.dataset_val or args.dataset
npy = args.has_npy
rand_ratio = args.random_ratio
if isinstance(name, list):
dataset = datasets.creates(name, split_id=split_id)
else:
dataset = datasets.create(
name,
split_id=split_id,
)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
test_loader = DataLoader(
Preprocessor(dataset.query, transform=test_transformer, has_npy=npy),
batch_size=batch_size, # * 2
num_workers=workers,
shuffle=False,
pin_memory=False)
return test_loader
def main(args):
cudnn.benchmark = True
cudnn.enabled = True
# import warnings
#
# warnings.filterwarnings("ignore")
save_path = args.logs_dir
sys.stdout = Logger(osp.join(args.logs_dir, 'log'+ str(args.merge_percent)+ time.strftime(".%m.%d_%H.%M.%S") + '.txt'))
resume_step, ckpt_file = -1, ''
if args.resume:
resume_step, ckpt_file = resume(args)
# get all unlabeled data for training
dataset_all = datasets.create(args.dataset, osp.join(args.data_dir, args.dataset))
new_train_data, cluster_id_labels = change_to_unlabel(dataset_all)
num_train_ids = len(np.unique(np.array(cluster_id_labels)))
nums_to_merge = int(num_train_ids * args.merge_percent)
BuMain = Bottom_up(model_name=args.arch, batch_size=args.batch_size,
num_classes=num_train_ids,
dataset=dataset_all,
u_data=new_train_data, save_path=args.logs_dir, max_frames=args.max_frames,
embeding_fea_size=args.fea, initial_steps=args.initial_steps, load_path=args.load_path)
print("==========\nArgs:{}\n==========".format(args))
for step in range(int(1/args.merge_percent)-1):
# if step < resume_step:
# continue
print('step: ', step)
BuMain.train(new_train_data, step, loss=args.loss) if step != resume_step else BuMain.resume(ckpt_file, step)
# if step > 5:
# if step//2==0:
#BuMain.evaluate(dataset_all.query, dataset_all.gallery, step)
# get new train data for the next iteration
print('----------------------------------------bottom-up clustering------------------------------------------------')
cluster_id_labels, new_train_data = BuMain.get_new_train_data(cluster_id_labels, nums_to_merge, size_penalty=args.size_penalty,
lambda_part=args.lambda_part)
print('\n\n')
def get_data(name, split_id, data_dir, height, width, batch_size, num_instances,
workers, combine_trainval):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id, download=False)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval if combine_trainval else dataset.train
num_classes = (dataset.num_trainval_ids if combine_trainval
else dataset.num_train_ids)
train_transformer = T.Compose([
T.RectScale(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(
Attribute_Preprocessor(train_set, root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size, num_workers=workers,
sampler=RandomIdentityAttributeSampler(train_set, num_instances),
pin_memory=True, drop_last=True)
test_loader = DataLoader(
Attribute_Preprocessor(list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir, transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
return dataset, num_classes, train_loader, test_loader
def get_data(name, data_dir, height, width, ratios, batch_size, workers):
root = osp.join(data_dir, name)
root = data_dir
dataset = datasets.create(name, root)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
num_classes = dataset.num_train_ids + 1 # plus 1 more label for the zero-padded feature
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
query_transformers = []
for ratio in ratios:
query_transformers.append(T.Compose([
T.ContVerticalCropDiscret(height, width, ratio),
T.ToTensor(),
normalizer]))
query_loaders = []
for query_transformer in query_transformers:
query_loaders.append(DataLoader(
Preprocessor(dataset.query, root=osp.join(dataset.images_dir,dataset.query_path),
transform=query_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True))
gallery_loader = DataLoader(
Preprocessor(dataset.gallery, root=osp.join(dataset.images_dir,dataset.gallery_path),
transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
return dataset, num_classes, query_loaders, gallery_loader
def evaluate(args):
# Settings
exp_dir = './logs/{}/{}'.format(args.dataset, args.exp)
target_epoch = args.epoch
batch_size = args.batchsize
gpu_ids = args.gpus
set_paths('paths')
os.environ['CUDA_VISIBLE_DEVICES'] = gpu_ids
args = json.load(open(osp.join(exp_dir, "args.json"), "r"))
# Load data
t = T.Compose([
T.RectScale(args['height'], args['width']),
T.CenterCrop((args['crop_height'], args['crop_width'])),
T.ToTensor(),
T.RGB_to_BGR(),
T.NormalizeBy(255),
])
dataset = datasets.create(args['dataset'],
'data/{}'.format(args['dataset']))
dataset_ = Preprocessor(list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=t)
dataloader = DataLoader(dataset_, batch_size=batch_size, shuffle=False)
# Load model
model = models.create(args['arch'],
dilation=args['dilation'],
use_relu=args['use_relu'],
initialize=False).cuda()
weight_file = osp.join(exp_dir, 'epoch_{}.pth.tar'.format(target_epoch))
model.load(load_checkpoint(weight_file))
model.eval()
# Evaluate
evaluator = Evaluator(model)
evaluator.evaluate(dataloader, dataset.query, dataset.gallery)
def get_data(name, data_dir, height, width, batch_size, workers, pose_aug):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root)
# use combined trainval set for training as default
preprocessor = Preprocessor(dataset.trainval,
root=dataset.images_dir,
with_pose=True,
pose_root=dataset.poses_dir,
pid_imgs=dataset.trainval_query,
height=height,
width=width,
pose_aug=pose_aug)
train_loader = DataLoader(preprocessor,
sampler=RandomPairSampler(dataset.trainval,
neg_pos_ratio=3),
batch_size=batch_size,
num_workers=workers,
pin_memory=False)
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=False)
return dataset, train_loader, test_loader
def get_data(name, data_dir, height, width, batch_size, workers):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, num_val=0.1)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
# use all training and validation images in target dataset
train_set = dataset.trainval
num_classes = dataset.num_trainval_ids
transformer = T.Compose([
T.Resize((height, width)),
T.ToTensor(),
normalizer,
])
extfeat_loader = DataLoader(Preprocessor(train_set,
root=dataset.images_dir,
transform=transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
return dataset, num_classes, extfeat_loader, test_loader
def get_data(name, split_id, data_dir, height, width, batch_size, workers):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
num_classes = dataset.num_trainval_ids
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
return dataset, num_classes, test_loader
def get_data(args):
(name, split_id,
data_dir, height, width,
batch_size, num_instances,
workers, combine_trainval) = (
args.dataset, args.split,
args.data_dir, args.height, args.width,
args.batch_size, args.num_instances,
args.workers, args.combine_trainval,)
pin_memory = args.pin_mem
name_val = args.dataset_val or args.dataset
npy = args.has_npy
rand_ratio = args.random_ratio
if isinstance(name, list):
dataset = datasets.creates(name, split_id=split_id,
cuhk03_classic_split=args.cu03_classic)
else:
dataset = datasets.create(name, split_id=split_id,
cuhk03_classic_split=args.cu03_classic)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
num_classes = dataset.num_train_pids
train_transformer = T.Compose([
T.RandomCropFlip(height, width, area=args.area),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
dop_info = DopInfo(num_classes)
print('dop info and its id are', dop_info)
new_train = []
for img_paths, pid, camid in dataset.train:
for img_path in img_paths:
new_train.append((img_path, pid, camid))
train_loader = DataLoader(
Preprocessor(new_train,
transform=train_transformer,
has_npy=npy),
batch_size=batch_size, num_workers=workers,
sampler=RandomIdentityWeightedSampler(
new_train, num_instances,
batch_size=batch_size,
rand_ratio=rand_ratio,
dop_info=dop_info,
),
# shuffle=True,
pin_memory=pin_memory, drop_last=True)
query_loader = DataLoader(
VideoDataset(dataset.query, seq_len=args.seq_len, sample='evenly', transform=test_transformer),
batch_size=args.test_batch_size, shuffle=False, num_workers=args.workers,
pin_memory=False, drop_last=False,
)
# print('this gallery', dataset.gallery)
gallery_loader = DataLoader(
VideoDataset(dataset.gallery, seq_len=args.seq_len, sample='evenly', transform=test_transformer),
batch_size=args.test_batch_size, shuffle=False, num_workers=args.workers,
pin_memory=False, drop_last=False,
)
return dataset, num_classes, train_loader, dop_info, query_loader, gallery_loader
def get_data(name, split_id, data_dir, height, width, batch_size, num_instances,
workers, combine_trainval):
if len(name)>1:
root = osp.join(data_dir)
dataset = datasets.create('merged', root, name, split_id=split_id)
else:
root = osp.join(data_dir, name[0])
dataset = datasets.create(name[0], root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval if combine_trainval else dataset.train
num_classes = (dataset.num_trainval_ids if combine_trainval
else dataset.num_train_ids)
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
# train_transformer = T.Compose([
# T.ThesisCrop(height, width),
# T.RandomHorizontalFlip(),
# T.ToTensor(),
# normalizer,
# ])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(
Preprocessor(train_set, root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size, num_workers=workers,
sampler=RandomIdentitySampler(train_set, num_instances),
pin_memory=True, drop_last=True)
val_loader = DataLoader(
Preprocessor(dataset.val, root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
if len(name)>1:
test_loader = defaultdict()
for dataset_name in name:
test_loader[dataset_name] = DataLoader(
Preprocessor(list(set(dataset.query[dataset_name]) |
set(dataset.gallery[dataset_name])),
root=dataset.images_dir, transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
else:
test_loader = DataLoader(
Preprocessor(list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir, transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
return dataset, num_classes, train_loader, val_loader, test_loader
def get_data(name, split_id, data_dir, height, width, batch_size, workers, combine_trainval, loss_mode='binary_loss', instances_num=4):
root = osp.join(data_dir, name)
dataset = create(name, root, split_id=split_id,num_frame=8)
#dataset = data_manager.init_vidreid_train_dataset(root=root, num_frame=1, name=name)
dataset_test = data_manager.init_vidreid_test_dataset(root=root, num_frame=3, split_id=split_id,name=name)
#dataset = create(name, root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.train
num_classes =dataset.num_train_pids
train_transformer = T.Compose([T.RandomSizedRectCrop(height, width), T.RandomSizedEarser(),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
if loss_mode == 'oim':
train_loader = DataLoader(
Preprocessor(train_set, root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=True, pin_memory=True, drop_last=True)
elif loss_mode == 'binary':
train_loader = DataLoader(Preprocessor(train_set, root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size, num_workers=workers,
sampler=RandomPairSampler(train_set), pin_memory=True)
elif loss_mode == 'triplet':
train_loader = DataLoader(
Preprocessor(train_set, root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size, num_workers=workers,
sampler=RandomIdentitySampler(train_set),
pin_memory=True, drop_last=True)
elif loss_mode == 'crfloss':
train_loader = DataLoader(
Preprocessor(train_set,
transform=train_transformer),
batch_size=batch_size, num_workers=workers,
sampler=RandomMultipleGallerySampler(train_set, instances_num),
pin_memory=True, drop_last=True)
'''train_loader = DataLoader(
Preprocessor(train_set, root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size, num_workers=workers,
sampler=RandomMultipleGallerySampler(train_set, instances_num),
pin_memory=True, drop_last=True)'''
else:
raise ValueError('NO such loss function')
query_loader = DataLoader(
VideoDataset(dataset_test.query, seq_len=16, sample='all', transform=test_transformer),
batch_size=1, shuffle=False, num_workers=workers,
pin_memory=True, drop_last=False,
)
gallery_loader = DataLoader(
VideoDataset(dataset_test.gallery, seq_len=16, sample='all', transform=test_transformer),
batch_size=1, shuffle=False, num_workers=workers,
pin_memory=True, drop_last=False,
)
return dataset, dataset_test, num_classes, train_loader, query_loader, gallery_loader
def main(args):
cudnn.benchmark = True
cudnn.enabled = True
# get all the labeled and unlabeled data for training
dataset_all = datasets.create(args.dataset,
osp.join(args.data_dir, args.dataset))
one_shot, u_data = get_one_shot_in_cam1(
dataset_all,
load_path="./examples/oneshot_{}_used_in_paper.pickle".format(
dataset_all.name))
def sampleing_number_curve(step): # p = 1时就是线性曲线
yr = min(math.floor(pow(step * len(u_data) / args.total_step, args.p)),
len(u_data))
return yr
def train_epoch(yr): #只有训练reid的时候采用
times = math.ceil((len(one_shot) + yr) / len(one_shot))
ep_k = max(math.floor(args.epoch / times), 1)
return ep_k, times
Ep = [] # 经验
AE = [] # 辅助经验
PE = [] # 实践经验
# 省略掉oneshot训练部分
# 输出实验信息
print(
"{}/{} is training with {}, the max_frames is {}, and will be saved to {}"
.format(args.exp_name, args.exp_order, args.dataset, args.max_frames,
args.logs_dir))
# 输出超参信息
print("parameters are setted as follows")
print("\ttotal_step:\t{}".format(args.total_step))
print("\tepoch:\t{}".format(args.epoch))
print("\tstep_size:\t{}".format(args.step_size))
print("\tbatch_size:\t{}".format(args.batch_size))
# 指定输出文件
# 第三部分要说明关键参数的设定
reid_path = osp.join(args.logs_dir, args.dataset, args.exp_name,
args.exp_order)
sys.stdout = Logger(
osp.join(reid_path, 'log' + time.strftime(".%m_%d_%H-%M-%S") + '.txt'))
P_reid = codecs.open(osp.join(reid_path, 'P_reid.txt'), mode='a')
S_file = codecs.open(osp.join(reid_path, 'S.txt'), mode='a') #记录选择准确率
L_file = codecs.open(osp.join(reid_path, 'L.txt'), mode='a') # 记录标签估计准确率
time_file = codecs.open(osp.join(reid_path, 'time.txt'), mode='a')
P_tagper = codecs.open(osp.join(reid_path, "P_tagper.txt"), mode='a')
# initial the EUG algorithm
# 注意不要去破坏公共部分的代码
reid = EUG(model_name=args.arch,
batch_size=args.batch_size,
mode=args.mode,
num_classes=dataset_all.num_train_ids,
data_dir=dataset_all.images_dir,
l_data=one_shot,
u_data=u_data,
save_path=reid_path,
max_frames=args.max_frames)
reid.resume(osp.join(reid_path, 'Dissimilarity_step_0-0.ckpt'), 0)
# 初始化循环模式
iter_mode = 2 # 2双循环模式 1单循环模式
step_time_list = []
# 开始循环
last_train_times = 0
for step in range(1, args.total_step): #10 h和11 的时候少循环1次
# 获取采样数量
num_reid = sampleing_number_curve(step)
num_tagper = min(math.ceil(num_reid * args.baba), len(u_data))
train_ep, train_times = train_epoch(num_reid)
# 克隆种子得到标签器
stage_time = 0
print(
"### step {} is training: num_reid={},num_tagper={}, train_ep={},train_times={}"
.format(step, num_reid, num_tagper, train_ep, train_times))
if iter_mode == 2:
time1 = time.time()
tagper = EUG(model_name=args.arch,
batch_size=args.batch_size,
mode=args.mode,
num_classes=dataset_all.num_train_ids,
data_dir=dataset_all.images_dir,
l_data=one_shot,
u_data=u_data,
save_path=reid_path,
max_frames=args.max_frames)
tagper.resume(
osp.join(
reid_path, 'Dissimilarity_step_{}-{}.ckpt'.format(
step - 1, math.ceil(last_train_times))), step - 1)
last_train_times = train_times - 1
# 实践
PE_pred_y, PE_pred_score, PE_label_pre = reid.estimate_label_atm6(
u_data, Ep, one_shot) # 针对u_data进行标签估计
selected_idx_RR = reid.select_top_data(PE_pred_score, num_reid)
select_pre_R = reid.get_select_pre(selected_idx_RR, PE_pred_y,
u_data)
selected_idx_RT = reid.select_top_data(PE_pred_score, num_tagper)
select_pre_T = reid.get_select_pre(selected_idx_RT, PE_pred_y,
u_data)
# 训练tagper
new_train_data = tagper.generate_new_train_data_only(
selected_idx_RT, PE_pred_y,
u_data) # 这个选择准确率应该是和前面的label_pre是一样的.
train_tagper_data = one_shot + new_train_data
tagper.train(train_tagper_data,
step,
tagper=1,
epochs=args.epoch,
step_size=args.step_size,
init_lr=0.1)
time2 = time.time()
# 性能评估
mAP, top1, top5, top10, top20 = tagper.evaluate(
dataset_all.query, dataset_all.gallery) if args.ev else (0, 0,
0, 0,
0)
P_tagper.write(
"step:{} mAP:{:.2%} top1:{:.2%} top5:{:.2%} top10:{:.2%} top20:{:.2%}\n"
.format(int(step), mAP, top1, top5, top10, top20))
print(
"step:{} mAP:{:.2%} top1:{:.2%} top5:{:.2%} top10:{:.2%} top20:{:.2%}\n"
.format(int(step), mAP, top1, top5, top10, top20))
time3 = time.time()
AE_pred_y, AE_pred_score, AE_label_pre = tagper.estimate_label_atm6(
u_data, Ep, one_shot) # 针对u_data进行标签估计
selected_idx_TR = reid.select_top_data(AE_pred_score, num_reid)
AE_select_pre = reid.get_select_pre(selected_idx_TR, AE_pred_y,
u_data)
#下面需要进行知识融合 KF
AEs = normalization(AE_pred_score)
PEs = normalization(PE_pred_score)
KF = np.array(
[PE_pred_y[i] == AE_pred_y[i] for i in range(len(u_data))])
KF_score = np.array([
KF[i] * (PEs[i] + AEs[i]) + (1 - KF[i]) * abs(PEs[i] - AEs[i])
for i in range(len(u_data))
])
KF_label = np.array([
KF[i] * PE_pred_y[i] + (1 - KF[i]) *
(PE_pred_y[i] if PEs[i] >= AEs[i] else AE_pred_y[i])
for i in range(len(u_data))
])
u_label = np.array([label for _, label, _, _ in u_data])
is_label_right = np.array([
1 if u_label[i] == KF_label[i] else 0
for i in range(len(u_label))
])
KF_label_pre = sum(is_label_right) / len(u_label)
#获取Ep
selected_idx_Ep = tagper.select_top_data(KF_score, num_reid)
Ep, Ep_select_pre = tagper.move_unlabel_to_label_cpu(
selected_idx_Ep, KF_label, u_data)
L_file.write(
"step:{} PE_labele_pre:{:.2%} AE_label_pre:{:.2%} KF_label_pre:{:.2%}\n"
.format(step, PE_label_pre, AE_label_pre, KF_label_pre))
print(
"step:{} PE_labele_pre:{:.2%} AE_label_pre:{:.2%} KF_label_pre:{:.2%}\n"
.format(step, PE_label_pre, AE_label_pre, KF_label_pre))
S_file.write(
"step:{} num_reid:{} num_tagper:{} select_pre_R:{:.2%} select_pre_T:{:.2%} AE_select_pre:{:.2%} Ep_select_pre:{:.2%}\n"
.format(step, num_reid, num_tagper, select_pre_R, select_pre_T,
AE_select_pre, Ep_select_pre))
print(
"step:{} num_reid:{} num_tagper:{} select_pre_R:{:.2%} select_pre_T:{:.2%} AE_select_pre:{:.2%} Ep_select_pre:{:.2%}\n"
.format(step, num_reid, num_tagper, select_pre_R, select_pre_T,
AE_select_pre, Ep_select_pre))
time4 = time.time()
stage_time = time4 - time3 + time2 - time1
elif iter_mode == 1:
time1 = time.time()
PE_pred_y, PE_pred_score, PE_label_pre = reid.estimate_label_atm3(
u_data, Ep, one_shot) # 针对u_data进行标签估计
selected_idx_RR = reid.select_top_data(PE_pred_score, num_reid)
Ep, Ep_select_pre = reid.move_unlabel_to_label_cpu(
selected_idx_RR, PE_pred_y, u_data)
P_reid.write(
"step:{} num_reid:{} PE_label_pre:{:.2%} Ep_select_pre:{:.2%}\n"
.format(step, num_reid, PE_label_pre,
Ep_select_pre)) # Ep_select_pre 和select_pre_R 是一样的.
print(
"step:{} num_reid:{} PE_label_pre:{:.2%} Ep_select_pre:{:.2%}\n"
.format(step, num_reid, PE_label_pre,
Ep_select_pre)) # Ep_select_pre 和select_pre_R 是一样的.
time2 = time.time()
stage_time = time2 - time1
# 训练种子
time1 = time.time()
train_seed_data = Ep + one_shot
for i in range(train_times):
reid.train_atm06(train_seed_data,
step,
i,
epochs=train_ep,
step_size=args.step_size,
init_lr=0.1)
mAP, top1, top5, top10, top20 = reid.evaluate(
dataset_all.query, dataset_all.gallery) if args.ev else (0, 0,
0, 0,
0)
P_reid.write(
"step:{} times:{} mAP:{:.2%} top1:{:.2%} top5:{:.2%} top10:{:.2%} top20:{:.2%}\n"
.format(int(step), i, mAP, top1, top5, top10, top20))
print(
"step:{} times:{} mAP:{:.2%} top1:{:.2%} top5:{:.2%} top10:{:.2%} top20:{:.2%}\n"
.format(int(step), i, mAP, top1, top5, top10, top20))
if num_tagper == len(u_data):
iter_mode = 1 # 进入单循环模式.
time2 = time.time()
train_time = time2 - time1
step_time = stage_time + train_time
step_time_list.append(step_time)
time_file.write(
"step:{} stage_time:{} train_time:{} step_time:{}\n".format(
int(step), stage_time, train_time, step_time))
print("stage_time = %02d:%02d:%02.6f" % (changetoHSM(stage_time)))
print("train_time = %02d:%02d:%02.6f" % (changetoHSM(train_time)))
print("step_time = %02d:%02d:%02.6f" % (changetoHSM(step_time)))
all_time = sum(step_time_list)
print("training is over ,cost %02d:%02d:%02.6f" % (changetoHSM(all_time)))
P_reid.close()
P_tagper.close()
S_file.close()
L_file.close()
time_file.close()
def main(args):
# gd = gif_drawer2()
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
print("game begin!")
cudnn.benchmark = True
cudnn.enabled = True
save_path = args.logs_dir
total_step = math.ceil(math.pow(
(100 / args.EF), (1 / args.q))) + 1 # 这里应该取上限或者 +2 多一轮进行one-shot训练的
print("total_step:{}".format(total_step))
sys.stdout = Logger(
osp.join(
args.logs_dir, 'log' + str(args.EF) + "_" + str(args.q) +
time.strftime(".%m_%d_%H-%M-%S") + '.txt'))
data_file = codecs.open(
osp.join(
args.logs_dir, 'data' + str(args.EF) + "_" + str(args.q) +
time.strftime(".%m_%d_%H-%M-%S") + '.txt'), 'a')
time_file = codecs.open(
osp.join(
args.logs_dir, 'time' + str(args.EF) + "_" + str(args.q) +
time.strftime(".%m_%d_%H-%M-%S") + '.txt'), 'a')
# get all the labeled and unlabeled data for training
dataset_all = datasets.create(args.dataset,
osp.join(args.data_dir, args.dataset))
num_all_examples = len(dataset_all.train)
l_data, u_data = get_one_shot_in_cam1(
dataset_all,
load_path="./examples/oneshot_{}_used_in_paper.pickle".format(
dataset_all.name))
resume_step, ckpt_file = -1, ''
if args.resume: # 重新训练的时候用
resume_step, ckpt_file = resume(args)
# initial the EUG algorithm
eug = EUG(model_name=args.arch,
batch_size=args.batch_size,
mode=args.mode,
num_classes=dataset_all.num_train_ids,
data_dir=dataset_all.images_dir,
l_data=l_data,
u_data=u_data,
save_path=args.logs_dir,
max_frames=args.max_frames)
nums_to_select = 0
new_train_data = l_data
step = 0
# to_list = []
step_size = []
base_step = args.bs
top_list = [] # top1
isout = 0 #用来标记是否应该结束训练
# data_file = open("")
start_time = time.time()
while (not isout):
onetimeS = time.time()
print(
"This is running {} with EF ={}%, q = {} step {}:\t Nums_been_selected {}, \t Logs-dir {}"
.format(args.mode, args.EF, args.q, step, nums_to_select,
save_path))
onetime_trainS = time.time()
eug.train(new_train_data, step, epochs=20, step_size=15,
init_lr=0.1) if step != resume_step else eug.resume(
ckpt_file, step)
onetime_trainE = time.time()
onetime_train = onetime_trainE - onetime_trainS
h, m, s = changetoHSM(onetime_train)
print("joselyn msg: traning is over,cost %02d:%02d:%02.6f" % (h, m, s))
# evluate
onetime_evaluateS = time.time()
mAP, top1, top5, top10, top20 = eug.evaluate(dataset_all.query,
dataset_all.gallery)
onetime_evaluateE = time.time()
onetime_evaluate = onetime_evaluateE - onetime_evaluateS
h, m, s = changetoHSM(onetime_evaluate)
step_size.append(nums_to_select)
if nums_to_select == len(u_data):
isout = 1
print("joselyn msg: evaluate is over,cost %02d:%02d:%02.6f" %
(h, m, s))
# pseudo-label and confidence sc
nums_to_select = min(
math.ceil(
len(u_data) * math.pow((step + 1), args.q) * args.EF / 100),
len(u_data)) # 指数渐进策略
onetime_estimateS = time.time()
pred_y, pred_score, label_pre, id_num = eug.estimate_label()
onetime_estimateE = time.time()
onetime_estimate = onetime_estimateE - onetime_estimateS
h, m, s = changetoHSM(onetime_estimate)
print("joselyn msg: estimate labels is over,cost %02d:%02d:%02.6f" %
(h, m, s))
# select data
# selected_idx = eug.select_top_data(pred_score, nums_to_select)
# NLVM
selected_idx = eug.select_top_data_NLVM(pred_score, nums_to_select)
# selected_idx = eug.select_top_data(pred_score, nums_to_select,id_num,pred_y,u_data) #for 同比
print("joselyn msg: select top data is over")
# add new data
new_train_data, select_pre = eug.generate_new_train_data(
selected_idx, pred_y)
# new_train_data,select_pre = eug.generate_new_train_data(selected_idx, pred_y) #for 同比
print("joselyn msg: generate new train data is over")
# gd.draw(step_size[step]/len(u_data),top1,mAP,label_pre,select_pre)
onetimeE = time.time()
onetime = onetimeE - onetimeS
h, m, s = changetoHSM(onetime)
data_file.write(
"step:{} top1:{:.2%} nums_selected:{} selected_percent:{:.2%} mAP:{:.2%} label_pre:{:.2%} select_pre:{:.2%}\n"
.format(int(step), top1, step_size[step],
step_size[step] / len(u_data), mAP, label_pre, select_pre))
time_file.write(
"step:{} traning:{:.8} evaluate:{:.8} estimate:{:.8} onetime:{:.8}\n"
.format(int(step), onetime_train, onetime_evaluate,
onetime_estimate, onetime))
print(
"step:{} top1:{:.2%} nums_selected:{} selected_percent:{:.2%} mAP:{:.2%} label_pre:{:.2%} select_pre:{:.2%}"
.format(int(step), top1, step_size[step],
step_size[step] / len(u_data), mAP, label_pre, select_pre))
print("onetime cost %02d:%02d:%02.6f" % (h, m, s))
step = step + 1
data_file.close()
time_file.close()
end_time = time.time()
alltime = end_time - start_time
h, m, s = changetoHSM(alltime)
print("alltime cost %02d:%02d:%02.6f" % (h, m, s))
def get_data(name, split_id, data_dir, height, width, batch_size,
num_instances, workers, combine_trainval):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval if combine_trainval else dataset.train
num_classes = (dataset.num_trainval_ids
if combine_trainval else dataset.num_train_ids)
query = dataset.query
query_ids = [pid for _, pid, _ in query]
query_fnames = [fname for fname, _, _ in query]
query_cams = [cam for _, _, cam in query]
query_ids_unique = list(set(query_ids))
query_fnames_new, query_ids_new, query_cams_new = [], [], []
gallery_fnames_new, gallery_ids_new, gallery_cams_new = [], [], []
for k in query_ids_unique:
idx = query_ids.index(k)
query_ids_new.append(k)
query_fnames_new.append(query_fnames[idx])
query_cams_new.append(query_cams[idx])
new_idx = idx + 1
while query_cams[idx] == query_cams[new_idx]:
new_idx += 1
gallery_ids_new.append(k)
gallery_fnames_new.append(query_fnames[new_idx])
gallery_cams_new.append(query_cams[new_idx])
query_num = len(query_ids_unique)
query_test_num = 100 # 2 GPU
split_num = query_num // query_test_num
test_set = []
tmp = []
for k in range(split_num):
for i in range(2):
for j in range(k * query_test_num, (k + 1) * query_test_num):
if i == 0:
tmp.extend((query_fnames_new[j], query_ids_new[j],
query_cams_new[j]))
else:
tmp.extend((gallery_fnames_new[j], gallery_ids_new[j],
gallery_cams_new[j]))
test_set.append(tmp)
tmp = []
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(Preprocessor(train_set,
root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
sampler=RandomIdentitySampler(
train_set, num_instances),
pin_memory=True,
drop_last=True)
val_loader = DataLoader(Preprocessor(dataset.val,
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
"""
test_loader = DataLoader(
Preprocessor(test_set, root=dataset.images_dir,
transform=test_transformer),
batch_size=2*query_test_num, num_workers=workers,
shuffle=False, pin_memory=True)
"""
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
return dataset, num_classes, train_loader, val_loader, test_loader
def get_data(name, split_id, data_dir, height, width, batch_size,
num_instances, workers, combine_trainval):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval if combine_trainval else dataset.train
num_classes = (dataset.num_trainval_ids
if combine_trainval else dataset.num_train_ids)
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(Preprocessor(train_set,
root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
sampler=RandomIdentitySampler(
train_set, num_instances),
pin_memory=True,
drop_last=True)
train_loader_head = DataLoader(Preprocessor(
train_set,
root="/home/bfs/zty/reid_market/examples/data/market1501/images_head",
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
sampler=RandomIdentitySampler(
train_set, num_instances),
pin_memory=True,
drop_last=True)
train_loader_upper = DataLoader(Preprocessor(
train_set,
root="/home/bfs/zty/reid_market/examples/data/market1501/images_upper",
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
sampler=RandomIdentitySampler(
train_set, num_instances),
pin_memory=True,
drop_last=True)
train_loader_lower = DataLoader(Preprocessor(
train_set,
root="/home/bfs/zty/reid_market/examples/data/market1501/images_lower",
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
sampler=RandomIdentitySampler(
train_set, num_instances),
pin_memory=True,
drop_last=True)
val_loader = DataLoader(Preprocessor(dataset.val,
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
val_loader_head = DataLoader(Preprocessor(
dataset.val,
root="/home/bfs/zty/reid_market/examples/data/market1501/images_head",
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
val_loader_upper = DataLoader(Preprocessor(
dataset.val,
root="/home/bfs/zty/reid_market/examples/data/market1501/images_upper",
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
val_loader_lower = DataLoader(Preprocessor(
dataset.val,
root="/home/bfs/zty/reid_market/examples/data/market1501/images_lower",
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
test_loader_head = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root="/home/bfs/zty/reid_market/examples/data/market1501/images_head",
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
test_loader_upper = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root="/home/bfs/zty/reid_market/examples/data/market1501/images_upper",
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
test_loader_lower = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root="/home/bfs/zty/reid_market/examples/data/market1501/images_lower",
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
return dataset, num_classes, train_loader, train_loader_head, train_loader_upper, train_loader_lower,\
val_loader, val_loader_head, val_loader_upper, val_loader_lower, test_loader, test_loader_head, \
test_loader_upper, test_loader_lower
batch_size = 50
gpu_ids = '0'
set_paths('paths')
os.environ['CUDA_VISIBLE_DEVICES'] = gpu_ids
args = json.load(open(osp.join(exp_dir, "args.json"), "r"))
# Load data
t = T.Compose([
T.RectScale(args['height'], args['width']),
T.CenterCrop((args['crop_height'], args['crop_width'])),
T.ToTensor(),
T.RGB_to_BGR(),
T.NormalizeBy(255),
])
dataset = datasets.create(args['dataset'], 'data/{}'.format(args['dataset']))
dataset_ = Preprocessor(list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=t)
dataloader = DataLoader(dataset_, batch_size=batch_size, shuffle=False)
# Load model
model = models.create(args['arch'],
dilation=args['dilation'],
use_relu=args['use_relu'],
initialize=False).cuda()
weight_file = osp.join(exp_dir, 'epoch_{}.pth.tar'.format(target_epoch))
model.load(load_checkpoint(weight_file))
model.eval()
# Evaluate
def get_data(args):
(name, split_id,
data_dir, height, width,
batch_size, num_instances,
workers, combine_trainval) = (
args.dataset, args.split,
args.data_dir, args.height, args.width,
args.batch_size, args.num_instances,
args.workers, args.combine_trainval,)
pin_memory = args.pin_mem
name_val = args.dataset_val
npy = args.has_npy
rand_ratio = args.random_ratio
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id, mode=args.dataset_mode)
# pid2lbl = dataset.pid2lbl
# np.unique(list(pid2lbl.keys())).shape
# np.unique(list(pid2lbl.values())).shape
# pid2lbl[7]
root = osp.join(data_dir, name_val)
dataset_val = datasets.create(name_val, root, split_id=split_id, mode=args.dataset_mode)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval if combine_trainval else dataset.train
num_classes = (dataset.num_trainval_ids if combine_trainval
else dataset.num_train_ids)
train_transformer = T.Compose([
T.RandomCropFlip(height, width, area=args.area),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
dop_info = DopInfo(num_classes)
print('dop info and its id are', dop_info)
trainval_t = np.asarray(dataset.trainval, dtype=[('fname', object),
('pid', int),
('cid', int)])
trainval_t = trainval_t.view(np.recarray)
trainval_t = trainval_t[:np.where(trainval_t.pid == 10)[0].min()]
trainval_test_loader = DataLoader(Preprocessor(
# dataset.val,
# dataset.query,
# random.choices(trainval_t, k=1367 * 3),
trainval_t.tolist(),
root=dataset.images_dir,
transform=test_transformer,
has_npy=npy),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=pin_memory)
train_loader = DataLoader(
Preprocessor(train_set, root=dataset.images_dir,
transform=train_transformer,
has_npy=npy),
batch_size=batch_size, num_workers=workers,
sampler=RandomIdentityWeightedSampler(
train_set, num_instances,
batch_size=batch_size,
rand_ratio=rand_ratio,
dop_info=dop_info,
),
# shuffle=True,
pin_memory=pin_memory, drop_last=True)
val_loader = DataLoader(
Preprocessor(dataset_val.val, root=dataset_val.images_dir,
transform=test_transformer,
has_npy=npy),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=pin_memory)
query_ga = np.concatenate([
np.asarray(dataset_val.query).reshape(-1, 3),
np.asarray(list(set(dataset_val.gallery) - set(dataset_val.query))).reshape(-1, 3)
])
query_ga = np.rec.fromarrays((query_ga[:, 0], query_ga[:, 1].astype(int), query_ga[:, 2].astype(int)),
names=['fnames', 'pids', 'cids'])
if args.vis:
pids_chs = np.unique(query_ga.pids)[:10]
query_ga = query_ga[np.where(np.isin(query_ga.pids, pids_chs))[0]]
query_ga = query_ga.tolist()
test_loader = DataLoader(
Preprocessor(query_ga,
root=dataset_val.images_dir,
transform=test_transformer,
has_npy=npy),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=False) # todo for market and dukemtmc
dataset.val = dataset_val.val
dataset.query = dataset_val.query
dataset.gallery = dataset_val.gallery
dataset.images_dir = dataset_val.images_dir
if args.vis:
query = np.asarray(dataset.query, dtype=[('fname', object),
('pids', int),
('cid', int)])
query = query.view(np.recarray)
query = query[np.where(np.isin(query.pids, pids_chs))[0]]
dataset.query = query.tolist()
gallery = np.asarray(dataset.gallery, dtype=[('fname', object),
('pids', int),
('cid', int)])
gallery = gallery.view(np.recarray)
gallery = gallery[np.where(np.isin(gallery.pids, pids_chs))[0]]
dataset.gallery = gallery.tolist()
# dataset.num_val_ids
return dataset, num_classes, train_loader, val_loader, test_loader, dop_info, trainval_test_loader
def main(args):
np.random.seed(args.seed)
torch.manual_seed(args.seed)
cudnn.benchmark = True
# Redirect print to both console and log file
sys.stdout = Logger(osp.join(args.logs_dir, 'log.txt'))
print(args)
shutil.copy(sys.argv[0], osp.join(args.logs_dir,
osp.basename(sys.argv[0])))
# Create data loaders
if args.height is None or args.width is None:
args.height, args.width = (256, 128)
dataset, num_classes, train_loader, val_loader, test_loader = \
get_data(args.dataset, args.split, args.data_dir, args.height,
args.width, args.batch_size * 8, args.workers,
)
# Create model
model = models.create("ft_net_inter",
num_classes=num_classes,
stride=args.stride)
# Load from checkpoint
start_epoch = 0
best_top1 = 0
top1 = 0
is_best = False
if args.checkpoint is not None:
if args.evaluate:
checkpoint = load_checkpoint(args.checkpoint)
param_dict = model.state_dict()
for k, v in checkpoint['state_dict'].items():
if 'model' in k:
param_dict[k] = v
model.load_state_dict(param_dict)
else:
model.model.load_param(args.checkpoint)
model = model.cuda()
# Distance metric
metric = None
# Evaluator
evaluator = Evaluator(model, use_cpu=args.use_cpu)
if args.evaluate:
print("Test:")
evaluator.evaluate(test_loader, dataset.query, dataset.gallery, metric)
return
train_transformer = [
T.Resize((args.height, args.width), interpolation=3),
T.RandomHorizontalFlip(),
T.Pad(10),
T.RandomCrop((args.height, args.width)),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
T.RandomErasing(probability=0.5),
]
train_transformer = T.Compose(train_transformer)
for cluster_epoch in range(args.cluster_epochs):
# -------------------------Stage 1 intra camera training--------------------------
# Cluster and generate new dataset and model
cluster_result = get_intra_cam_cluster_result(model, train_loader,
args.class_number_stage1,
args.linkage)
cluster_datasets = [
datasets.create("cluster", osp.join(args.data_dir, args.dataset),
cluster_result[cam_id], cam_id)
for cam_id in cluster_result.keys()
]
cluster_dataloaders = [
DataLoader(Preprocessor(dataset.train_set,
root=dataset.images_dir,
transform=train_transformer),
batch_size=args.batch_size,
num_workers=args.workers,
shuffle=True,
pin_memory=False,
drop_last=True) for dataset in cluster_datasets
]
param_dict = model.model.state_dict()
model = models.create("ft_net_intra",
num_classes=[
args.class_number_stage1
for cam_id in cluster_result.keys()
],
stride=args.stride)
model_param_dict = model.model.state_dict()
for k, v in model_param_dict.items():
if k in param_dict.keys():
model_param_dict[k] = param_dict[k]
model.model.load_state_dict(model_param_dict)
model = model.cuda()
criterion = nn.CrossEntropyLoss().cuda()
# Optimizer
param_groups = make_params(model, args.lr, args.weight_decay)
optimizer = torch.optim.SGD(param_groups, momentum=0.9)
# Trainer
trainer = IntraCameraTrainer(model,
criterion,
warm_up_epoch=args.warm_up)
print("start training")
# Start training
for epoch in range(0, args.epochs_stage1):
trainer.train(
cluster_epoch,
epoch,
cluster_dataloaders,
optimizer,
print_freq=args.print_freq,
)
# -------------------------------------------Stage 2 inter camera training-----------------------------------
mix_rate = get_mix_rate(args.mix_rate,
cluster_epoch,
args.cluster_epochs,
power=args.decay_factor)
cluster_result = get_inter_cam_cluster_result(model,
train_loader,
args.class_number_stage2,
args.linkage,
mix_rate,
use_cpu=args.use_cpu)
cluster_dataset = datasets.create(
"cluster", osp.join(args.data_dir, args.dataset), cluster_result,
0)
cluster_dataloaders = DataLoader(
Preprocessor(cluster_dataset.train_set,
root=cluster_dataset.images_dir,
transform=train_transformer),
batch_size=args.batch_size_stage2,
num_workers=args.workers,
sampler=RandomIdentitySampler(cluster_dataset.train_set,
args.batch_size_stage2,
args.instances),
pin_memory=False,
drop_last=True)
param_dict = model.model.state_dict()
model = models.create("ft_net_inter",
num_classes=args.class_number_stage2,
stride=args.stride)
model.model.load_state_dict(param_dict)
model = model.cuda()
# Criterion
criterion_entropy = nn.CrossEntropyLoss().cuda()
criterion_triple = TripletLoss(margin=args.margin).cuda()
# Optimizer
param_groups = make_params(model,
args.lr * args.batch_size_stage2 / 32,
args.weight_decay)
optimizer = torch.optim.SGD(param_groups, momentum=0.9)
# Trainer
trainer = InterCameraTrainer(
model,
criterion_entropy,
criterion_triple,
warm_up_epoch=args.warm_up,
)
print("start training")
# Start training
for epoch in range(0, args.epochs_stage2):
trainer.train(cluster_epoch,
epoch,
cluster_dataloaders,
optimizer,
print_freq=args.print_freq)
if (cluster_epoch + 1) % 5 == 0:
evaluator = Evaluator(model, use_cpu=args.use_cpu)
top1, mAP = evaluator.evaluate(test_loader,
dataset.query,
dataset.gallery,
metric,
return_mAP=True)
is_best = top1 > best_top1
best_top1 = max(top1, best_top1)
save_checkpoint(
{
'state_dict': model.state_dict(),
'epoch': cluster_epoch + 1,
'best_top1': best_top1,
'cluster_epoch': cluster_epoch + 1,
},
is_best,
fpath=osp.join(args.logs_dir, 'checkpoint.pth.tar'))
if cluster_epoch == (args.cluster_epochs - 1):
save_checkpoint(
{
'state_dict': model.state_dict(),
'epoch': cluster_epoch + 1,
'best_top1': best_top1,
'cluster_epoch': cluster_epoch + 1,
},
False,
fpath=osp.join(args.logs_dir, 'latest.pth.tar'))
print('\n * cluster_epoch: {:3d} top1: {:5.1%} best: {:5.1%}{}\n'.
format(cluster_epoch, top1, best_top1, ' *' if is_best else ''))
# Final test
print('Test with best model:')
checkpoint = load_checkpoint(osp.join(args.logs_dir, 'model_best.pth.tar'))
model.load_state_dict(checkpoint['state_dict'])
best_rank1, mAP = evaluator.evaluate(test_loader,
dataset.query,
dataset.gallery,
metric,
return_mAP=True)
def main(args):
# 声明动态绘图器
gd = gif_drawer()
cudnn.benchmark = True
cudnn.enabled = True
# get all the labeled and unlabeled data for training
dataset_all = datasets.create(args.dataset,
osp.join(args.data_dir, args.dataset))
l_data, u_data = get_one_shot_in_cam1(
dataset_all,
load_path="./examples/oneshot_{}_used_in_paper.pickle".format(
dataset_all.name))
NN = len(l_data) + len(u_data)
# 总的训练step数的计算
total_step = math.ceil(math.pow(
(100 / args.EF),
(1 / args.q))) # 这里应该取上限或者 +2 多一轮进行one-shot训练的 # EUG base 采样策略
# total_step = math.ceil((2 * NN * args.step_s + args.yita + len(u_data)) / (args.yita + NN + len(l_data))) + 2 # big start 策略
# 输出该轮训练关键的提示信息
print(
"{} training begin with dataset:{},batch_size:{},epoch:{},step_size:{},max_frames:{},total_step:{},EF:{},q:{},yita:{},step_s:{}"
.format(args.exp_name, args.dataset, args.batch_size, args.epoch,
args.step_size, args.max_frames, total_step + 1, args.EF,
args.q, args.yita, args.step_s))
# 指定输出文件
# 第三部分要说明关键参数的设定
sys.stdout = Logger(
osp.join(args.logs_dir, args.dataset, args.exp_name, args.exp_order,
'log' + time.strftime(".%m_%d_%H-%M-%S") + '.txt'))
data_file = codecs.open(osp.join(args.logs_dir, args.dataset,
args.exp_name, args.exp_order,
'data.txt'),
mode='a')
time_file = codecs.open(osp.join(args.logs_dir, args.dataset,
args.exp_name, args.exp_order,
'time.txt'),
mode='a')
save_path = osp.join(args.logs_dir, args.dataset, args.exp_name,
args.exp_order)
resume_step, ckpt_file = -1, ''
if args.resume: # 重新训练的时候用
resume_step, ckpt_file = resume(args)
# initial the EUG algorithm
eug = EUG(model_name=args.arch,
batch_size=args.batch_size,
mode=args.mode,
num_classes=dataset_all.num_train_ids,
data_dir=dataset_all.images_dir,
l_data=l_data,
u_data=u_data,
save_path=save_path,
max_frames=args.max_frames)
# 训练之前初始化数据
nums_to_select = 0
new_train_data = l_data
step = 0
if args.resume:
step = resume_step
nums_to_select = min(
math.ceil(len(u_data) * math.pow((step), args.q) * args.EF / 100),
len(u_data))
step_size = []
isout = 0 #用来标记是否应该结束训练
# 开始的时间记录
exp_start = time.time()
while (not isout):
print(
"{} training begin with dataset:{},batch_size:{},epoch:{},step:{}/{} saved to {}."
.format(args.exp_name, args.dataset, args.batch_size, args.epoch,
step + 1, total_step + 1, save_path))
print(
"key parameters contain EF:{},q:{}. Nums_been_selected:{}".format(
args.EF, args.q, nums_to_select))
# 开始训练
train_start = time.time()
eug.train(new_train_data,
step,
epochs=args.epoch,
step_size=args.step_size,
init_lr=0.1) if step != resume_step else eug.resume(
ckpt_file, step)
# 开始评估
evaluate_start = time.time()
# mAP, top1, top5, top10, top20 = 0,0,0,0,0
mAP, top1, top5, top10, top20 = eug.evaluate(dataset_all.query,
dataset_all.gallery)
# 标签估计
estimate_start = time.time()
# pred_y, pred_score, label_pre, id_num = 0,0,0,0
# pred_y, pred_score, label_pre, id_num = eug.estimate_label()
estimate_end = time.time()
# 循环退出判断
if nums_to_select == len(u_data):
isout = 1
# nums_to_select 的设定
new_nums_to_select = min(
math.ceil(
len(u_data) * math.pow((step + 1), args.q) * args.EF / 100),
len(u_data)) # EUG 基础指数渐进策略
# new_nums_to_select = min(math.ceil((len(u_data)-args.yita)*(step-1)/(total_step-2))+args.yita,len(u_data)) # big start
# selected_idx = eug.select_top_data(pred_score, new_nums_to_select)
new_train_data = eug.generate_new_train_data1(new_nums_to_select)
# 输出该epoch的信息
data_file.write(
"step:{} mAP:{:.2%} top1:{:.2%} top5:{:.2%} top10:{:.2%} top20:{:.2%} nums_selected:{} selected_percent:{:.2%}\n"
.format(int(step + 1), mAP, top1, top5, top10, top20,
nums_to_select, nums_to_select / len(u_data)))
print(
"step:{} mAP:{:.2%} top1:{:.2%} top5:{:.2%} top10:{:.2%} top20:{:.2%} nums_selected:{} selected_percent:{:.2%} \n"
.format(int(step + 1), mAP, top1, top5, top10, top20,
nums_to_select, nums_to_select / len(u_data)))
if args.clock:
train_time = evaluate_start - train_start
evaluate_time = estimate_start - evaluate_start
estimate_time = estimate_end - estimate_start
epoch_time = train_time + estimate_time
time_file.write(
"step:{} train:{} evaluate:{} estimate:{} epoch:{}\n".format(
int(step + 1), train_time, evaluate_time, estimate_time,
epoch_time))
nums_to_select = new_nums_to_select
step = step + 1
data_file.close()
if (args.clock):
exp_end = time.time()
exp_time = exp_end - exp_start
h, m, s = changetoHSM(exp_time)
print("experiment is over, cost %02d:%02d:%02.6f" % (h, m, s))
time_file.close()
def get_data(name, split_id, data_dir, height, width, batch_size,
num_instances, workers, combine_trainval, flip_prob, padding,
re_prob):
root = osp.join(data_dir, name)
print(root)
dataset = datasets.create(name, root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
trainvallabel = dataset.trainvallabel
train_set = dataset.trainval if combine_trainval else dataset.train
num_classes = (dataset.num_trainval_ids
if combine_trainval else dataset.num_train_ids)
train_transformer = T.Compose([
T.Resize((height, width)),
T.RandomHorizontalFlip(),
T.Pad(padding),
T.RandomCrop((height, width)),
T.ToTensor(), normalizer,
RandomErasing(probability=re_prob, mean=[0.485, 0.456, 0.406])
])
# train_transformer = T.Compose([
# T.RandomSizedRectCrop(height, width),
# T.RandomHorizontalFlip(),
# T.ToTensor(),
# normalizer,
# ])
test_transformer = T.Compose([
T.Resize((height, width)),
T.ToTensor(),
normalizer,
])
val_loader = DataLoader(Preprocessor(dataset.val,
root=dataset.images_dir,
transform=test_transformer),
batch_size=32,
num_workers=workers,
shuffle=False,
pin_memory=True)
query_loader = DataLoader(Preprocessor(list(set(dataset.query)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=32,
num_workers=workers,
sampler=CamSampler(list(set(dataset.query)),
[2, 5]),
shuffle=False,
pin_memory=True)
gallery_loader = DataLoader(Preprocessor(list(set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=32,
num_workers=workers,
sampler=CamSampler(list(set(dataset.gallery)),
[0, 1, 3, 4], 4),
shuffle=False,
pin_memory=True)
train_loader = DataLoader(Preprocessor(train_set,
root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
sampler=RandomIdentitySampler(
train_set, num_instances),
pin_memory=True,
drop_last=True)
return dataset, num_classes, train_loader, trainvallabel, val_loader, query_loader, gallery_loader
def main(args):
sys.stdout = Logger(osp.join(args.logs_dir, 'log.txt'))
sys.stderr = Logger(osp.join(args.logs_dir, 'err.txt'))
lz.init_dev(args.gpu)
print('config is {}'.format(vars(args)))
if args.seed is not None:
np.random.seed(args.seed)
torch.manual_seed(args.seed)
cudnn.benchmark = True
# Create data loaders
assert args.num_instances > 1, "num_instances should be greater than 1"
assert args.batch_size % args.num_instances == 0, \
'num_instances should divide batch_size'
dataset = datasets.create(args.dataset, root=args.root + '/' + args.dataset,
split_id=args.split_id, mode=args.dataset_mode)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval
num_classes = dataset.num_trainval_ids
train_transformer = T.Compose([
T.RandomCropFlip(args.height, args.width, area=args.area),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectScale(args.height, args.width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(
Preprocessor(train_set, root=dataset.images_dir,
transform=train_transformer,
),
batch_size=args.batch_size, num_workers=args.workers,
sampler=RandomIdentityWeightedSampler(
train_set, args.num_instances,
batch_size=args.batch_size,
rand_ratio=args.rand_ratio,
),
# shuffle=True,
pin_memory=args.pin_memory, drop_last=True)
test_loader = DataLoader(
Preprocessor(dataset.val,
root=dataset.images_dir,
transform=test_transformer,
),
batch_size=args.batch_size, num_workers=args.workers,
shuffle=False, pin_memory=False)
# Create model
model = models.create(args.arch,
dropout=args.dropout,
pretrained=args.pretrained,
block_name=args.block_name,
block_name2=args.block_name2,
num_features=args.num_classes,
num_classes=num_classes,
num_deform=args.num_deform,
fusion=args.fusion,
)
print(model)
param_mb = sum(p.numel() for p in model.parameters()) / 1000000.0
logging.info(' Total params: %.2fM' % (param_mb))
# Load from checkpoint
start_epoch = best_top1 = 0
if args.resume:
while not osp.exists(args.resume):
lz.logging.warning(' no chkpoint {} '.format(args.resume))
time.sleep(20)
if torch.cuda.is_available():
checkpoint = load_checkpoint(args.resume)
else:
checkpoint = load_checkpoint(args.resume, map_location='cpu')
# model.load_state_dict(checkpoint['state_dict'])
db_name = args.logs_dir + '/' + args.logs_dir.split('/')[-1] + '.h5'
load_state_dict(model, checkpoint['state_dict'])
with lz.Database(db_name) as db:
if 'cent' in checkpoint:
db['cent'] = to_numpy(checkpoint['cent'])
db['xent'] = to_numpy(checkpoint['state_dict']['embed2.weight'])
if args.restart:
start_epoch_ = checkpoint['epoch']
best_top1_ = checkpoint['best_top1']
print("=> Start epoch {} best top1 {:.1%}"
.format(start_epoch_, best_top1_))
else:
start_epoch = checkpoint['epoch']
best_top1 = checkpoint['best_top1']
print("=> Start epoch {} best top1 {:.1%}"
.format(start_epoch, best_top1))
if args.gpu is None:
model = nn.DataParallel(model)
elif len(args.gpu) == 1:
model = nn.DataParallel(model).cuda()
else:
model = nn.DataParallel(model, device_ids=range(len(args.gpu))).cuda()
# Distance metric
metric = DistanceMetric(algorithm=args.dist_metric)
# Evaluator
evaluator = Evaluator(model, gpu=args.gpu, conf=args.eval_conf, args=args)
if args.evaluate:
res = evaluator.evaluate(test_loader, dataset.query, dataset.gallery, metric,
final=True, prefix='test')
lz.logging.info('eval {}'.format(res))
return res
# Criterion
if not args.xent_smooth:
xent = nn.CrossEntropyLoss()
else:
xent = CrossEntropyLabelSmooth(num_classes=num_classes)
setattr(xent, 'name', 'xent')
criterion = [TripletLoss(margin=args.margin, mode='hard', args=args),
CenterLoss(num_classes=num_classes, feat_dim=args.num_classes,
margin2=args.margin2,
margin3=args.margin3, mode=args.mode,
push_scale=args.push_scale,
args=args),
xent
]
if args.gpu is not None:
criterion = [c.cuda() for c in criterion]
# Optimizer
fast_params = []
for name, param in model.named_parameters():
if name == 'module.embed1.weight' or name == 'module.embed2.weight':
fast_params.append(param)
fast_params_ids = set(map(fid, fast_params))
normal_params = [p for p in model.parameters() if fid(p) not in fast_params_ids]
param_groups = [
{'params': fast_params, 'lr_mult': args.lr_mult},
{'params': normal_params, 'lr_mult': 1.},
]
if args.optimizer_cent == 'sgd':
optimizer_cent = torch.optim.SGD(criterion[1].parameters(), lr=args.lr_cent, )
else:
optimizer_cent = torch.optim.Adam(criterion[1].parameters(), lr=args.lr_cent, )
if args.optimizer == 'adam':
optimizer = torch.optim.Adam(
# model.parameters(),
param_groups,
lr=args.lr,
betas=args.adam_betas,
eps=args.adam_eps, # adam hyperparameter
weight_decay=args.weight_decay)
elif args.optimizer == 'sgd':
optimizer = torch.optim.SGD(
# filter(lambda p: p.requires_grad, model.parameters()),
param_groups,
lr=args.lr,
weight_decay=args.weight_decay, momentum=0.9,
nesterov=True)
else:
raise NotImplementedError
if args.cls_pretrain:
args_cp = copy.deepcopy(args)
args_cp.cls_weight = 1
args_cp.tri_weight = 0
trainer = XentTrainer(model, criterion, dbg=False,
logs_at=args_cp.logs_dir + '/vis', args=args_cp)
for epoch in range(start_epoch, args_cp.epochs):
hist = trainer.train(epoch, train_loader, optimizer)
save_checkpoint({
'state_dict': model.module.state_dict(),
'cent': criterion[1].centers,
'epoch': epoch + 1,
'best_top1': best_top1,
}, True, fpath=osp.join(args.logs_dir, 'checkpoint.{}.pth'.format(epoch))) #
print('Finished epoch {:3d} hist {}'.
format(epoch, hist))
# Trainer
trainer = TCXTrainer(model, criterion, dbg=True,
logs_at=args.logs_dir + '/vis', args=args, dop_info=dop_info)
# Schedule learning rate
def adjust_lr(epoch, optimizer=optimizer, base_lr=args.lr, steps=args.steps, decay=args.decay):
exp = len(steps)
for i, step in enumerate(steps):
if epoch < step:
exp = i
break
lr = base_lr * decay ** exp
lz.logging.info('use lr {}'.format(lr))
for param_group in optimizer.param_groups:
param_group['lr'] = lr * param_group.get('lr_mult', 1)
def adjust_bs(epoch, args):
if args.batch_size_l == []:
return args
res = 0
for i, step in enumerate(args.bs_steps):
if epoch > step:
res = i + 1
print(epoch, res)
if res >= len(args.num_instances_l):
res = -1
args.batch_size = args.batch_size_l[res]
args.num_instances = args.num_instances_l[res]
return args
writer = SummaryWriter(args.logs_dir)
writer.add_scalar('param', param_mb, global_step=0)
# schedule = CyclicLR(optimizer)
schedule = None
# Start training
for epoch in range(start_epoch, args.epochs):
# warm up
# mAP, acc,rank5 = evaluator.evaluate(val_loader, dataset.val, dataset.val, metric)
adjust_lr(epoch=epoch)
args = adjust_bs(epoch, args)
hist = trainer.train(epoch, train_loader, optimizer, print_freq=args.print_freq, schedule=schedule,
optimizer_cent=optimizer_cent)
for k, v in hist.items():
writer.add_scalar('train/' + k, v, epoch)
writer.add_scalar('lr', optimizer.param_groups[0]['lr'], epoch)
writer.add_scalar('bs', args.batch_size, epoch)
writer.add_scalar('num_instances', args.num_instances, epoch)
if not args.log_middle:
continue
if epoch < args.start_save:
continue
if epoch % 15 == 0:
save_checkpoint({
'state_dict': model.module.state_dict(),
'cent': criterion[1].centers,
'epoch': epoch + 1,
'best_top1': best_top1,
}, False, fpath=osp.join(args.logs_dir, 'checkpoint.{}.pth'.format(epoch)))
if epoch not in args.log_at:
continue
save_checkpoint({
'state_dict': model.module.state_dict(),
'cent': criterion[1].centers,
'epoch': epoch + 1,
'best_top1': best_top1,
}, False, fpath=osp.join(args.logs_dir, 'checkpoint.{}.pth'.format(epoch)))
# res = evaluator.evaluate(val_loader, dataset.val, dataset.val, metric)
# for n, v in res.items():
# writer.add_scalar('train/'+n, v, epoch)
res = evaluator.evaluate(test_loader, dataset.query, dataset.gallery, metric, epoch=epoch)
for n, v in res.items():
writer.add_scalar('test/' + n, v, epoch)
top1 = res['top-1']
is_best = top1 > best_top1
best_top1 = max(top1, best_top1)
save_checkpoint({
'state_dict': model.module.state_dict(),
'cent': criterion[1].centers,
'epoch': epoch + 1,
'best_top1': best_top1,
}, is_best, fpath=osp.join(args.logs_dir, 'checkpoint.{}.pth'.format(epoch))) #
print('\n * Finished epoch {:3d} top1: {:5.1%} best: {:5.1%}{}\n'.
format(epoch, top1, best_top1, ' *' if is_best else ''))
# Final test
res = evaluator.evaluate(test_loader, dataset.query, dataset.gallery, metric)
for n, v in res.items():
writer.add_scalar('test/' + n, v, args.epochs)
if osp.exists(osp.join(args.logs_dir, 'model_best.pth')) and args.test_best:
print('Test with best model:')
checkpoint = load_checkpoint(osp.join(args.logs_dir, 'model_best.pth'))
model.module.load_state_dict(checkpoint['state_dict'])
metric.train(model, train_loader)
res = evaluator.evaluate(test_loader, dataset.query, dataset.gallery, metric, final=True)
for n, v in res.items():
writer.add_scalar('test/' + n, v, args.epochs + 1)
lz.logging.info('final eval is {}'.format(res))
writer.close()
json_dump(res, args.logs_dir + '/res.json', 'w')
return res
def main(args):
cudnn.benchmark = True
cudnn.enabled = True
# get all the labeled and unlabeled data for training
dataset_all = datasets.create(args.dataset,
osp.join(args.data_dir, args.dataset))
l_data, u_data = get_one_shot_in_cam1(
dataset_all,
load_path="./examples/oneshot_{}_used_in_paper.pickle".format(
dataset_all.name))
#声明伪标签数据列表
p_data = []
s_data = [] # 表示选择出来的伪标签样本
new_train_data = []
mv_num = math.ceil(len(u_data) / args.total_step) # 最后一轮必定不足add_num的数量
tagper_num = math.ceil(len(u_data) / args.train_tagper_step)
# 输出实验信息
print(
"{}/{} is training with {}, the max_frames is {}, and will be saved to {}"
.format(args.exp_name, args.exp_order, args.dataset, args.max_frames,
args.logs_dir))
# 输出超参信息
print("parameters are setted as follows")
print("\ttotal_step:\t{}".format(args.total_step))
print("\ttagper_step:\t{}".format(args.train_tagper_step))
print("\tepoch:\t{}".format(args.epoch))
print("\tstep_size:\t{}".format(args.step_size))
print("\tbatch_size:\t{}".format(args.batch_size))
print("\tstop_vari_step:\t{}".format(args.stop_vari_step))
print("\tpercent_vari:\t{}".format(args.percent_vari))
# 指定输出文件
# 第三部分要说明关键参数的设定
reid_path = osp.join(args.logs_dir, args.dataset, args.exp_name,
args.exp_order)
sys.stdout = Logger(
osp.join(reid_path, 'log' + time.strftime(".%m_%d_%H-%M-%S") + '.txt'))
data_file = codecs.open(osp.join(reid_path, 'data.txt'), mode='a')
time_file = codecs.open(osp.join(reid_path, 'time.txt'), mode='a')
tagper_file = codecs.open(osp.join(reid_path, "tagper_data.txt"), mode='a')
# initial the EUG algorithm
reid = EUG(model_name=args.arch,
batch_size=args.batch_size,
mode=args.mode,
num_classes=dataset_all.num_train_ids,
data_dir=dataset_all.images_dir,
l_data=l_data,
u_data=u_data,
save_path=reid_path,
max_frames=args.max_frames)
tagper = EUG(model_name=args.arch,
batch_size=args.batch_size,
mode=args.mode,
num_classes=dataset_all.num_train_ids,
data_dir=dataset_all.images_dir,
l_data=l_data,
u_data=u_data,
save_path=reid_path,
max_frames=args.max_frames)
# 开始的时间记录
exp_start = time.time()
for step in range(args.total_step + 1): #加1是为了保证所有的数据都能加入到训练集中
print(
"---------------------------------training step:{}/{}-------------------------------------"
.format(step + 1, args.total_step + 1))
# 开始训练
reid_start = time.time()
train_reid_data = l_data + s_data # 在这个过程中,保持了one_shot不变了
if (step == 0) and not args.is_baseline:
reid.resume(
osp.join(reid_path, 'Dissimilarity_step_{}.ckpt'.format(step)),
step)
else:
reid.train(train_reid_data,
step,
tagper=0,
epochs=args.epoch,
step_size=args.step_size,
init_lr=0.1)
# 开始评估
# mAP, top1, top5, top10, top20 =0,0,0,0,0
mAP, top1, top5, top10, top20 = reid.evaluate(dataset_all.query,
dataset_all.gallery)
# 测试 train tagper之前的select_pre
pred_y, pred_score, label_pre, dists = reid.estimate_label_dist(
u_data, l_data)
# selected_idx = reid.select_top_data(pred_score, min(mv_num * (step + 1), len(u_data)))
new_nums_to_select = min(mv_num * (step + 1), len(u_data))
if step < args.stop_vari_step:
real_percent_vari = min(
(1 - args.percent_vari) * (step + 1) /
(args.stop_vari_step + 1) + args.percent_vari, 1)
new_expend_nums_to_select = min(
len(u_data), math.ceil(new_nums_to_select / real_percent_vari))
selected_idx = reid.select_top_data_nlvm_b1(
pred_score, dists, new_expend_nums_to_select,
new_nums_to_select)
else:
real_percent_vari = 1
new_expend_nums_to_select = 0
selected_idx = reid.select_top_data(pred_score, new_nums_to_select)
# new_train_data, select_pre = reid.generate_new_train_data(selected_idx, pred_y,u_data)
select_pre = reid.get_select_pre(selected_idx, pred_y, u_data)
reid_end = time.time()
tagper_start = time.time()
'''第一个tagper可以resume'''
# if step == 0 and not args.is_baseline:
# tagper.resume(osp.join(reid_path,'tagper','Dissimilarity_step_0.ckpt'), 0)
# else:
# tagper.resume(osp.join(reid_path, 'Dissimilarity_step_{}.ckpt'.format(step)), step)
# selected_idx = tagper.select_top_data(pred_score, min(tagper_num*(step+1),len(u_data))) #训练tagper的数量也递增
# new_train_data = tagper.generate_new_train_data_only(selected_idx, pred_y, u_data) # 这个选择准确率应该是和前面的label_pre是一样的.
# train_tagper_data = one_shot+l_data+new_train_data
# tagper.train(train_tagper_data, step, tagper=1, epochs=args.epoch, step_size=args.step_size, init_lr=0.1)
'''所有的tagper都重新训练'''
# if step != 0:
if True:
tagper.resume(
osp.join(reid_path, 'Dissimilarity_step_{}.ckpt'.format(step)),
step)
select_num_for_tagper = min(tagper_num * (step + 1), len(u_data))
real_percent_vari = min(
(1 - args.percent_vari) * (step + 1) /
(args.train_tagper_step + 1) + args.percent_vari, 1)
expand_num_for_tagper = min(
len(u_data),
math.ceil(select_num_for_tagper / real_percent_vari))
# selected_idx_for_tagper = tagper.select_top_data(pred_score, min(tagper_num * (step + 1), len(u_data))) # 训练tagper的数量也递增
selected_idx_for_tagper = reid.select_top_data_nlvm_b1(
pred_score, dists, expand_num_for_tagper,
select_num_for_tagper)
new_train_data = tagper.generate_new_train_data_only(
selected_idx_for_tagper, pred_y,
u_data) # 这个选择准确率应该是和前面的label_pre是一样的.
train_tagper_data = l_data + new_train_data
tagper.train(train_tagper_data,
step,
tagper=1,
epochs=args.epoch,
step_size=args.step_size,
init_lr=0.1)
# else: # 如果是0 就是直接resume
# tagper.resume(osp.join(reid_path,'tagper1','Dissimilarity_step_{}.ckpt'.format(step)), step)
# 开始评估
# mAP, top1, top5, top10, top20 =0,0,0,0,0
tmAP, ttop1, ttop5, ttop10, ttop20 = tagper.evaluate(
dataset_all.query, dataset_all.gallery)
tpred_y, tpred_score, tlabel_pre, tdists = tagper.estimate_label_dist(
u_data, l_data)
# 下面正对 reid 移动数据.
selected_idx = tagper.select_top_data_nlvm_b1(
tpred_score, tdists, new_expend_nums_to_select, new_nums_to_select)
# selected_idx = tagper.select_top_data(tpred_score, min(mv_num * (step + 1), len(u_data))) # 从所有 u_data 里面选
s_data, tselect_pre = tagper.move_unlabel_to_label_cpu(
selected_idx, tpred_y, u_data)
tapger_end = time.time()
data_file.write(
"step:{} mAP:{:.2%} top1:{:.2%} top5:{:.2%} top10:{:.2%} top20:{:.2%} len(l_data):{} label_pre:{:.2%} select_pre:{:.2%}\n"
.format(int(step + 1), mAP, top1, top5, top10, top20, len(s_data),
label_pre, select_pre))
print(
"reid step:{} mAP:{:.2%} top1:{:.2%} top5:{:.2%} top10:{:.2%} top20:{:.2%} len(l_data):{} label_pre:{:.2%} select_pre:{:.2%} \n"
.format(int(step + 1), mAP, top1, top5, top10, top20, len(s_data),
label_pre, select_pre))
tagper_file.write(
"step:{} mAP:{:.2%} top1:{:.2%} top5:{:.2%} top10:{:.2%} top20:{:.2%} len(l_data):{} label_pre:{:.2%} select_pre:{:.2%}\n"
.format(int(step + 1), tmAP, ttop1, ttop5, ttop10, ttop20,
len(s_data), tlabel_pre, tselect_pre))
print(
"tagper step:{} mAP:{:.2%} top1:{:.2%} top5:{:.2%} top10:{:.2%} top20:{:.2%} len(l_data):{} label_pre:{:.2%} select_pre:{:.2%}\n"
.format(int(step + 1), tmAP, ttop1, ttop5, ttop10, ttop20,
len(s_data), tlabel_pre, tselect_pre))
if args.clock:
reid_time = reid_end - reid_start
tagper_time = tapger_end - tagper_start
step_time = tapger_end - reid_start
time_file.write(
"step:{} reid_time:{} tagper_time:{} step_time:{}\n".format(
int(step + 1), reid_time, tagper_time, step_time))
h, m, s = changetoHSM(step_time)
print("this step is over, cost %02d:%02d:%02.6f" % (h, m, s))
data_file.close()
tagper_file.close()
if (args.clock):
exp_end = time.time()
exp_time = exp_end - exp_start
h, m, s = changetoHSM(exp_time)
print("experiment is over, cost %02d:%02d:%02.6f" % (h, m, s))
time_file.close()
def get_data(name,
data_dir,
height,
width,
ratio,
batch_size,
workers,
num_instances=8):
root = osp.join(data_dir, name)
root = data_dir
dataset_set = []
query_loader_set = []
gallery_loader_set = []
num_classes = 752
for i in range(5):
dataset = datasets.create(name,
root,
query_group=i + 1,
gallery_group=i + 1)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
listnormalizer = T.ListNormalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
query_transformer = T.Compose([
T.ContVerticalCropDiscret(height, width, ratio),
T.ToTensor(),
normalizer,
])
query_loader = DataLoader(Preprocessor(dataset.query,
root=osp.join(
dataset.images_dir,
dataset.query_path),
transform=query_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
gallery_loader = DataLoader(Preprocessor(dataset.gallery,
root=osp.join(
dataset.images_dir,
dataset.gallery_path),
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
dataset_set.append(dataset)
query_loader_set.append(query_loader)
gallery_loader_set.append(gallery_loader)
return dataset_set, num_classes, query_loader_set, gallery_loader_set
def get_data(dataname,
data_dir,
height,
width,
batch_size,
camstyle=0,
re=0,
num_instances=4,
workers=8):
root = osp.join(data_dir, dataname)
dataset = datasets.create(dataname, root)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
num_classes = dataset.num_train_ids
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
T.RandomErasing(EPSILON=re),
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(
Preprocessor(dataset.train,
root=osp.join(dataset.images_dir, dataset.train_path),
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
sampler=RandomIdentitySampler(dataset.train, num_instances),
pin_memory=True,
drop_last=True)
query_loader = DataLoader(Preprocessor(dataset.query,
root=osp.join(
dataset.images_dir,
dataset.query_path),
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
gallery_loader = DataLoader(Preprocessor(dataset.gallery,
root=osp.join(
dataset.images_dir,
dataset.gallery_path),
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
return dataset, num_classes, train_loader, query_loader, gallery_loader
def main(args):
# 声明动态绘图器
gd = gif_drawer()
cudnn.benchmark = True
cudnn.enabled = True
# get all the labeled and unlabeled data for training
dataset_all = datasets.create(args.dataset,
osp.join(args.data_dir, args.dataset))
l_data, u_data = get_one_shot_in_cam1(
dataset_all,
load_path="./examples/oneshot_{}_used_in_paper.pickle".format(
dataset_all.name))
# 输出该轮训练关键的提示信息
print(
"{} training begin with dataset:{},batch_size:{},epoch:{},step_size:{},max_frames:{},percent:{}"
.format(args.exp_name, args.dataset, args.batch_size, args.epoch,
args.step_size, args.max_frames, args.percent))
# 指定输出文件
# 第三部分要说明关键参数的设定
sys.stdout = Logger(
osp.join(args.logs_dir, args.dataset, args.exp_name, args.exp_order,
'log_mSA' + time.strftime(".%m_%d_%H-%M-%S") + '.txt'))
data_file = codecs.open(osp.join(args.logs_dir, args.dataset,
args.exp_name, args.exp_order,
'data.txt'),
mode='a')
time_file = codecs.open(osp.join(args.logs_dir, args.dataset,
args.exp_name, args.exp_order,
'time.txt'),
mode='a')
save_path = osp.join(args.logs_dir, args.dataset, args.exp_name,
args.exp_order)
# initial the EUG algorithm
eug = EUG(model_name=args.arch,
batch_size=args.batch_size,
mode=args.mode,
num_classes=dataset_all.num_train_ids,
data_dir=dataset_all.images_dir,
l_data=l_data,
u_data=u_data,
save_path=save_path,
max_frames=args.max_frames)
# 开始的时间记录
exp_start = time.time()
eug.resume(osp.join(save_path, 'Dissimilarity_step_0.ckpt'), 0)
pred_y, pred_score, label_pre, id_num = eug.estimate_label()
select_num = math.ceil(len(u_data) * args.percent) # 用于训练tagper的数量
selected_idx = eug.select_top_data(pred_score, select_num)
train_data, select_pre = eug.generate_new_train_data(selected_idx, pred_y)
eug.train(train_data,
1,
epochs=args.epoch,
step_size=args.step_size,
init_lr=0.1)
pred_y, pred_score, label_pre, id_num = eug.estimate_label() #tagper的数据
data_file.write('percent:{} label_pre:{}'.format(args.percent, label_pre))
select_pre = [5, 10, 20, 30, 40]
for sp in select_pre: #对采样比例做便利\
select_num = math.ceil(len(u_data) * sp / 100)
selected_idx = eug.select_top_data(pred_score, select_num)
_, select_pre = eug.generate_new_train_data(selected_idx, pred_y)
data_file.write(' sp{}:{}'.format(sp, select_pre))
data_file.write('\n')
data_file.close()
if (args.clock):
exp_end = time.time()
exp_time = exp_end - exp_start
h, m, s = changetoHSM(exp_time)
print("experiment is over, cost %02d:%02d:%02.6f" % (h, m, s))
time_file.close()
def main(args):
cudnn.benchmark = True
cudnn.enabled = True
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
if not args.no_log:
log_path = os.path.join(args.logs_dir, args.exp_name)
if not osp.exists(log_path):
os.makedirs(log_path)
log_name = args.exp_name + "_log_" \
+ time.strftime("%Y-%m-%d_%H-%M-%S", time.gmtime()) + '.txt'
sys.stdout = Logger(osp.join(log_path, log_name))
print(args)
if args.seed is not None:
cudnn.benchmark = False
cudnn.deterministic = True
# fix random seed
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
random.seed(args.seed)
print("set random seed={}".format(args.seed))
snap_dir = osp.join(args.snap_dir, args.exp_name)
# get all unlabeled data for training
dataset_all = datasets.create(args.dataset,
osp.join(args.data_dir, args.dataset))
new_train_data, cluster_id_labels = change_to_unlabel(dataset_all)
num_train_ids = len(np.unique(np.array(cluster_id_labels)))
nums_to_merge = int(num_train_ids * args.merge_percent)
BuMain = Bottom_up(model_name=args.arch,
batch_size=args.batch_size,
num_classes=num_train_ids,
dataset=dataset_all,
u_data=new_train_data,
save_path=args.logs_dir,
max_frames=args.max_frames,
embeding_fea_size=args.fea)
start_step, train_data, labels = BuMain.load_checkpoint(args.resume_path)
if train_data is not None:
new_train_data = train_data
if labels is not None:
cluster_id_labels = labels
for step in range(start_step, int(1 / args.merge_percent) - 1):
print('step: ', step)
BuMain.train(new_train_data, step, loss=args.loss)
BuMain.evaluate(dataset_all.query, dataset_all.gallery)
# get new train data for the next iteration
print(
'---------------------bottom-up clustering-----------------------')
if args.mode == 'buc':
cluster_id_labels, new_train_data = BuMain.get_new_train_data(
cluster_id_labels,
nums_to_merge,
size_penalty=args.size_penalty)
elif args.mode == 'dbc':
cluster_id_labels, new_train_data = BuMain.get_new_train_data_dbc(
cluster_id_labels, nums_to_merge, penalty=args.size_penalty)
else:
raise ValueError('Unknown BUC mode: {}'.format(args.mode))
if args.save_snap:
BuMain.save_checkpoint(snap_dir, step, new_train_data,
cluster_id_labels)
print('\n')
parser.add_argument('--checkpoint', default='', type=str)
args = parser.parse_args()
config = Config(model_name=args.arch,
img_translation=None,
height=args.height,
width=args.width)
# config1.height = 224
# config1.width = 224
# config1.epochs = 200
dataset = args.dataset
cur_path = os.getcwd()
logs_dir = os.path.join(cur_path, 'logs')
data_dir = os.path.join(cur_path, 'data', dataset)
data = datasets.create(dataset, data_dir)
# train_data,untrain_data = dp.split_dataset(data.trainval, 0.2)
if args.checkpoint is not None:
model = models.create(config.model_name,
num_features=config.num_features,
dropout=config.dropout,
num_classes=config.num_classes)
model = torch.nn.DataParallel(model).cuda()
save_pth = os.path.join(config.logs_dir, config.model_name,
'%s' % (args.checkpoint))
if os.path.exists(save_pth) is not True:
raise ValueError('wrong model pth')
checkpoint = load_checkpoint(save_pth)
def get_data(name, data_dir):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root)
return dataset
def get_data(dataname,
data_dir,
height,
width,
batch_size,
camstyle=0,
re=0,
workers=8,
camstyle_path=None):
if dataname in [
"mmf6-indicam", "mmf6-indicam-10", "mmf6-pairs", "mmf6-pairs-10"
]:
root = osp.join(data_dir, "mmf6")
else:
root = osp.join(data_dir, dataname)
dataset = datasets.create(dataname, root, camstyle_path=camstyle_path)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
num_classes = dataset.num_train_ids
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
T.RandomErasing(EPSILON=re),
])
test_transformer = T.Compose([
T.Resize((height, width), interpolation=3),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(Preprocessor(dataset.train,
root=None,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=True,
pin_memory=True,
drop_last=True)
query_loader = DataLoader(Preprocessor(dataset.query,
root=None,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
gallery_loader = DataLoader(Preprocessor(dataset.gallery,
root=None,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
if camstyle <= 0:
camstyle_loader = None
else:
camstyle_loader = DataLoader(Preprocessor(dataset.camstyle,
root=None,
transform=train_transformer),
batch_size=camstyle,
num_workers=workers,
shuffle=True,
pin_memory=True,
drop_last=True)
return dataset, num_classes, train_loader, query_loader, gallery_loader, camstyle_loader
def main(args):
father = Path('/mnt/')
if father.exists(): # 是在服务器上
data_dir = Path('/mnt/share/datasets/RE-ID/data') # 服务器
logs_dir = Path('/mnt/home/{}'.format(args.log_name)) # 服务器
else: #本地
data_dir = Path('/home/joselyn/workspace/ATM_SERIES/data') # 本地跑用这个
logs_dir = Path('/home/joselyn/workspace/ATM_SERIES/{}'.format(
args.log_name)) # 本地跑用这个
cudnn.benchmark = True
cudnn.enabled = True
save_path = os.path.join(logs_dir, args.dataset, args.exp_name,
args.exp_order) # 到编号位置.
total_step = 100 // args.EF + 1
sys.stdout = Logger(
osp.join(
save_path,
'log' + str(args.EF) + time.strftime(".%m_%d_%H:%M:%S") + '.txt'))
dataf_file = open(osp.join(save_path, 'dataf.txt'),
'a') # 保存性能数据. #特征空间中的性能问题.
data_file = open(osp.join(save_path, 'data.txt'),
'a') # 保存性能数据. #特征空间中的性能问题.
kf_file = open(osp.join(save_path, 'kf.txt'), 'a')
# 数据格式为 label_pre_r, select_pre_r,label_pre_t, select_pre_t ,加上了了tagper的数据.
tagper_path = osp.join(save_path, 'tagper') #tagper存储路径.
if not Path(tagper_path).exists():
os.mkdir(tagper_path)
'''# 记录配置信息 和路径'''
print('-' * 20 + 'config_info' + '-' * 20)
config_file = open(osp.join(save_path, 'config.txt'), 'w')
config_info = str(args).split('(')[1].strip(')').split(',')
config_info.sort()
for one in config_info:
key, value = map(str, one.split('='))
config_file.write(key.strip() + '=' + value.strip('\'') + '\n')
print(key.strip() + '=' + value.strip('\''))
config_file.write('save_path=' + save_path)
print('save_path=' + save_path)
print('-' * 20 + 'config_info' + '-' * 20)
config_file.close()
train_time_file = open(osp.join(save_path, 'time.txt'),
'a') # 只记录训练所需要的时间.
# 数据格式为 step_time total_time.
total_time = 0
# get all the labeled and unlabeled data for training
dataset_all = datasets.create(args.dataset,
osp.join(data_dir, args.dataset))
num_all_examples = len(dataset_all.train)
l_data, u_data = get_init_shot_in_cam1(
dataset_all,
load_path="./examples/{}_init_{}.pickle".format(
dataset_all.name, args.init),
init=args.init)
resume_step, ckpt_file = -1, ''
if args.resume:
resume_step, ckpt_file = resume(save_path)
# initial the EUG algorithm
eug = EUG(batch_size=args.batch_size,
num_classes=dataset_all.num_train_ids,
dataset=dataset_all,
l_data=l_data,
u_data=u_data,
save_path=save_path,
max_frames=args.max_frames,
embeding_fea_size=args.fea,
momentum=args.momentum,
lamda=args.lamda)
tagper = EUG(batch_size=args.batch_size,
num_classes=dataset_all.num_train_ids,
dataset=dataset_all,
l_data=l_data,
u_data=u_data,
save_path=tagper_path,
max_frames=args.max_frames,
embeding_fea_size=args.fea,
momentum=args.momentum,
lamda=args.lamda)
new_train_data = l_data
unselected_data = u_data
iter_mode = 2 #迭代模式,确定是否训练tagper
for step in range(total_step):
# for resume
if step < resume_step:
continue
ratio = (step + 1) * args.EF / 100
ratio_t = (step + 1 + args.t) * args.EF / 100
nums_to_select = int(len(u_data) * ratio)
nums_to_select_tagper = int(len(u_data) * ratio_t)
if nums_to_select >= len(u_data):
break
# args.vsm_lambda的衰减 0.5
vsm_lambda = args.vsm_lambda * step / (
1 - (total_step / 2)) + args.vsm_lambda
# yml: ... Ritio {} ...
print(
"Runing: EF={}%, step {}:\t Nums_to_be_select {} \t Ritio {}\t Logs-dir {}"
.format(args.EF, step, nums_to_select, ratio, save_path))
# train the model or load ckpt
start_time = time.time()
print("training reid model")
eug.train(new_train_data,
unselected_data,
step,
loss=args.loss,
epochs=args.epochs,
step_size=args.step_size,
init_lr=0.1) if step != resume_step else eug.resume(
ckpt_file, step)
# 只对eug进行性能评估
# mAP, rank1, rank5, rank10, rank20 = 0,0,0,0,0
mAP, rank1, rank5, rank10, rank20 = eug.evaluate(
dataset_all.query, dataset_all.gallery)
# 把数据写到data文件里.
data_file.write('{} {:.2%} {:.2%} {:.2%} {:.2%} {:.2%}\n'.format(
step, mAP, rank1, rank5, rank10, rank20))
pred_y, pred_score, label_pre, dists = eug.estimate_label_vsm()
selected_idx = eug.select_top_data_vsm(
pred_score, dists, args.topk, vsm_lambda,
min(nums_to_select_tagper,
len(u_data) - 50) if iter_mode == 2 else min(
nums_to_select,
len(u_data))) #直接翻两倍取数据. -50个样本,保证unselected_data数量不为0
new_train_data, unselected_data, select_pre = eug.generate_new_train_data(
selected_idx, pred_y, step, args.experiment) # yml:新增后两个参数
raw_label_pre, raw_select_pre = label_pre, select_pre
t_label_pre, t_select_pre = 0, 0
raw_select_pre_t = 0
# label_pre_t,select_pre_t=0,0
if iter_mode == 2:
raw_select_pre_t = raw_select_pre
print("training tagper model")
selected_idx = eug.select_top_data_vsm(
pred_score, dists, args.topk, vsm_lambda,
min(nums_to_select, len(u_data)))
_, _, raw_select_pre = eug.generate_new_train_data(
selected_idx, pred_y, step, args.experiment) # yml:新增后两个参数
# kf_file.write('{} {:.2%} {:.2%}'.format(step, label_pre, select_pre))
tagper.resume(osp.join(save_path, 'step_{}.ckpt'.format(step)),
step)
tagper.train(new_train_data,
unselected_data,
step,
loss=args.loss,
epochs=args.epochs,
step_size=args.step_size,
init_lr=0.1)
pred_y, pred_score, label_pre, dists = tagper.estimate_label_vsm()
selected_idx = tagper.select_top_data_vsm(
pred_score, dists, args.topk, vsm_lambda,
min(nums_to_select, len(u_data))) # 采样目标数量
new_train_data, unselected_data, select_pre = tagper.generate_new_train_data(
selected_idx, pred_y, step, args.experiment) # yml:新增后两个参数
t_label_pre, t_select_pre = label_pre, select_pre
label_pre, select_pre = t_label_pre, t_select_pre
if nums_to_select_tagper >= len(u_data):
iter_mode = 1 #切换模式
print('tagper is stop')
else: #mode = 1
# raw_select_pre = raw_select_pre_t
# raw_select_pre_t = 0
label_pre, select_pre = raw_label_pre, raw_select_pre
end_time = time.time()
step_time = end_time - start_time
total_time = step_time + total_time
train_time_file.write('{} {:.6} {:.6}\n'.format(
step, step_time, total_time))
kf_file.write('{} {} {} {:.2%} {:.2%} {:.2%} {:.2%} {:.2%}\n'.format(
step, nums_to_select, nums_to_select_tagper, raw_label_pre,
raw_select_pre, raw_select_pre_t, t_label_pre, t_select_pre))
dataf_file.write('{} {:.2%} {:.2%}\n'.format(step, label_pre,
select_pre))
dataf_file.close()
train_time_file.close()
