def get_loaders(opt):
""" Make dataloaders for train and validation sets
"""
# train loader
norm_method = Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
spatial_transform = Compose([
Scale((opt.sample_size, opt.sample_size)),
Resize(256),
CenterCrop(224),
ToTensor(), norm_method
])
temporal_transform = TemporalRandomCrop(25)
target_transform = ClassLabel()
training_data = get_training_set(opt, spatial_transform,
temporal_transform, target_transform)
train_loader = torch.utils.data.DataLoader(training_data,
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.num_workers,
pin_memory=True)
# validation loader
target_transform = ClassLabel()
temporal_transform = LoopPadding(25)
validation_data = get_validation_set(opt, spatial_transform,
temporal_transform, target_transform)
val_loader = torch.utils.data.DataLoader(validation_data,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.num_workers,
pin_memory=True)
return train_loader, val_loader
def get_loaders(opt): """ Make dataloaders for train and validation sets """ # train loader opt.mean = get_mean(opt.norm_value, dataset=opt.mean_dataset) if opt.no_mean_norm and not opt.std_norm: norm_method = Normalize([0, 0, 0], [1, 1, 1]) elif not opt.std_norm: norm_method = Normalize(opt.mean, [1, 1, 1]) else: norm_method = Normalize(opt.mean, opt.std) spatial_transform = Compose([ # crop_method, Scale((opt.sample_size, opt.sample_size)), # RandomHorizontalFlip(), ToTensor(opt.norm_value), norm_method ]) temporal_transform = TemporalRandomCrop(16) target_transform = ClassLabel() training_data = get_training_set(opt, spatial_transform, temporal_transform, target_transform) train_loader = torch.utils.data.DataLoader( training_data, batch_size=opt.batch_size, shuffle=True, num_workers=opt.num_workers, pin_memory=True) # validation loader spatial_transform = Compose([ Scale((opt.sample_size, opt.sample_size)), # CenterCrop(opt.sample_size), ToTensor(opt.norm_value), norm_method ]) target_transform = ClassLabel() temporal_transform = LoopPadding(16) validation_data = get_validation_set( opt, spatial_transform, temporal_transform, target_transform) val_loader = torch.utils.data.DataLoader( validation_data, batch_size=opt.batch_size, shuffle=False, num_workers=opt.num_workers, pin_memory=True) return train_loader, val_loader
spatial_transform = transforms.Compose([
augmentation.RandomHorizontalFlip(consistent=True),
augmentation.MultiScaleRandomCrop(args.scales,
args.sample_size,
consistent=True),
augmentation.RandomHorizontalFlip(consistent=True),
augmentation.ColorJitter(brightness=0.5,
contrast=0.5,
saturation=0.5,
hue=0.25,
p=0.3,
consistent=True),
augmentation.ToTensor(),
augmentation.Normalize()
])
temporal_transform = TemporalRandomCrop(args.sample_duration,
args.downsample)
target_transform = ClassLabel()
training_data = get_training_set(args, spatial_transform,
temporal_transform, target_transform)
# batchsampler = ContrastiveSampler(training_data.lookup, args.class_num, args.sample_num, len(training_data))
# train_loader = torch.utils.data.DataLoader(
# training_data,
# batch_sampler=batchsampler,
# num_workers=args.n_threads,
# pin_memory=True,
# )
train_loader = torch.utils.data.DataLoader(training_data,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.n_threads,
pin_memory=True)
if opt.train_crop == 'random':
crop_method = MultiScaleRandomCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'corner':
crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'center':
crop_method = MultiScaleCornerCrop(opt.scales,
opt.sample_size,
crop_positions=['c'])
elif opt.train_crop == 'custom':
crop_method = RandomSampleCrop(opt.sample_size)
clip_transform = None
spatial_transform = Compose(
[ToTensor(opt.norm_value),
ColorJitter(0.05, 0.05), norm_method])
temporal_transform = TemporalRandomCrop(
int(opt.sample_duration * opt.t_stride))
training_data = get_training_set(opt, spatial_transform,
temporal_transform)
train_loader = torch.utils.data.DataLoader(training_data,
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.n_threads,
pin_memory=True)
train_logger = Logger(os.path.join(opt.result_path, 'train.log'),
['epoch', 'loss', 'acc', 'lr'])
train_batch_logger = Logger(
os.path.join(opt.result_path, 'train_batch.log'),
['epoch', 'batch', 'iter', 'loss', 'acc', 'lr'])
if not opt.no_train:
assert opt.train_crop in ['random', 'corner', 'center']
if opt.train_crop == 'random':
crop_method = MultiScaleRandomCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'corner':
crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'center':
crop_method = MultiScaleCornerCrop(opt.scales,
opt.sample_size,
crop_positions=['c'])
spatial_transform = Compose([
crop_method,
RandomHorizontalFlip(),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = Compose([TemporalRandomCrop(opt.sample_duration)])
target_transform = ClassLabel()
training_data = get_training_set(opt, spatial_transform,
temporal_transform, target_transform)
train_loader = torch.utils.data.DataLoader(training_data,
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.n_threads,
pin_memory=True)
train_logger = Logger(os.path.join(opt.result_path, 'train.log'),
['epoch', 'loss', 'acc', 'lr'])
train_batch_logger = Logger(
os.path.join(opt.result_path, 'train_batch.log'),
['epoch', 'batch', 'iter', 'loss', 'acc', 'lr'])
if opt.nesterov:
def get_train_utils(opt, model_parameters):
assert opt.train_crop in ['random', 'corner', 'center']
spatial_transform = []
if opt.train_crop == 'random':
spatial_transform.append(
RandomResizedCrop(
opt.sample_size, (opt.train_crop_min_scale, 1.0),
(opt.train_crop_min_ratio, 1.0 / opt.train_crop_min_ratio)))
elif opt.train_crop == 'corner':
scales = [1.0]
scale_step = 1 / (2**(1 / 4))
for _ in range(1, 5):
scales.append(scales[-1] * scale_step)
spatial_transform.append(MultiScaleCornerCrop(opt.sample_size, scales))
elif opt.train_crop == 'center':
spatial_transform.append(Resize(opt.sample_size))
spatial_transform.append(CenterCrop(opt.sample_size))
normalize = get_normalize_method(opt.mean, opt.std, opt.no_mean_norm,
opt.no_std_norm)
if not opt.no_hflip:
spatial_transform.append(RandomHorizontalFlip())
spatial_transform.append(ToArray())
if opt.colorjitter:
spatial_transform.append(ColorJitter())
if opt.input_type == 'flow':
spatial_transform.append(PickFirstChannels(n=2))
spatial_transform.append(ScaleValue(opt.value_scale))
spatial_transform.append(normalize)
spatial_transform = Compose(spatial_transform)
assert opt.train_t_crop in ['random', 'center']
temporal_transform = []
if opt.sample_t_stride > 1:
temporal_transform.append(TemporalSubsampling(opt.sample_t_stride))
if opt.train_t_crop == 'random':
temporal_transform.append(TemporalRandomCrop(opt.sample_duration))
elif opt.train_t_crop == 'center':
temporal_transform.append(TemporalCenterCrop(opt.sample_duration))
temporal_transform = TemporalCompose(temporal_transform)
train_data = get_training_data(opt.video_path, opt.annotation_path,
opt.dataset, opt.input_type, opt.file_type,
spatial_transform, temporal_transform)
train_loader = paddle.batch(train_data.reader, batch_size=opt.batch_size)
train_logger = Logger(opt.result_path / 'train.log',
['epoch', 'loss', 'acc', 'lr'])
train_batch_logger = Logger(
opt.result_path / 'train_batch.log',
['epoch', 'batch', 'iter', 'loss', 'acc', 'lr'])
assert opt.lr_scheduler in ['plateau', 'multistep']
assert not (opt.lr_scheduler == 'plateau' and opt.no_val)
if opt.lr_scheduler == 'plateau':
scheduler = ReduceLROnPlateau(learning_rate=opt.learning_rate,
mode='min',
patience=opt.plateau_patience)
else:
scheduler = MultiStepDecay(learning_rate=opt.learning_rate,
milestones=opt.multistep_milestones)
optimizer = fluid.optimizer.MomentumOptimizer(
learning_rate=scheduler,
momentum=opt.momentum,
parameter_list=model_parameters,
use_nesterov=opt.nesterov,
regularization=fluid.regularizer.L2Decay(
regularization_coeff=opt.weight_decay))
return (train_loader, train_logger, train_batch_logger, optimizer,
scheduler)
norm_method = Normalize(args.mean, [1, 1, 1])
ucf_crop = MultiScaleRandomCrop(args.scales, args.sample_size)
ucf_spatial = Compose([
RandomHorizontalFlip(),
ucf_crop,
ToTensor(args.norm_value),
norm_method,
])
kinetics_spatial = Compose([
RandomHorizontalFlip(),
ucf_crop,
ToTensor(args.norm_value),
norm_method,
])
ucf_temporal = TemporalRandomCrop(args.sample_duration, args.downsample)
# ucf_temporal = TemporalCenterCrop(args.sample_duration, args.downsample)
# kinetics_temporal = TransformTwice(TemporalRandomCrop(args.sample_duration, args.downsample))
kinetics_temporal = TemporalRandomCrop(args.sample_duration,
args.downsample)
# kinetics_temporal = TemporalCenterCrop(args.sample_duration, args.downsample)
spatial_transform = [ucf_spatial, kinetics_spatial]
temporal_transform = [ucf_temporal, kinetics_temporal]
target_transform = [ClassLabel(), ClassLabel_fromarray(labels)]
# here the second is irrelevant since we don't use their labels
combined_dataset = UCF_with_Kinetics(args.l_vids_path,
args.l_annotation_path,
args.ul_vids_path,
args.ul_annotation_path,
def get_train_utils(opt, model_parameters):
assert opt.train_crop in ['random', 'corner', 'center']
spatial_transform = []
if opt.train_crop == 'random':
spatial_transform.append(
RandomResizedCrop(
opt.sample_size, (opt.train_crop_min_scale, 1.0),
(opt.train_crop_min_ratio, 1.0 / opt.train_crop_min_ratio)))
elif opt.train_crop == 'corner':
scales = [1.0]
scale_step = 1 / (2**(1 / 4))
for _ in range(1, 5):
scales.append(scales[-1] * scale_step)
spatial_transform.append(MultiScaleCornerCrop(opt.sample_size, scales))
elif opt.train_crop == 'center':
spatial_transform.append(Resize(opt.sample_size))
spatial_transform.append(CenterCrop(opt.sample_size))
normalize = get_normalize_method(opt.mean, opt.std, opt.no_mean_norm,
opt.no_std_norm)
if not opt.no_hflip:
spatial_transform.append(RandomHorizontalFlip())
if opt.colorjitter:
spatial_transform.append(ColorJitter())
spatial_transform.append(ToTensor())
if opt.input_type == 'flow':
spatial_transform.append(PickFirstChannels(n=2))
spatial_transform.append(ScaleValue(opt.value_scale))
spatial_transform.append(normalize)
spatial_transform = Compose(spatial_transform)
assert opt.train_t_crop in ['random', 'center']
temporal_transform = []
if opt.sample_t_stride > 1:
temporal_transform.append(TemporalSubsampling(opt.sample_t_stride))
if opt.train_t_crop == 'random':
temporal_transform.append(TemporalRandomCrop(opt.sample_duration))
elif opt.train_t_crop == 'center':
temporal_transform.append(TemporalCenterCrop(opt.sample_duration))
temporal_transform = TemporalCompose(temporal_transform)
train_data = get_training_data(opt.video_path, opt.annotation_path,
opt.dataset, opt.input_type, opt.file_type,
spatial_transform, temporal_transform)
if opt.distributed:
train_sampler = torch.utils.data.distributed.DistributedSampler(
train_data)
else:
train_sampler = None
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=opt.batch_size,
shuffle=(train_sampler is None),
num_workers=opt.n_threads,
pin_memory=True,
sampler=train_sampler,
worker_init_fn=worker_init_fn)
if opt.is_master_node:
train_logger = Logger(opt.result_path / 'train.log',
['epoch', 'loss', 'acc', 'lr'])
train_batch_logger = Logger(
opt.result_path / 'train_batch.log',
['epoch', 'batch', 'iter', 'loss', 'acc', 'lr'])
else:
train_logger = None
train_batch_logger = None
if opt.nesterov:
dampening = 0
else:
dampening = opt.dampening
optimizer = SGD(model_parameters,
lr=opt.learning_rate,
momentum=opt.momentum,
dampening=dampening,
weight_decay=opt.weight_decay,
nesterov=opt.nesterov)
assert opt.lr_scheduler in ['plateau', 'multistep']
assert not (opt.lr_scheduler == 'plateau' and opt.no_val)
if opt.lr_scheduler == 'plateau':
scheduler = lr_scheduler.ReduceLROnPlateau(
optimizer, 'min', patience=opt.plateau_patience)
else:
scheduler = lr_scheduler.MultiStepLR(optimizer,
opt.multistep_milestones)
return (train_loader, train_sampler, train_logger, train_batch_logger,
optimizer, scheduler)
if not opt.no_train:
assert opt.train_crop in ['random', 'corner', 'center']
if opt.train_crop == 'random':
crop_method = MultiScaleRandomCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'corner':
crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'center':
crop_method = MultiScaleCornerCrop(opt.scales,
opt.sample_size,
crop_positions=['c'])
spatial_transform = Compose([
crop_method,
RandomHorizontalFlip(),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = TemporalRandomCrop(opt.sample_duration *
9) # 9 segments
target_transform = ClassLabel()
training_data = get_training_set(opt, spatial_transform,
temporal_transform, target_transform)
train_loader = torch.utils.data.DataLoader(training_data,
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.n_threads,
pin_memory=True,
drop_last=True)
train_logger = Logger(os.path.join(opt.result_path, 'train.log'),
['epoch', 'loss', 'acc', 'lr'])
train_batch_logger = Logger(
os.path.join(opt.result_path, 'train_batch.log'),
['epoch', 'batch', 'iter', 'loss', 'acc', 'lr'])
def main():
opt = parse_opts()
ecd_name, cls_name = opt.model_name.split('-')
ecd_model = get_encoder_net(ecd_name)
cls_model = get_end_net(cls_name)
cfg.encoder_model = ecd_name
cfg.classification_model = cls_name
if opt.debug:
cfg.debug = opt.debug
else:
if opt.tensorboard == 'TEST':
cfg.tensorboard = opt.model_name
else:
cfg.tensorboard = opt.tensorboard
cfg.flag = opt.flag
model = cls_model(cfg,
encoder=CNNencoder(
cfg,
ecd_model(pretrained=True, path=opt.encoder_model)))
cfg.video_path = os.path.join(cfg.root_path, cfg.video_path)
cfg.annotation_path = os.path.join(cfg.root_path, cfg.annotation_path)
cfg.list_all_member()
torch.manual_seed(cfg.manual_seed)
print('##########################################')
print('####### model 仅支持单GPU')
print('##########################################')
model = model.cuda()
print(model)
criterion = nn.CrossEntropyLoss()
if cfg.cuda:
criterion = criterion.cuda()
norm_method = Normalize([0, 0, 0], [1, 1, 1])
print('##########################################')
print('####### train')
print('##########################################')
assert cfg.train_crop in ['random', 'corner', 'center']
if cfg.train_crop == 'random':
crop_method = (cfg.scales, cfg.sample_size)
elif cfg.train_crop == 'corner':
crop_method = MultiScaleCornerCrop(cfg.scales, cfg.sample_size)
elif cfg.train_crop == 'center':
crop_method = MultiScaleCornerCrop(cfg.scales,
cfg.sample_size,
crop_positions=['c'])
spatial_transform = Compose([
crop_method,
RandomHorizontalFlip(),
ToTensor(cfg.norm_value), norm_method
])
temporal_transform = TemporalRandomCrop(cfg.sample_duration)
target_transform = ClassLabel()
training_data = get_training_set(cfg, spatial_transform,
temporal_transform, target_transform)
train_loader = torch.utils.data.DataLoader(training_data,
batch_size=cfg.batch_size,
shuffle=True,
num_workers=cfg.n_threads,
drop_last=False,
pin_memory=True)
optimizer = model.get_optimizer(lr1=cfg.lr, lr2=cfg.lr2)
scheduler = lr_scheduler.ReduceLROnPlateau(optimizer,
'min',
patience=cfg.lr_patience)
print('##########################################')
print('####### val')
print('##########################################')
spatial_transform = Compose([
Scale(cfg.sample_size),
CenterCrop(cfg.sample_size),
ToTensor(cfg.norm_value), norm_method
])
temporal_transform = LoopPadding(cfg.sample_duration)
target_transform = ClassLabel()
validation_data = get_validation_set(cfg, spatial_transform,
temporal_transform, target_transform)
val_loader = torch.utils.data.DataLoader(validation_data,
batch_size=cfg.batch_size,
shuffle=False,
num_workers=cfg.n_threads,
drop_last=False,
pin_memory=True)
print('##########################################')
print('####### run')
print('##########################################')
if cfg.debug:
logger = None
else:
path = get_log_dir(cfg.logdir, name=cfg.tensorboard, flag=cfg.flag)
logger = Logger(logdir=path)
cfg.save_config(path)
for i in range(cfg.begin_epoch, cfg.n_epochs + 1):
train_epoch(i, train_loader, model, criterion, optimizer, cfg, logger)
validation_loss = val_epoch(i, val_loader, model, criterion, cfg,
logger)
scheduler.step(validation_loss)
def create_dataloader(args):
if args.root_path != '':
args.video_path = os.path.join(args.root_path, args.video_path)
args.annotation_path = os.path.join(args.root_path,
args.annotation_path)
args.result_path = os.path.join(args.root_path, args.result_path)
if args.resume_path:
args.resume_path = os.path.join(args.root_path, args.resume_path)
if args.pretrain_path:
# args.pretrain_path = os.path.join(args.root_path, args.pretrain_path)
args.pretrain_path = os.path.abspath(args.pretrain_path)
args.scales = [args.initial_scale]
for i in range(1, args.n_scales):
args.scales.append(args.scales[-1] * args.scale_step)
args.mean = get_mean(args.norm_value, dataset=args.mean_dataset)
args.std = get_std(args.norm_value)
if args.no_mean_norm and not args.std_norm:
norm_method = Normalize([0, 0, 0], [1, 1, 1])
elif not args.std_norm:
norm_method = Normalize(args.mean, [1, 1, 1])
else:
norm_method = Normalize(args.mean, args.std)
assert args.train_crop in ['random', 'corner', 'center']
if args.train_crop == 'random':
crop_method = MultiScaleRandomCrop(args.scales, args.sample_size)
elif args.train_crop == 'corner':
crop_method = MultiScaleCornerCrop(args.scales, args.sample_size)
elif args.train_crop == 'center':
crop_method = MultiScaleCornerCrop(args.scales,
args.sample_size,
crop_positions=['c'])
spatial_transform = Compose([
crop_method,
RandomHorizontalFlip(),
ToTensor(args.norm_value), norm_method
])
temporal_transform = TemporalRandomCrop(args.sample_duration)
target_transform = ClassLabel()
training_data = get_training_set(args, spatial_transform,
temporal_transform, target_transform)
train_loader = torch.utils.data.DataLoader(training_data,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.n_threads,
pin_memory=True)
spatial_transform = Compose([
# Scale(args.sample_size),
Scale(int(args.sample_size / args.scale_in_test)),
# CenterCrop(args.sample_size),
CornerCrop(args.sample_size, args.crop_position_in_test),
ToTensor(args.norm_value),
norm_method
])
temporal_transform = TemporalCenterCrop(args.sample_duration)
target_transform = ClassLabel()
validation_data = get_validation_set(args, spatial_transform,
temporal_transform, target_transform)
val_loader = torch.utils.data.DataLoader(validation_data,
batch_size=1,
shuffle=False,
num_workers=args.n_threads,
pin_memory=True)
return train_loader, val_loader
crop_method = MultiScaleRandomCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'corner':
crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'center':
crop_method = MultiScaleCornerCrop(opt.scales,
opt.sample_size,
crop_positions=['c'])
spatial_transform = Compose([
crop_method,
RandomHorizontalFlip(),
ToTensor(opt.norm_value), norm_method
])
if opt.train_temp_crop == 'sparse':
temp_crop_method = TemporalSparseSample(opt.sample_duration)
else:
temp_crop_method = TemporalRandomCrop(opt.sample_duration)
if opt.train_reverse:
temporal_transform = Compose(
[ReverseFrames(opt.sample_duration), temp_crop_method])
elif opt.train_shuffle:
temporal_transform = Compose(
[ShuffleFrames(opt.sample_duration), temp_crop_method])
else:
temporal_transform = temp_crop_method
target_transform = ClassLabel()
training_data = get_training_set(opt, spatial_transform,
temporal_transform, target_transform)
train_loader = torch.utils.data.DataLoader(training_data,
batch_size=opt.batch_size,
shuffle=True,
def main():
resnet_in = generate_model(opt)
resnet_in.module.fc = Identity()
model = ReNet34(resnet_in, encode_length=encode_length)
if opt.no_mean_norm and not opt.std_norm:
norm_method = Normalize([0, 0, 0], [1, 1, 1])
elif not opt.std_norm:
norm_method = Normalize(opt.mean, [1, 1, 1])
else:
norm_method = Normalize(opt.mean, opt.std)
if not opt.no_train:
assert opt.train_crop in ['random', 'corner', 'center']
if opt.train_crop == 'random':
crop_method = MultiScaleRandomCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'corner':
crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'center':
crop_method = MultiScaleCornerCrop(opt.scales,
opt.sample_size,
crop_positions=['c'])
## train loader
spatial_transform = Compose([
crop_method,
RandomHorizontalFlip(),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = TemporalRandomCrop(opt.sample_duration)
target_transform = ClassLabel()
training_data = get_training_set(opt, spatial_transform,
temporal_transform, target_transform)
train_loader = torch.utils.data.DataLoader(training_data,
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.n_threads,
pin_memory=True)
## test loader
spatial_transform = Compose([
Scale(int(opt.sample_size / opt.scale_in_test)),
CornerCrop(opt.sample_size, opt.crop_position_in_test),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = LoopPadding(opt.sample_duration)
target_transform = ClassLabel()
test_data = get_test_set(opt, spatial_transform, temporal_transform,
target_transform)
test_loader = torch.utils.data.DataLoader(test_data,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.n_threads,
pin_memory=True)
## Database loader
spatial_transform = Compose([
Scale(int(opt.sample_size / opt.scale_in_test)),
CornerCrop(opt.sample_size, opt.crop_position_in_test),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = LoopPadding(opt.sample_duration)
target_transform = ClassLabel()
validation_data = get_validation_set(opt, spatial_transform,
temporal_transform,
target_transform)
database_loader = torch.utils.data.DataLoader(
validation_data,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.n_threads,
pin_memory=True)
if opt.nesterov:
dampening = 0
else:
dampening = opt.dampening
optimizer = optim.SGD(model.parameters(),
lr=opt.learning_rate,
momentum=opt.momentum,
dampening=dampening,
weight_decay=opt.weight_decay,
nesterov=opt.nesterov)
scheduler = lr_scheduler.ReduceLROnPlateau(optimizer,
'min',
patience=opt.lr_patience)
if opt.resume_path:
print('loading checkpoint {}'.format(opt.resume_path))
checkpoint = torch.load(opt.resume_path)
assert opt.arch == checkpoint['arch']
opt.begin_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
if not opt.no_train:
optimizer.load_state_dict(checkpoint['optimizer'])
for state in optimizer.state.values():
for k, v in state.items():
if torch.is_tensor(v):
state[k] = v.cuda()
print('run')
for epoch in range(opt.begin_epoch, opt.n_epochs + 1):
model.cuda().train()
for i, (images, labels) in enumerate(train_loader):
images = Variable(images.cuda())
labels = Variable(labels.cuda().long())
# Forward + Backward + Optimize
optimizer.zero_grad()
x, _, b = model(images)
target_b = F.cosine_similarity(b[:int(labels.size(0) / 2)],
b[int(labels.size(0) / 2):])
target_x = F.cosine_similarity(x[:int(labels.size(0) / 2)],
x[int(labels.size(0) / 2):])
loss = F.mse_loss(target_b, target_x)
loss.backward()
optimizer.step()
scheduler.step()
# Test the Model
if (epoch + 1) % 10 == 0:
model.eval()
retrievalB, retrievalL, queryB, queryL = compress(
database_loader, test_loader, model)
result_map = calculate_top_map(qB=queryB,
rB=retrievalB,
queryL=queryL,
retrievalL=retrievalL,
topk=100)
print('--------mAP@100: {}--------'.format(result_map))
def objective(trial):
opt = parse_opts()
if trial:
opt.weight_decay = trial.suggest_uniform('weight_decay', 0.01, 0.1)
opt.learning_rate = trial.suggest_uniform('learning_rate', 1 - 5,
1 - 4)
if opt.root_path != '':
opt.video_path = os.path.join(opt.root_path, opt.video_path)
opt.annotation_path = os.path.join(opt.root_path, opt.annotation_path)
opt.result_path = os.path.join(opt.root_path, opt.result_path)
if opt.resume_path:
opt.resume_path = os.path.join(opt.root_path, opt.resume_path)
if opt.pretrain_path:
opt.pretrain_path = os.path.join(opt.root_path, opt.pretrain_path)
opt.scales = [opt.initial_scale]
for i in range(1, opt.n_scales):
opt.scales.append(opt.scales[-1] * opt.scale_step)
opt.arch = '{}-{}'.format(opt.model, opt.model_depth)
opt.mean = get_mean(opt.norm_value, dataset=opt.mean_dataset)
opt.std = get_std(opt.norm_value)
print(opt)
with open(os.path.join(opt.result_path, 'opts.json'), 'w') as opt_file:
json.dump(vars(opt), opt_file)
torch.manual_seed(opt.manual_seed)
model, parameters = generate_model(opt)
print(model)
criterion = nn.CrossEntropyLoss()
if not opt.no_cuda:
criterion = criterion.cuda()
if opt.no_mean_norm and not opt.std_norm:
norm_method = Normalize([0, 0, 0], [1, 1, 1])
elif not opt.std_norm:
norm_method = Normalize(opt.mean, [1, 1, 1])
else:
norm_method = Normalize(opt.mean, opt.std)
# norm_method = Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
if not opt.no_train:
assert opt.train_crop in ['random', 'corner', 'center']
if opt.train_crop == 'random':
crop_method = MultiScaleRandomCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'corner':
crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'center':
crop_method = MultiScaleCornerCrop(opt.scales,
opt.sample_size,
crop_positions=['c'])
spatial_transform = Compose([
crop_method,
RandomHorizontalFlip(),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = TemporalRandomCrop(opt.sample_duration)
target_transform = ClassLabel()
training_data = get_training_set(opt, spatial_transform,
temporal_transform, target_transform)
train_loader = torch.utils.data.DataLoader(
training_data,
batch_size=opt.batch_size,
# sampler option is mutually exclusive with shuffle
shuffle=False,
sampler=ImbalancedDatasetSampler(training_data),
num_workers=opt.n_threads,
pin_memory=True)
train_logger = Logger(os.path.join(opt.result_path, 'train.log'),
['epoch', 'loss', 'acc', 'lr'])
train_batch_logger = Logger(
os.path.join(opt.result_path, 'train_batch.log'),
['epoch', 'batch', 'iter', 'loss', 'acc', 'lr'])
optimizer = optim.Adam(parameters,
lr=opt.learning_rate,
weight_decay=opt.weight_decay)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
verbose=True,
factor=0.1**0.5)
if not opt.no_val:
spatial_transform = Compose([
Scale(opt.sample_size),
CenterCrop(opt.sample_size),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = LoopPadding(opt.sample_duration)
target_transform = ClassLabel()
validation_data = get_validation_set(opt, spatial_transform,
temporal_transform,
target_transform)
val_loader = torch.utils.data.DataLoader(
validation_data,
batch_size=opt.batch_size,
shuffle=False,
sampler=ImbalancedDatasetSampler(validation_data),
num_workers=opt.n_threads,
pin_memory=True)
val_logger = Logger(os.path.join(opt.result_path, 'val.log'),
['epoch', 'loss', 'acc'])
if opt.resume_path:
print('loading checkpoint {}'.format(opt.resume_path))
checkpoint = torch.load(opt.resume_path)
assert opt.arch == checkpoint['arch']
opt.begin_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
if not opt.no_train:
optimizer.load_state_dict(checkpoint['optimizer'])
print('run')
writer = SummaryWriter(
comment=
f"_wd{opt.weight_decay}_lr{opt.learning_rate}_ft_begin{opt.ft_begin_index}_pretrain{not opt.pretrain_path == ''}"
)
for i in range(opt.begin_epoch, opt.n_epochs + 1):
if not opt.no_train:
epoch, losses_avg, accuracies_avg = train_epoch(
i, train_loader, model, criterion, optimizer, opt,
train_logger, train_batch_logger)
writer.add_scalar('loss/train', losses_avg, epoch)
writer.add_scalar('acc/train', accuracies_avg, epoch)
if not opt.no_val:
epoch, val_losses_avg, val_accuracies_avg = val_epoch(
i, val_loader, model, criterion, opt, val_logger)
writer.add_scalar('loss/val', val_losses_avg, epoch)
writer.add_scalar('acc/val', val_accuracies_avg, epoch)
if not opt.no_train and not opt.no_val:
scheduler.step(val_losses_avg)
print('=' * 100)
if opt.test:
spatial_transform = Compose([
Scale(int(opt.sample_size / opt.scale_in_test)),
CornerCrop(opt.sample_size, opt.crop_position_in_test),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = LoopPadding(opt.sample_duration)
target_transform = VideoID()
test_data = get_test_set(opt, spatial_transform, temporal_transform,
target_transform)
test_loader = torch.utils.data.DataLoader(test_data,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.n_threads,
pin_memory=True)
test.test(test_loader, model, opt, test_data.class_names)
writer.close()
return val_losses_avg
norm_method = Normalize(args.mean, [1, 1, 1])
else:
norm_method = Normalize(args.mean, args.std)
spatial_transform = Compose([
Scale(args.sample_size),
CenterCrop(args.sample_size),
ToTensor(args.norm_value), norm_method
])
# if not args.test_temp_crop == 'sparse':
if args.compared_temp_transform == 'shuffle':
temp_transform = ShuffleFrames(args.sample_duration)
else:
temp_transform = ReverseFrames(args.sample_duration)
temp_crop_method = TemporalRandomCrop(args.sample_duration)
# if args.compared_temp_transform == 'reverse':
# temp_transform = Compose([
# ReverseFrames(args.sample_duration),
# temp_crop_method
# ])
# elif args.compared_temp_transform == 'shuffle':
# temp_transform = Compose([
# ShuffleFrames(args.sample_duration),
# temp_crop_method
# ])
# else:
# temp_transform = temp_crop_method
target_transform = ClassLabel()
# validation_data = get_validation_set(
# args, spatial_transform, temp_transform, target_transform,
# res = self.fc2(res)
# res = self.softmax(res)
# w = torch.arange(6).float().cuda()
# return torch.sum(res*w, dim=1)
# return torch.cosine_similarity(out, out2)
if __name__ == '__main__':
# preprocess('stsbenchmark/sts-train.csv','train')
# exit()
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
torch.manual_seed(1000)
# for data train:
temporal_transform = TemporalRandomCrop(16)#todo
training_data = stsbenchmark('stsbenchmark', 'sts-train.csv', temporal_transform,'train')
train_loader = torch.utils.data.DataLoader(
training_data,
batch_size=64,
shuffle=True,
num_workers=4,
pin_memory=True)
train_logger = Logger(
os.path.join('result', 'train.log'),
['epoch', 'loss', 'lr','cor'])
train_batch_logger = Logger(
os.path.join('result', 'train_batch.log'),
ucf_transform = Compose([
RandomHorizontalFlip(),
ucf_crop,
ToTensor(opt.norm_value), norm_method,
])
kinetics_transform = transforms.Compose([
transforms.Resize(128),
transforms.CenterCrop(112),
ToTensor(opt.norm_value), norm_method,
])
spatial_transform.append(ucf_transform)
spatial_transform.append(kinetics_transform)
temporal_transform.append(TemporalRandomCrop(opt.sample_duration, opt.downsample))
temporal_transform.append(TemporalCenterCrop(opt.sample_duration, opt.downsample))
target_transform.append(ClassLabel())
target_transform.append(ClassLabel_fromarray(labels))
kinetics_clustered = Kinetics_clustered(opt.ul_vids_path,
opt.ul_annotation_path,
'training',
1,
spatial_transform=spatial_transform[0],
temporal_transform=temporal_transform[0],
target_transform=target_transform[1])
train_loader = torch.utils.data.DataLoader(dataset=kinetics_clustered,
batch_size=opt.batch_size,
shuffle=True,
if not opt.no_train:
assert opt.train_crop in ['random', 'corner', 'center']
if opt.train_crop == 'random':
crop_method = MultiScaleRandomCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'corner':
crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'center':
crop_method = MultiScaleCornerCrop(opt.scales,
opt.sample_size,
crop_positions=['c'])
spatial_transform = Compose([
crop_method,
RandomHorizontalFlip(),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = TemporalRandomCrop(opt.sample_duration)
target_transform = ClassLabel()
training_data = get_training_set(opt, spatial_transform,
temporal_transform, target_transform)
train_loader = torch.utils.data.DataLoader(training_data,
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.n_threads,
pin_memory=True)
train_logger = Logger(os.path.join(opt.result_path, 'train.log'),
['epoch', 'loss', 'acc', 'lr'])
train_batch_logger = Logger(
os.path.join(opt.result_path, 'train_batch.log'),
['epoch', 'batch', 'iter', 'loss', 'acc', 'lr'])
if opt.nesterov:
def main():
opt = parse_opts()
# Path configurations
opt.annotation_path = os.path.join(opt.annotation_directory,
opt.annotation_path)
save_result_dir_name = \
os.path.join(opt.result_path,
get_prefix() + '_{}{}_{}_epochs'.format(opt.model, opt.model_depth, opt.n_epochs))
if not os.path.exists(save_result_dir_name):
os.mkdir(save_result_dir_name)
opt.result_path = os.path.join(opt.result_path, save_result_dir_name)
# For data generator
opt.scales = [opt.initial_scale]
for epoch in range(1, opt.n_scales):
opt.scales.append(opt.scales[-1] * opt.scale_step)
opt.arch = '{}-{}'.format(opt.model, opt.model_depth)
# Model
model, parameters = generate_model(opt)
# print(model)
# Loss function
criterion = nn.CrossEntropyLoss()
if not opt.no_cuda:
criterion = criterion.cuda()
# Normalizing
if not opt.no_mean_norm:
opt.mean = get_mean(opt.norm_value, dataset=opt.mean_dataset)
opt.std = get_std(opt.norm_value, dataset=opt.std_dataset)
norm_method = Normalize(opt.mean, opt.std)
else:
norm_method = Normalize([0, 0, 0], [1, 1, 1])
print(opt)
with open(os.path.join(opt.result_path, 'opts.json'), 'w') as opt_file:
json.dump(vars(opt), opt_file)
# **************************** TRAINING CONFIGURATIONS ************************************
assert opt.train_crop in ['corner', 'center']
if opt.train_crop == 'corner':
crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'center':
crop_method = MultiScaleCornerCrop(opt.scales,
opt.sample_size,
crop_positions=['c'])
# Пространственное преобразование
spatial_transform = Compose([
crop_method,
#RandomHorizontalFlip(),
ToTensor(opt.norm_value),
norm_method
])
# Временное преобразование
temporal_transform = TemporalRandomCrop(opt.sample_duration)
# Целевое преобразование
target_transform = ClassLabel()
train_loader_list = []
if not opt.no_cross_validation:
annotation_list = os.listdir(opt.annotation_directory)
for annotation in annotation_list:
opt.annotation_path = os.path.join(opt.annotation_directory,
annotation)
training_data = get_training_set(opt, spatial_transform,
temporal_transform,
target_transform)
train_loader = torch.utils.data.DataLoader(
training_data,
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.n_threads,
pin_memory=True)
train_loader_list.append(train_loader)
else:
training_data = get_training_set(opt, spatial_transform,
temporal_transform, target_transform)
train_loader = torch.utils.data.DataLoader(training_data,
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.n_threads,
pin_memory=True)
train_loader_list.append(train_loader)
train_logger = Logger(os.path.join(opt.result_path, 'train.log'),
['epoch', 'loss', 'acc', 'lr'])
train_batch_logger = Logger(
os.path.join(opt.result_path, 'train_batch.log'),
['epoch', 'batch', 'iter', 'loss', 'acc', 'lr'])
optimizer = optim.SGD(parameters,
lr=opt.learning_rate,
momentum=opt.momentum,
dampening=opt.dampening,
weight_decay=opt.weight_decay)
scheduler = lr_scheduler.ReduceLROnPlateau(optimizer,
'min',
patience=opt.lr_patience)
# ***************************** VALIDATION CONFIGURATIONS *********************************
spatial_transform = Compose([
Scale(opt.sample_size),
CenterCrop(opt.sample_size),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = LoopPadding(opt.sample_duration)
target_transform = ClassLabel()
val_loader_list = []
if not opt.no_cross_validation:
annotation_list = os.listdir(opt.annotation_directory)
for annotation in annotation_list:
opt.annotation_path = os.path.join(opt.annotation_directory,
annotation)
validation_data = get_validation_set(opt, spatial_transform,
temporal_transform,
target_transform)
val_loader = torch.utils.data.DataLoader(validation_data,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.n_threads,
pin_memory=True)
val_loader_list.append(val_loader)
else:
validation_data = get_validation_set(opt, spatial_transform,
temporal_transform,
target_transform)
val_loader = torch.utils.data.DataLoader(validation_data,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.n_threads,
pin_memory=True)
val_loader_list.append(val_loader)
val_logger = Logger(os.path.join(opt.result_path, 'val.log'),
['epoch', 'loss', 'acc'])
# **************************************** TRAINING ****************************************
epoch_avg_time = AverageMeter()
train_loss_list = []
train_acc_list = []
valid_acc_list = []
best_accuracy = 0
current_train_data = 0
current_valid_data = 0
opt.frequence_cross_validation = round(opt.n_epochs /
opt.n_cross_validation_sets + 0.5)
for epoch in range(opt.begin_epoch, opt.n_epochs + 1):
epoch_start_time = time.time()
print('Epoch #' + str(epoch))
# optimizer = regulate_learning_rate(optimizer, epoch, opt.frequence_regulate_lr)
train_loader = train_loader_list[current_train_data]
if not opt.no_cross_validation and epoch % opt.frequence_cross_validation == 0:
print('\t##### Cross-validation: switch training data #####')
current_train_data = (current_train_data +
1) % len(train_loader_list)
train_loader = train_loader_list[current_train_data]
train_loss, train_acc = train_epoch(epoch, train_loader, model,
criterion, optimizer, opt,
train_logger, train_batch_logger)
val_loader = val_loader_list[current_valid_data]
if not opt.no_cross_validation and epoch % opt.frequence_cross_validation == 0:
print('\t##### Cross-validation: switch validation data #####')
current_valid_data = (current_valid_data +
1) % len(val_loader_list)
val_loader = val_loader_list[current_valid_data]
validation_acc = val_epoch(epoch, val_loader, model, criterion, opt,
val_logger)
train_loss_list.append(train_loss)
train_acc_list.append(train_acc)
valid_acc_list.append(validation_acc)
# Save model with best accuracy
if validation_acc > best_accuracy:
best_accuracy = validation_acc
save_file_path = os.path.join(opt.result_path, 'best_model.pth')
states = {
'epoch': epoch + 1,
'arch': opt.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict()
}
torch.save(states, save_file_path)
epoch_end_time = time.time() - epoch_start_time
epoch_avg_time.update(epoch_end_time)
print('\tTime left: ' +
str(round(epoch_avg_time.avg *
(opt.n_epochs - epoch) / 60, 1)) + ' minutes')
# ******************************* SAVING RESULTS OF TRAINING ******************************
save_pictures(np.linspace(1, opt.n_epochs, opt.n_epochs),
train_loss_list, 'red', 'Loss',
os.path.join(opt.result_path, 'train_loss.png'))
save_pictures(np.linspace(1, opt.n_epochs, opt.n_epochs), train_acc_list,
'blue', 'Accuracy',
os.path.join(opt.result_path, 'train_accuracy.png'))
save_pictures(np.linspace(1, opt.n_epochs, opt.n_epochs), valid_acc_list,
'blue', 'Accuracy',
os.path.join(opt.result_path, 'validation_accuracy.png'))
