def get_video_transform(data_name, split_name, opt):
normalizer = video_transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
t_list = []
if split_name == 'train':
t_list = [
video_transforms.RandomResizedCrop(opt.crop_size),
video_transforms.RandomHorizontalFlip(),
video_transforms.ColorJitter(brightness=0.1, contrast=0.1, hue=0.1)
]
else:
t_list = [
video_transforms.Resize(256),
video_transforms.CenterCrop(opt.crop_size)
]
t_end = [video_transforms.ToTensor(), normalizer]
transform = video_transforms.Compose(t_list + t_end)
return transform
def __init__(self, opt, train=True):
super(VideoDataset, self).__init__(opt, train=train)
# Dataroot & Transform
if opt.dataset == 'mgif':
data_root = './dataset/moving-gif'
vtrans = [vtransforms.Scale(size=128)]
elif opt.dataset == 'kth':
data_root = './dataset/kth_action/'
vtrans = [
vtransforms.CenterCrop(size=120),
vtransforms.Scale(size=128)
]
elif opt.dataset == 'penn':
data_root = './dataset/penn_action/'
vtrans = [vtransforms.Scale(size=128)]
if self.train:
vtrans += [vtransforms.RandomHorizontalFlip()]
vtrans += [vtransforms.RandomRotation()]
vtrans += [vtransforms.ToTensor(scale=True)]
vtrans += [vtransforms.Normalize(0.5, 0.5)] if opt.input_norm else []
self.vtrans = T.Compose(vtrans)
if self.train:
self.image_path = os.path.join(data_root, 'train')
else:
self.image_path = os.path.join(data_root, 'test')
threshold = self.window_size if opt.irregular else self.sample_size
if opt.dataset in ['kth', 'sintel', 'ucf101', 'penn']:
self.image_list = os.listdir(self.image_path)
elif opt.dataset in ['mgif', 'stickman']:
self.image_list = remove_files_under_sample_size(
image_path=self.image_path, threshold=threshold)
self.image_list = sorted(self.image_list)
assert CONFIG["RGB_I3D_LOAD_MODEL_PATH"] or CONFIG[
"FLOW_I3D_LOAD_MODEL_PATH"]
# Setup Dataset and Dataloader
if CONFIG["DATASET"] == "original":
Dataset = bbdb_dataset.OriginalBBDBDataset
elif CONFIG["DATASET"] == "binary":
Dataset = bbdb_dataset.BinaryBBDBDataset
else:
assert False
with open("data_split.min.json", "r") as fp:
data_split = json.load(fp)
test_transforms = transforms.Compose([
video_transforms.Resize(256),
video_transforms.CenterCrop(224),
])
dataset = Dataset(segment_filepaths=data_split["test"],
segment_length=CONFIG["SEGMENT_LENGTH"],
frameskip=CONFIG["FRAMESKIP"],
transform=test_transforms)
dataloader = DataLoader(dataset,
batch_size=CONFIG["BATCH_SIZE"],
pin_memory=True)
# Setup I3D
# TODO(seungjaeryanlee): Allow choosing both
if CONFIG["RGB_I3D_LOAD_MODEL_PATH"]:
rgb_i3d = InceptionI3d(400, in_channels=3)
rgb_i3d.replace_logits(dataset.NUM_LABELS)
rgb_i3d.load_state_dict(torch.load(CONFIG["RGB_I3D_LOAD_MODEL_PATH"]))
def main():
global args, best_prec1
args = parser.parse_args()
# create model
print("Building model ... ")
model = build_model()
print("Model %s is loaded. " % (args.modality + "_" + args.arch))
if not os.path.exists(args.resume):
os.makedirs(args.resume)
print("Saving everything to directory %s." % (args.resume))
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
cudnn.benchmark = True
# Data transforming
clip_mean = [0.485, 0.456, 0.406] * args.new_length
clip_std = [0.229, 0.224, 0.225] * args.new_length
normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std)
if args.modality == "rgb":
scale_ratios = [1.0, 0.875, 0.75, 0.66]
elif args.modality == "flow":
scale_ratios = [1.0, 0.875, 0.75]
else:
print("No such modality. Only rgb and flow supported.")
train_transform = video_transforms.Compose([
video_transforms.Scale((256)),
video_transforms.MultiScaleCrop((224, 224), scale_ratios),
video_transforms.RandomHorizontalFlip(),
video_transforms.ToTensor(),
normalize,
])
val_transform = video_transforms.Compose([
video_transforms.Scale((256)),
video_transforms.CenterCrop((224)),
video_transforms.ToTensor(),
normalize,
])
# data loading
train_setting_file = "train_%s_split%d.txt" % (args.modality, args.split)
train_split_file = os.path.join(args.settings, args.dataset,
train_setting_file)
val_setting_file = "val_%s_split%d.txt" % (args.modality, args.split)
val_split_file = os.path.join(args.settings, args.dataset,
val_setting_file)
if not os.path.exists(train_split_file) or not os.path.exists(
val_split_file):
print(
"No split file exists in %s directory. Preprocess the dataset first"
% (args.settings))
train_dataset = datasets.__dict__[args.dataset](
args.data,
train_split_file,
"train",
args.new_length,
video_transform=train_transform)
val_dataset = datasets.__dict__[args.dataset](
args.data,
val_split_file,
"val",
args.new_length,
video_transform=val_transform)
print('{} samples found, {} train samples and {} test samples.'.format(
len(val_dataset) + len(train_dataset), len(train_dataset),
len(val_dataset)))
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
if args.evaluate:
validate(val_loader, model, criterion)
return
for epoch in range(args.start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
# train for one epoch
train(train_loader, model, criterion, optimizer, epoch)
# evaluate on validation set
prec1 = validate(val_loader, model, criterion)
# remember best prec@1 and save checkpoint
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
if (epoch + 1) % args.save_freq == 0:
checkpoint_name = "%03d_%s" % (epoch + 1, "checkpoint.pth.tar")
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
}, is_best, checkpoint_name, args.resume)
def main():
global args, best_prec1
args = parser.parse_args()
print(args.modality + " network trained whith the split " +
str(args.split) + ".")
# create model
print("Building model ... ")
exits_model, model = build_model(int(args.start_epoch))
if not exits_model:
return
else:
print("Model %s is loaded. " % (args.arch))
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# create file where we allocate the models by each args.save_freq epochs
if not os.path.exists(args.resume):
os.makedirs(args.resume)
print("Saving everything to directory %s." % (args.resume))
cudnn.benchmark = True
# Data transforming
if args.modality == "rgb" or args.modality == "rhythm" or args.modality == "history":
is_color = True
scale_ratios = [1.0, 0.875, 0.75, 0.66]
clip_mean = [0.485, 0.456, 0.406] * args.new_length
clip_std = [0.299, 0.224, 0.225] * args.new_length
elif args.modality == "flow":
is_color = False
scale_ratios = [1.0, 0.875, 0.75]
clip_mean = [0.5, 0.5] * args.new_length
clip_std = [0.226, 0.226] * args.new_length
else:
print("No such modality. Only rgb and flow supported.")
new_size = 299 if args.arch == 'rgb_inception_v3' else 224
normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std)
train_transform = video_transforms.Compose([
#video_transforms.Scale((256)),
video_transforms.MultiScaleCrop((new_size, new_size), scale_ratios),
video_transforms.RandomHorizontalFlip(),
video_transforms.ToTensor(),
normalize,
])
val_transform = video_transforms.Compose([
# video_transforms.Scale((256)),
video_transforms.CenterCrop((new_size)),
video_transforms.ToTensor(),
normalize,
])
#createNewDataset("train_%s_split%d.txt" , "new_train.txt")
#createNewDataset("val_%s_split%d.txt", "new_test.txt")
# data loading
#train_setting_file = 'new_train.txt'
modality_ = "rgb" if (args.modality == "rhythm"
or args.modality == "history") else args.modality
train_setting_file = "train_%s_split%d.txt" % (modality_, args.split)
train_split_file = os.path.join(args.settings, args.dataset,
train_setting_file)
#val_setting_file = 'new_test.txt'
val_setting_file = "val_%s_split%d.txt" % (modality_, args.split)
val_split_file = os.path.join(args.settings, args.dataset,
val_setting_file)
if not os.path.exists(train_split_file) or not os.path.exists(
val_split_file):
print(
"No split file exists in %s directory. Preprocess the dataset first"
% (args.settings))
train_dataset = datasets.__dict__['dataset'](
root=args.data,
source=train_split_file,
phase="train",
modality=args.modality,
is_color=is_color,
new_length=args.new_length,
new_width=args.new_width,
new_height=args.new_height,
video_transform=train_transform)
val_dataset = datasets.__dict__['dataset'](root=args.data,
source=val_split_file,
phase="val",
modality=args.modality,
is_color=is_color,
new_length=args.new_length,
new_width=args.new_width,
new_height=args.new_height,
video_transform=val_transform)
print('{} samples found, {} train samples and {} test samples.'.format(
len(val_dataset) + len(train_dataset), len(train_dataset),
len(val_dataset)))
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
if args.evaluate:
validate(val_loader, model, criterion)
return
for epoch in range(args.start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
# train for one epoch
train(train_loader, model, criterion, optimizer, epoch)
# evaluate on validation set
prec1 = 0.0
if (epoch + 1) % args.save_freq == 0:
prec1 = validate(val_loader, model, criterion)
# remember best prec@1 and save checkpoint
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
if (epoch + 1) % args.save_freq == 0:
checkpoint_name = "%03d_%s" % (epoch + 1, "checkpoint_" +
args.modality + "_split_" +
str(args.split) + ".pth.tar")
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
}, is_best, checkpoint_name, args.resume)
import shutil
import math
import numpy as np
from PIL import Image
from tsn_dataset import TSNDataSet
from p3d_model import P3D199,get_optim_policies
import video_transforms
from tsn_models import TSN
from torch.nn.utils import clip_grad_norm
val_transform=video_transforms.Compose(
[
video_transforms.Resize((182,242)),
video_transforms.CenterCrop(160),
video_transforms.ToTensor(),
video_transforms.Normalize((0.485,0.456,0.406),
(0.229,0.224,0.225))]
)
val_loader=torch.utils.data.DataLoader(
TSNDataSet("","tsntest_01.lst",
num_segments=2,
new_length=16,
modality="RGB",
image_tmpl="frame{:06d}.jpg",
transform=val_transform,
random_shift=False),
batch_size=1,
shuffle=False,
def main():
global args, best_prec1, model, writer, best_loss, length, width, height, input_size, scheduler
args = parser.parse_args()
training_continue = args.contine
if '3D' in args.arch:
if 'I3D' in args.arch or 'MFNET3D' in args.arch:
if '112' in args.arch:
scale = 0.5
else:
scale = 1
else:
if '224' in args.arch:
scale = 1
else:
scale = 0.5
elif 'r2plus1d' in args.arch:
scale = 0.5
else:
scale = 1
print('scale: %.1f' % (scale))
input_size = int(224 * scale)
width = int(340 * scale)
height = int(256 * scale)
saveLocation = "./checkpoint/" + args.dataset + "_" + args.arch + "_split" + str(
args.split)
if not os.path.exists(saveLocation):
os.makedirs(saveLocation)
writer = SummaryWriter(saveLocation)
# create model
if args.evaluate:
print("Building validation model ... ")
model = build_model_validate()
optimizer = AdamW(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
elif training_continue:
model, startEpoch, optimizer, best_prec1 = build_model_continue()
for param_group in optimizer.param_groups:
lr = param_group['lr']
#param_group['lr'] = lr
print(
"Continuing with best precision: %.3f and start epoch %d and lr: %f"
% (best_prec1, startEpoch, lr))
else:
print("Building model with ADAMW... ")
model = build_model()
optimizer = AdamW(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
startEpoch = 0
if HALF:
model.half() # convert to half precision
for layer in model.modules():
if isinstance(layer, nn.BatchNorm2d):
layer.float()
print("Model %s is loaded. " % (args.arch))
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
criterion2 = nn.MSELoss().cuda()
scheduler = lr_scheduler.ReduceLROnPlateau(optimizer,
'min',
patience=5,
verbose=True)
print("Saving everything to directory %s." % (saveLocation))
if args.dataset == 'ucf101':
dataset = './datasets/ucf101_frames'
elif args.dataset == 'hmdb51':
dataset = './datasets/hmdb51_frames'
elif args.dataset == 'smtV2':
dataset = './datasets/smtV2_frames'
elif args.dataset == 'window':
dataset = './datasets/window_frames'
elif args.dataset == 'haa500_basketball':
dataset = './datasets/haa500_basketball_frames'
else:
print("No convenient dataset entered, exiting....")
return 0
cudnn.benchmark = True
modality = args.arch.split('_')[0]
if "3D" in args.arch or 'tsm' in args.arch or 'slowfast' in args.arch or 'r2plus1d' in args.arch:
if '64f' in args.arch:
length = 64
elif '32f' in args.arch:
length = 32
else:
length = 16
else:
length = 1
# Data transforming
if modality == "rgb" or modality == "pose":
is_color = True
scale_ratios = [1.0, 0.875, 0.75, 0.66]
if 'I3D' in args.arch:
if 'resnet' in args.arch:
clip_mean = [0.45, 0.45, 0.45] * args.num_seg * length
clip_std = [0.225, 0.225, 0.225] * args.num_seg * length
else:
clip_mean = [0.5, 0.5, 0.5] * args.num_seg * length
clip_std = [0.5, 0.5, 0.5] * args.num_seg * length
#clip_std = [0.25, 0.25, 0.25] * args.num_seg * length
elif 'MFNET3D' in args.arch:
clip_mean = [0.48627451, 0.45882353, 0.40784314
] * args.num_seg * length
clip_std = [0.234, 0.234, 0.234] * args.num_seg * length
elif "3D" in args.arch:
clip_mean = [114.7748, 107.7354, 99.4750] * args.num_seg * length
clip_std = [1, 1, 1] * args.num_seg * length
elif "r2plus1d" in args.arch:
clip_mean = [0.43216, 0.394666, 0.37645] * args.num_seg * length
clip_std = [0.22803, 0.22145, 0.216989] * args.num_seg * length
elif "rep_flow" in args.arch:
clip_mean = [0.5, 0.5, 0.5] * args.num_seg * length
clip_std = [0.5, 0.5, 0.5] * args.num_seg * length
elif "slowfast" in args.arch:
clip_mean = [0.45, 0.45, 0.45] * args.num_seg * length
clip_std = [0.225, 0.225, 0.225] * args.num_seg * length
else:
clip_mean = [0.485, 0.456, 0.406] * args.num_seg * length
clip_std = [0.229, 0.224, 0.225] * args.num_seg * length
elif modality == "pose":
is_color = True
scale_ratios = [1.0, 0.875, 0.75, 0.66]
clip_mean = [0.485, 0.456, 0.406] * args.num_seg
clip_std = [0.229, 0.224, 0.225] * args.num_seg
elif modality == "flow":
is_color = False
scale_ratios = [1.0, 0.875, 0.75, 0.66]
if 'I3D' in args.arch:
clip_mean = [0.5, 0.5] * args.num_seg * length
clip_std = [0.5, 0.5] * args.num_seg * length
elif "3D" in args.arch:
clip_mean = [127.5, 127.5] * args.num_seg * length
clip_std = [1, 1] * args.num_seg * length
else:
clip_mean = [0.5, 0.5] * args.num_seg * length
clip_std = [0.226, 0.226] * args.num_seg * length
elif modality == "both":
is_color = True
scale_ratios = [1.0, 0.875, 0.75, 0.66]
clip_mean = [0.485, 0.456, 0.406, 0.5, 0.5] * args.num_seg * length
clip_std = [0.229, 0.224, 0.225, 0.226, 0.226] * args.num_seg * length
else:
print("No such modality. Only rgb and flow supported.")
normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std)
if "3D" in args.arch and not ('I3D' in args.arch):
train_transform = video_transforms.Compose([
video_transforms.MultiScaleCrop((input_size, input_size),
scale_ratios),
video_transforms.RandomHorizontalFlip(),
video_transforms.ToTensor2(),
normalize,
])
val_transform = video_transforms.Compose([
video_transforms.CenterCrop((input_size)),
video_transforms.ToTensor2(),
normalize,
])
else:
train_transform = video_transforms.Compose([
video_transforms.MultiScaleCrop((input_size, input_size),
scale_ratios),
video_transforms.RandomHorizontalFlip(),
video_transforms.ToTensor(),
normalize,
])
val_transform = video_transforms.Compose([
video_transforms.CenterCrop((input_size)),
video_transforms.ToTensor(),
normalize,
])
# data loading
train_setting_file = "train_%s_split%d.txt" % (modality, args.split)
train_split_file = os.path.join(args.settings, args.dataset,
train_setting_file)
val_setting_file = "val_%s_split%d.txt" % (modality, args.split)
val_split_file = os.path.join(args.settings, args.dataset,
val_setting_file)
if not os.path.exists(train_split_file) or not os.path.exists(
val_split_file):
print(
"No split file exists in %s directory. Preprocess the dataset first"
% (args.settings))
train_dataset = datasets.__dict__[args.dataset](
root=dataset,
source=train_split_file,
phase="train",
modality=modality,
is_color=is_color,
new_length=length,
new_width=width,
new_height=height,
video_transform=train_transform,
num_segments=args.num_seg)
val_dataset = datasets.__dict__[args.dataset](
root=dataset,
source=val_split_file,
phase="val",
modality=modality,
is_color=is_color,
new_length=length,
new_width=width,
new_height=height,
video_transform=val_transform,
num_segments=args.num_seg)
print('{} samples found, {} train samples and {} test samples.'.format(
len(val_dataset) + len(train_dataset), len(train_dataset),
len(val_dataset)))
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
if args.evaluate:
prec1, prec3, _ = validate(val_loader, model, criterion, criterion2,
modality)
return
for epoch in range(startEpoch, args.epochs):
# if learning_rate_index > max_learning_rate_decay_count:
# break
# adjust_learning_rate(optimizer, epoch)
train(train_loader, model, criterion, criterion2, optimizer, epoch,
modality)
# evaluate on validation set
prec1 = 0.0
lossClassification = 0
if (epoch + 1) % args.save_freq == 0:
prec1, prec3, lossClassification = validate(
val_loader, model, criterion, criterion2, modality)
writer.add_scalar('data/top1_validation', prec1, epoch)
writer.add_scalar('data/top3_validation', prec3, epoch)
writer.add_scalar('data/classification_loss_validation',
lossClassification, epoch)
scheduler.step(lossClassification)
# remember best prec@1 and save checkpoint
is_best = prec1 >= best_prec1
best_prec1 = max(prec1, best_prec1)
# best_in_existing_learning_rate = max(prec1, best_in_existing_learning_rate)
#
# if best_in_existing_learning_rate > prec1 + 1:
# learning_rate_index = learning_rate_index
# best_in_existing_learning_rate = 0
if (epoch + 1) % args.save_freq == 0:
checkpoint_name = "%03d_%s" % (epoch + 1, "checkpoint.pth.tar")
if is_best:
print("Model works well")
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'best_loss': best_loss,
'optimizer': optimizer.state_dict(),
}, is_best, checkpoint_name, saveLocation)
checkpoint_name = "%03d_%s" % (epoch + 1, "checkpoint.pth.tar")
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'best_loss': best_loss,
'optimizer': optimizer.state_dict(),
}, is_best, checkpoint_name, saveLocation)
writer.export_scalars_to_json("./all_scalars.json")
writer.close()
def flow_test(param_model):
model = param_model
video_list = os.listdir(video)
f = open(video + "flow_result.txt", 'w')
for file in video_list:
if file.endswith("mp4"):
f.write(file + "\n")
# file_data = OrderedDict()
frame_count = 0
clip_mean = [0.5, 0.5] * 10
clip_std = [0.226, 0.226] * 10
normalize = video_transforms.Normalize(mean=clip_mean,
std=clip_std)
# config the transform to match the network's format
transform = video_transforms.Compose([
# video_transforms.Scale((256)),
video_transforms.CenterCrop((224)),
video_transforms.ToTensor(),
normalize,
])
# prepare the translation dictionary label-action
data_handler = UCF101_splitter(
os.getcwd() + '/datasets/ucf101_splits/', None)
data_handler.get_action_index()
class_to_idx = data_handler.action_label
idx_to_class = {v: k for k, v in class_to_idx.items()}
# Start looping on frames received from webcam
vs = cv2.VideoCapture(video + file)
softmax = torch.nn.Softmax()
nn_output = torch.FloatTensor(2 * 10, 224, 224)
count = 0
idx = 0
temp = ''
x = []
sampled_list = []
while (vs.isOpened()):
ret, image = vs.read()
if ret is False:
break
else:
image = cv2.resize(image, (342, 256),
interpolation=cv2.INTER_LINEAR)
x.append(temp)
if count == 11:
sampled_list = []
# input_var = torch.autograd.Variable(clip_input, volatile=True)
temp = ''
input_var = clip_input.view(1, 20, 224, 224).cuda()
output = model(input_var)
output = softmax(output)
output = output.data.cpu().numpy()
preds = output.argsort()[0][-5:][::-1]
pred_classes = [(idx_to_class[str(pred + 1)],
output[0, pred]) for pred in preds]
value = 0
for i in range(5):
if pred_classes[i][0] == label:
value = pred_classes[i][1]
temp += '{} - {:.2f}\n'.format(
pred_classes[i][0], pred_classes[i][1])
f.write(str(value) + "\n")
nn_output = torch.FloatTensor(2 * 10, 224, 224)
count = 1
if count == 0:
old_frame = image.copy()
prev = cv2.cvtColor(old_frame, cv2.COLOR_BGR2GRAY)
else:
next = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(
prev, next, 1, 0.5, 3, 15, 3, 5, 1.2, 0)
horz = cv2.normalize(flow[..., 0], None, 0, 255,
cv2.NORM_MINMAX)
vert = cv2.normalize(flow[..., 1], None, 0, 255,
cv2.NORM_MINMAX)
horz = horz.astype('uint8')
vert = vert.astype('uint8')
imgH = Image.fromarray(horz)
imgV = Image.fromarray(vert)
sampled_list.append(np.expand_dims(imgH, 2))
sampled_list.append(np.expand_dims(imgV, 2))
clip_input = np.concatenate(sampled_list, axis=2)
clip_input = transform(clip_input)
clip_input = clip_input.float().cuda(async=True)
imgH.close()
imgV.close()
prev = next.copy()
count += 1
idx += 1
f.write("----\n")
# file_data[file] = flow_value
# with open('flow.json', 'w', encoding="utf-8") as make_file:
# json.dump(file_data, make_file, ensure_ascii=False, indent="\t")
print(idx)
vs.release()
f.close()
def main(args):
global best_prec1, best_loss
input_size = int(224 * args.scale)
width = int(340 * args.scale)
height = int(256 * args.scale)
if not os.path.exists(args.savelocation):
os.makedirs(args.savelocation)
now = time.time()
savelocation = os.path.join(args.savelocation, str(now))
os.makedirs(savelocation)
logging.basicConfig(filename=os.path.join(savelocation, "log.log"),
level=logging.INFO)
model = build_model(args.arch, args.pre, args.num_seg, args.resume)
optimizer = AdamW(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
criterion = nn.CrossEntropyLoss().cuda()
criterion2 = nn.MSELoss().cuda()
scheduler = lr_scheduler.ReduceLROnPlateau(optimizer,
'min',
patience=5,
verbose=True)
# if args.dataset=='sign':
# dataset="/data/AUTSL/train_img_c"
# elif args.dataset=="signd":
# dataset="/data/AUTSL/train_img_c"
# elif args.dataset=="customd":
# dataset="/data/AUTSL/train_img_c"
# else:
# print("no dataset")
# return 0
cudnn.benchmark = True
length = 64
scale_ratios = [1.0, 0.875, 0.75, 0.66]
clip_mean = [0.43216, 0.394666, 0.37645] * args.num_seg * length
clip_std = [0.22803, 0.22145, 0.216989] * args.num_seg * length
normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std)
train_transform = video_transforms.Compose([
video_transforms.CenterCrop(input_size),
video_transforms.ToTensor2(),
normalize,
])
val_transform = video_transforms.Compose([
video_transforms.CenterCrop((input_size)),
video_transforms.ToTensor2(),
normalize,
])
# test_transform = video_transforms.Compose([
# video_transforms.CenterCrop((input_size)),
# video_transforms.ToTensor2(),
# normalize,
# ])
# test_file = os.path.join(args.datasetpath, args.testlist)
if not os.path.exists(args.trainlist) or not os.path.exists(args.vallist):
print(
"No split file exists in %s directory. Preprocess the dataset first"
% (args.datasetpath))
train_dataset = datasets.__dict__[args.dataset](
root=args.datasetpath,
source=args.trainlist,
phase="train",
modality="rgb",
is_color=True,
new_length=length,
new_width=width,
new_height=height,
video_transform=train_transform,
num_segments=args.num_seg)
val_dataset = datasets.__dict__[args.dataset](
root=args.datasetpath,
source=args.vallist,
phase="val",
modality="rgb",
is_color=True,
new_length=length,
new_width=width,
new_height=height,
video_transform=val_transform,
num_segments=args.num_seg)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
best_prec1 = 0
for epoch in range(0, args.epochs):
train(length, input_size, train_loader, model, criterion, criterion2,
optimizer, epoch)
if (epoch + 1) % args.save_freq == 0:
is_best = False
prec1, prec3, lossClassification = validate(
length, input_size, val_loader, model, criterion, criterion2)
scheduler.step(lossClassification)
if prec1 >= best_prec1:
is_best = True
best_prec1 = prec1
checkpoint_name = "%03d_%s" % (epoch + 1, "checkpoint.pth.tar")
text = "save checkpoint {}".format(checkpoint_name)
print(text)
logging.info(text)
save_checkpoint(
{
"epoch": epoch + 1,
"arch": args.arch,
"state_dict": model.state_dict(),
"prec1": prec1,
"optimizer": optimizer.state_dict()
}, is_best, checkpoint_name, savelocation)
def main():
global args, best_prec1
args = parser.parse_args()
# create model
print("Building model ... ")
model = build_model()
if torch.cuda.is_available():
model = torch.nn.DataParallel(model)
print("Model %s is loaded. " % (args.arch))
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
if not os.path.exists(args.resume):
os.makedirs(args.resume)
print("Saving everything to directory %s." % (args.resume))
cudnn.benchmark = True
# Data transforming
if args.modality == "rgb":
is_color = True
scale_ratios = [1.0, 0.875, 0.75, 0.66]
clip_mean = [0.485, 0.456, 0.406] * args.new_length
clip_std = [0.229, 0.224, 0.225] * args.new_length
elif args.modality == "flow":
is_color = False
scale_ratios = [1.0, 0.875, 0.75]
clip_mean = [0.5, 0.5] * args.new_length
clip_std = [0.226, 0.226] * args.new_length
else:
print("No such modality. Only rgb and flow supported.")
normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std)
train_transform = video_transforms.Compose([
# video_transforms.Scale((256)),
video_transforms.MultiScaleCrop((224, 224), scale_ratios),
video_transforms.RandomHorizontalFlip(),
video_transforms.ToTensor(),
normalize,
])
val_transform = video_transforms.Compose([
# video_transforms.Scale((256)),
video_transforms.CenterCrop((224)),
video_transforms.ToTensor(),
normalize,
])
# data loading
# train_setting_file = "train_%s_split%d.txt" % (args.modality, args.split)
# train_split_file = os.path.join(args.settings, args.dataset, train_setting_file)
# val_setting_file = "val_%s_split%d.txt" % (args.modality, args.split)
# val_split_file = os.path.join(args.settings, args.dataset, val_setting_file)
# if not os.path.exists(train_split_file) or not os.path.exists(val_split_file):
# print("No split file exists in %s directory. Preprocess the dataset first" % (args.settings))
train_split_file = './datasets/settings/train_set_detail.csv'
val_split_file = './datasets/settings/val_set_detail.csv'
train_dataset = datasets.__dict__[args.dataset](
root=args.data, #neet to change
source=train_split_file,
phase="train",
modality=args.modality,
is_color=is_color,
new_length=args.new_length,
new_width=args.new_width,
new_height=args.new_height,
video_transform=train_transform,
name_pattern="frame%06d.jpg") # frame000001
val_dataset = datasets.__dict__[args.dataset](
root=args.data,
source=val_split_file,
phase="val",
modality=args.modality,
is_color=is_color,
new_length=args.new_length,
new_width=args.new_width,
new_height=args.new_height,
video_transform=val_transform,
name_pattern="frame%06d.jpg")
print('{} samples found, {} train samples and {} test samples.'.format(
len(val_dataset) + len(train_dataset), len(train_dataset),
len(val_dataset)))
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
if args.evaluate:
validate(val_loader, model, criterion)
return
model_path = '/home/thl/Desktop/challeng/checkpoints/Mulity_100step_900epoch_batch80/model_best.pth.tar'
params = torch.load(model_path)
model.load_state_dict(params['state_dict'])
for epoch in range(args.start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
# train for one epoch
train(train_loader, model, criterion, optimizer, epoch)
# evaluate on validation set
prec1 = 0.0
if (epoch + 1) % args.save_freq == 0:
prec1 = validate(val_loader, model, criterion)
# remember best prec@1 and save checkpoint
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
if (epoch + 1) % args.save_freq == 0:
checkpoint_name = "%03d_%s" % (epoch + 1, "checkpoint.pth.tar")
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
}, is_best, checkpoint_name, args.resume)
def rgb_test(param_model):
model = param_model
f = open(video + "rgb_result.txt", 'w')
video_list = os.listdir(video)
for file in video_list:
if file.endswith("mp4"):
f.write(file + "\n")
frame_count = 2
clip_mean = [0.485, 0.456, 0.406] * 1
clip_std = [0.229, 0.224, 0.225] * 1
normalize = video_transforms.Normalize(mean=clip_mean,
std=clip_std)
# config the transform to match the network's format
transform = video_transforms.Compose([
# video_transforms.Scale((256)),
video_transforms.CenterCrop((224)),
video_transforms.ToTensor(),
normalize,
])
# prepare the translation dictionary label-action
data_handler = UCF101_splitter(
os.getcwd() + '/datasets/ucf101_splits/', None)
data_handler.get_action_index()
class_to_idx = data_handler.action_label
idx_to_class = {v: k for k, v in class_to_idx.items()}
# Start looping on frames received from webcam
vs = cv2.VideoCapture(video + file)
softmax = torch.nn.Softmax()
nn_output = torch.tensor(np.zeros((1, 23)),
dtype=torch.float32).cuda()
sampled_list = []
first_count = 0
while True:
# read each frame and prepare it for feedforward in nn (resize and type)
ret, orig_frame = vs.read()
if ret is False:
break
else:
orig_frame = cv2.resize(orig_frame, (342, 256),
interpolation=cv2.INTER_LINEAR)
frame = cv2.cvtColor(orig_frame, cv2.COLOR_BGR2RGB)
frame = transform(frame).view(1, 3, 224, 224).cuda()
frame = frame.float().cuda(async=True)
# feed the frame to the neural network
nn_output = model(frame)
# vote for class with 25 consecutive frames
if frame_count % 10 == 0:
nn_output = softmax(nn_output)
nn_output = nn_output.data.cpu().numpy()
preds = nn_output.argsort()[0][-5:][::-1]
pred_classes = [(idx_to_class[str(pred + 1)],
nn_output[0, pred]) for pred in preds]
red = (0, 0, 255)
green = (0, 255, 0)
blue = (255, 0, 0)
white = (255, 255, 255)
yellow = (0, 255, 255)
cyan = (255, 255, 0)
magenta = (255, 0, 255)
thickness = 2
center_x = int(342 / 2.0)
center_y = int(256 / 2.0)
location = (center_x - 170, center_y + 80)
fontScale = 1.5
font = cv2.FONT_HERSHEY_SIMPLEX
y0, dy = 180, 20
value = 0
for i in range(5):
y = y0 + i * dy
if pred_classes[i][0] == label:
value = pred_classes[i][1]
# reset the process
f.write(str(value) + "\n")
nn_output = torch.tensor(np.zeros((1, 23)),
dtype=torch.float32).cuda()
# cv2.imwrite("temp/" + str(frame_count) + ".jpg", orig_frame)
# Display the resulting frame and the classified action
frame_count += 1
# When everything done, release the capture
f.write("----\n")
vs.release()
f.close()
def main():
global best_prec1
# create model
print("Building model ... ")
print("Building model ... ", file = f_log)
model = build_model(resume_path = args.resume)
print("Model %s is loaded. " % (args.arch))
print("Model %s is loaded. " % (args.arch), file = f_log)
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(), args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
cudnn.benchmark = True
# Data transforming
if args.modality == "rgb":
is_color = True
scale_ratios = [1.0, 0.875, 0.75, 0.66]
clip_mean = [0.485, 0.456, 0.406]
clip_std = [0.229, 0.224, 0.225]
elif args.modality == "tvl1_flow" or args.modality == "lk_flow":
is_color = False
scale_ratios = [1.0, 0.875, 0.75]
clip_mean = [0.5, 0.5]
clip_std = [0.226, 0.226]
else:
print("No such modality. Only rgb and flow supported.")
print("No such modality. Only rgb and flow supported.", file = f_log)
normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std)
train_transform = video_transforms.Compose([
#video_transforms.Scale((288)),
video_transforms.MultiScaleCrop((256, 256), scale_ratios),
video_transforms.RandomHorizontalFlip(),
video_transforms.ToTensor(),
normalize,
])
val_transform = video_transforms.Compose([
#video_transforms.Scale((288)),
video_transforms.CenterCrop((256)),
video_transforms.ToTensor(),
normalize,
])
# data loading
train_setting_file = "train_%s_split%d.txt" % (args.modality, args.split)
train_split_file = os.path.join(args.settings, args.dataset, train_setting_file)
val_setting_file = "val_%s_split%d.txt" % (args.modality, args.split)
val_split_file = os.path.join(args.settings, args.dataset, val_setting_file)
if not os.path.exists(train_split_file) or not os.path.exists(val_split_file):
print("No split file exists in %s directory. Preprocess the dataset first" % (args.settings))
print("No split file exists in %s directory. Preprocess the dataset first" % (args.settings), file = f_log )
train_dataset = datasets.__dict__[args.dataset](setting=train_split_file, root=args.data, train=True,
new_width=args.new_width, new_height=args.new_height, new_length=args.new_length,
target_width=args.new_width, target_height=args.new_height,
modality=args.modality, num_segments=args.num_segments, transform=train_transform,
name_pattern='frame%06d.jpg')
val_dataset = datasets.__dict__[args.dataset](setting=val_split_file, root=args.data, train=False,
new_width=args.new_width, new_height=args.new_height, new_length=args.new_length,
target_width=args.new_width, target_height=args.new_height,
modality=args.modality, num_segments=args.num_segments, transform=val_transform,
name_pattern='frame%06d.jpg')
print('{} samples found, {} train samples and {} test samples.'.format(len(val_dataset)+len(train_dataset),
len(train_dataset),
len(val_dataset)))
print('{} samples found, {} train samples and {} test samples.'.format(len(val_dataset)+len(train_dataset),
len(train_dataset),
len(val_dataset)), file = f_log )
train_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=args.batch_size, shuffle=True,
num_workers=args.workers) #, pin_memory=True)
val_loader = torch.utils.data.DataLoader(
val_dataset,
batch_size=args.batch_size, shuffle=True,
num_workers=args.workers) #, pin_memory=True)
if args.evaluate:
validate(val_loader, model, criterion)
return
for epoch in range(args.start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
# train for one epoch
print("start epoch ", epoch)
train(train_loader, model, criterion, optimizer, epoch)
# evaluate on validation set
prec1 = 0.0
if (epoch + 1) % args.save_freq == 0:
prec1 = validate(val_loader, model, criterion)
# remember best prec@1 and save checkpoint
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
if (epoch + 1) % args.save_freq == 0:
checkpoint_name = "%03d_%s" % (epoch + 1, "checkpoint.pth.tar")
save_checkpoint({
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer' : optimizer.state_dict(),
}, is_best, checkpoint_name, args.save_path)
def main():
global args, prec_list
prec_list = []
args = parser.parse_args()
full_path = logging(args)
print(args.modality + " network trained with the split " +
str(args.split) + ".")
# create model
print("Building model ... ")
exits_model, model = build_model(int(args.start_epoch),
args.pretrain_weights)
if not exits_model:
return
else:
print("Model %s is loaded. " % (args.arch))
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
cudnn.benchmark = True
# Data transforming
if args.modality == "rgb" or args.modality == "rgb2":
is_color = True
scale_ratios = [1.0, 0.875, 0.75, 0.66]
clip_mean = [0.485, 0.456, 0.406] * args.new_length
clip_std = [0.229, 0.224, 0.225] * args.new_length
elif args.modality == "flow" or args.modality == "rhythm":
is_color = False
scale_ratios = [1.0, 0.875, 0.75]
clip_mean = [0.5, 0.5] * args.new_length
clip_std = [0.226, 0.226] * args.new_length
else:
print("No such modality. Only rgb and flow supported.")
new_size = 299 if args.arch.find("inception_v3") > 0 else 224
normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std)
train_transform = video_transforms.Compose([
#video_transforms.Scale((256)),
video_transforms.MultiScaleCrop((new_size, new_size), scale_ratios),
video_transforms.RandomHorizontalFlip(),
video_transforms.ToTensor(),
normalize,
])
if args.es:
val_transform = video_transforms.Compose([
# video_transforms.Scale((256)),
video_transforms.CenterCrop((new_size)),
video_transforms.ToTensor(),
normalize,
])
modality_ = "rgb" if (args.modality == "rhythm"
or args.modality[:3] == "rgb") else "flow"
if args.modality == "rgb2":
createNewDataset("train_split%d.txt", "new_train.txt", modality_)
#createNewDataset("val_%s_split%d.txt", "new_val.txt",modality_)
# data loading
train_setting_file = "new_train.txt" if args.modality == "rgb2" else "train_split%d.txt" % (
args.split)
train_split_file = os.path.join(args.settings, args.dataset,
train_setting_file)
if not os.path.exists(
train_split_file): # or not os.path.exists(val_split_file):
print(
"No split file exists in %s directory. Preprocess the dataset first"
% (args.settings))
extension = ".png" if args.dataset == "hmdb51" and args.modality == "rhythm" else ".jpg"
direction_file = "direction.txt" if args.vr_approach == 3 else "direction_video.txt"
direction_path = os.path.join(args.settings, args.dataset, direction_file)
train_dataset = datasets.__dict__['dataset'](
root=args.data,
source=train_split_file,
phase="train",
modality=args.modality,
is_color=is_color,
new_length=args.new_length,
new_width=args.new_width,
new_height=args.new_height,
video_transform=train_transform,
approach_VR=args.vr_approach,
extension=extension,
direction_path=direction_path)
if args.es:
val_setting_file = "val_split%d.txt" % (args.split)
val_split_file = os.path.join(args.settings, args.dataset,
val_setting_file)
if not os.path.exists(val_split_file):
print(
"No split file exists in %s directory. Preprocess the dataset first"
% (args.settings))
val_dataset = datasets.__dict__['dataset'](
root=args.data,
source=val_split_file,
phase="val",
modality=args.modality,
is_color=is_color,
new_length=args.new_length,
new_width=args.new_width,
new_height=args.new_height,
video_transform=val_transform,
approach_VR=args.vr_approach,
extension=extension,
direction_path=direction_path)
print('{} samples found, {} train samples and {} validation samples.'.
format(
len(val_dataset) + len(train_dataset), len(train_dataset),
len(val_dataset)))
else:
print('{} train samples found.'.format(len(train_dataset)))
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
if args.es:
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
early_stop = EarlyStopping(verbose=True,
log_path=os.path.join(
full_path, "early_stopping.json"))
is_best = False
for epoch in range(args.start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
# train for one epoch
train(train_loader, model, criterion, optimizer, epoch)
if args.es:
# evaluate on validation set
losses = validate(val_loader, model, criterion)
is_best = early_stop(losses.avg, epoch)
if (epoch + 1) % args.save_freq == 0 or is_best:
checkpoint_name = "%03d_%s" % (epoch + 1, "checkpoint_" +
args.modality + "_split_" +
str(args.split) + ".pth.tar")
es_val = float('inf') if not args.es else early_stop.val_loss_min
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
'val_loss_min': es_val
}, is_best, checkpoint_name,
os.path.join(full_path, "checkpoints"))
prec_name = "%03d_%s" % (epoch + 1,
"prec_split_" + str(args.split) + ".txt")
save_precision(prec_name, os.path.join(full_path, "precision"))
if args.es and early_stop.early_stop:
break
if not args.es: # Final model
checkpoint_name = "%03d_%s" % (epoch + 1, "checkpoint_" +
args.modality + "_split_" +
str(args.split) + ".pth.tar")
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
'val_loss_min': float('inf')
}, True, checkpoint_name, os.path.join(full_path, "checkpoints"))
def main():
global args
args = parser.parse_args()
if args.dataset == 'ucf101':
num_categories = 101
elif args.dataset == 'hmdb51':
num_categories = 51
elif args.dataset == 'kinetics':
num_categories = 400
else:
raise ValueError('Unknown dataset ' + args.dataset)
start_frame = 0
model_start_time = time.time()
params = torch.load(args.weights)
#hard code
net = models.__dict__[args.arch](pretrained=False,
num_classes=num_categories)
net.load_state_dict(params['state_dict'])
net.cuda()
net.eval()
model_end_time = time.time()
model_time = model_end_time - model_start_time
print("Action recognition model is loaded in %4.4f seconds." %
(model_time))
###
if args.modality == "rgb":
new_length = 1
is_color = True
scale_ratios = [1.0, 0.875, 0.75, 0.66]
clip_mean = [0.485, 0.456, 0.406] * new_length
clip_std = [0.229, 0.224, 0.225] * new_length
elif args.modality == "flow":
new_length = 10
is_color = False
scale_ratios = [1.0, 0.875, 0.75]
clip_mean = [0.5, 0.5] * new_length
clip_std = [0.226, 0.226] * new_length
normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std)
val_transform = video_transforms.Compose([
video_transforms.GroupCenterCrop(net.input_size),
video_transforms.CenterCrop((224)),
video_transforms.ToTensor(),
normalize,
])
dataset = datasets.load_clip(root=args.data,
source=args.test_list,
phase="val",
modality=args.modality,
is_color=is_color,
new_length=new_length,
new_width=args.new_width,
new_height=args.new_height,
video_transform=val_transform)
data_loader = torch.utils.data.DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=args.workers * 2,
pin_memory=True)
data_gen = enumerate(data_loader)
total_num = len(data_loader.dataset)
output = []
def eval_video(video_data):
i, data, label = video_data
num_crop = 1
if args.modality == 'rgb':
length = 3
elif args.modality == 'flow':
length = 10
elif args.modality == 'RGBDiff':
length = 18
else:
raise ValueError("Unknown modality " + args.modality)
input_var = torch.autograd.Variable(data.view(-1, length, data.size(2),
data.size(3)),
volatile=True)
input_var = input_var.type(torch.FloatTensor).cuda()
rst = net(input_var).data.cpu().numpy().copy()
return i, rst.reshape(
(num_crop, args.test_segments,
num_categories)).mean(axis=0).reshape(
(args.test_segments, 1, num_categories)), label[0]
proc_start_time = time.time()
max_num = args.max_num if args.max_num > 0 else len(data_loader.dataset)
for i, (data, label) in data_gen:
data = data.float().cuda(async=True)
label = label.cuda(async=True)
input_var = torch.autograd.Variable(data, volatile=True)
target_var = torch.autograd.Variable(label, volatile=True)
rst = net(input_var).data.cpu().numpy()
#avg_pred_fc8 = np.mean(rst, axis=1)
# print(avg_spatial_pred_fc8.shape)
#result_list.append(avg_pred_fc8)
# avg_spatial_pred = softmax(avg_spatial_pred_fc8)
pred_index = np.argmax(rst)
#print (label.cpu().numpy())
#print (pred_index)
# print(rst)
# if i >= max_num:
# break
# rst = eval_video((i, data, label))
output.append(rst)
if label.cpu().numpy()[0] == pred_index:
match_count += 1
cnt_time = time.time() - proc_start_time
print('video {} done, total {}/{}, average {} sec/video'.format(
i, i + 1, total_num,
float(cnt_time) / (i + 1)))
print(match_count)
print(total_num)
print("Accuracy is %4.4f" % (float(match_count) / float(total_num)))
np.save("{}_sX_{}_{}.npy".format(args.dataset, args.modality, args.arch),
np.array(output)) #hard code
width = int(340 * scale)
height = int(256 * scale)
length = 32
clip_mean = [0.5, 0.5, 0.5] * length
clip_std = [0.5, 0.5, 0.5] * length
scale_ratios = [1.0, 0.875, 0.75, 0.66]
normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std)
train_transform = video_transforms.Compose([
video_transforms.MultiScaleCrop((input_size, input_size), scale_ratios),
video_transforms.RandomHorizontalFlip(),
video_transforms.ToTensor(),
normalize,
])
val_transform = video_transforms.Compose([
video_transforms.CenterCrop((input_size)),
video_transforms.ToTensor(),
normalize,
])
#define dataset and dataloader
train_split_file = "./datasets/settings/haa500_instruments/train_rgb_split1.txt"
val_split_file = "./datasets/settings/haa500_instruments/val_rgb_split1.txt"
dataset_name = "haa500_instruments"
dataset_root = "./datasets/haa500_instruments"
train_dataset = haa500_instruments(train_split_file=train_split_file,
dataset_root=dataset_root,
phase="train",
new_length=length,
new_width=width,
