def get_video(video_path=
path_gray , total_frame_num=200):
loader = VideoLoader(video_path)
loader.set_frame_property(total_frame_num=total_frame_num)
frame_matrix = loader.video_to_frame(to_gray=True)
return frame_matrix
class GameViewer:
def __init__(self, video_file_name, note_file_name):
self.video_loader = VideoLoader(video_file_name)
self.w, self.h = self.video_loader.get_size()
note_loader = NoteLoader(note_file_name)
notes = note_loader.get_note_list()
self.note_evaluator = NoteEvaluator(notes)
def __del__(self):
self.frame = None
def start(self):
while True:
frame, time = self.video_loader.read()
if frame is None:
break
time = time + 1400
# self.note_evaluator.update(time)
self.show(frame, time)
k = cv2.waitKey(100)
if k == 27:
break
def show(self, frame, time):
note_list = self.note_evaluator.get_hit_note_list(time, 100)
for note in note_list:
pos = get_position(self.w, self.h, note.pos)
draw_command(frame, pos, note.label)
cv2.imshow('capture', frame)
def get_video(
video_path="G:/pycharm/data/Hollywood2_format/AVIClips/actionclipautoautotrain00078.avi",
total_frame_num=200):
loader = VideoLoader(video_path)
loader.set_frame_property(total_frame_num=total_frame_num)
frame_matrix = loader.video_to_frame()
return frame_matrix
def get_video(
video_path="G:/pycharm/data/gray1/person16_handclapping_d1_uncomp.avi",
total_frame_num=200):
loader = VideoLoader(video_path)
loader.set_frame_property(total_frame_num=total_frame_num)
frame_matrix = loader.video_to_frame(to_gray=True)
return frame_matrix
def main():
"""
Main function to extract features.
"""
args = parse_args()
cfg = load_config(args)
# Set random seed from configs.
np.random.seed(cfg.RNG_SEED)
th.manual_seed(cfg.RNG_SEED)
failed_log = open(args.csv.split(".csv")[0]+"_failed.txt", "w")
assert args.target_framerate % args.min_num_features == 0
preprocess = Preprocessing(
"3d", cfg, target_fps=args.target_framerate,
size=224, clip_len=args.clip_len, padding_mode='tile',
min_num_clips=args.min_num_features)
if args.dataflow:
readvideo = ReadVideo(
preprocess,
framerate=args.target_framerate,
size=224,
centercrop=True,
pix_fmt=args.pix_fmt,
overwrite=args.overwrite
)
dataset = VideoDataFlow(args.csv)
# dataset = MultiThreadMapData(
# dataset, num_thread=args.num_decoding_thread,
# map_func=readvideo,
# buffer_size=1000)
# loader = MultiProcessRunnerZMQ(
# dataset, num_proc=1)
loader = MultiProcessMapDataZMQ(
dataset, num_proc=args.num_decoding_thread,
map_func=readvideo, strict=True)
loader.reset_state()
n_dataset = len(dataset)
else:
dataset = VideoLoader(
args.csv,
preprocess,
framerate=args.target_framerate,
size=224,
centercrop=True,
pix_fmt=args.pix_fmt,
overwrite=args.overwrite
)
n_dataset = len(dataset)
sampler = RandomSequenceSampler(n_dataset, 10)
loader = DataLoader(
dataset,
batch_size=1,
shuffle=False,
num_workers=args.num_decoding_thread,
sampler=sampler if n_dataset > 10 else None,
)
model = build_model(cfg)
perform_test(
loader, model, preprocess, cfg, args, failed_log, n_dataset)
def extract(args, csv):
dataset = VideoLoader(
csv,
framerate=1 if args.type == '2d' else 24,
size=224 if args.type != '3d' else 112,
centercrop=(args.type != '2d'),
)
n_dataset = len(dataset)
sampler = RandomSequenceSampler(n_dataset, 10)
loader = DataLoader(
dataset,
batch_size=1,
shuffle=False,
num_workers=args.num_decoding_thread,
sampler=sampler if n_dataset > 10 else None,
)
preprocess = Preprocessing(args.type)
model = get_model(args)
dim = 512 #2048
with th.no_grad():
for k, data in enumerate(loader):
input_file = data['input'][0]
output_file = data['output'][0]
if len(data['video'].shape) > 3:
print('Computing features of video {}/{}: {}'.format(
k + 1, n_dataset, input_file))
video = data['video'].squeeze()
if len(video.shape) == 4:
video = preprocess(video)
n_chunk = len(video)
#features = th.cuda.FloatTensor(n_chunk, dim).fill_(0)
n_iter = int(math.ceil(n_chunk / float(args.batch_size)))
features = th.cuda.FloatTensor(n_iter, dim).fill_(0)
for i in range(n_iter):
min_ind = i #* args.batch_size
max_ind = (i + 1) #* args.batch_size
video_batch = video[min_ind* args.batch_size:max_ind* args.batch_size].cuda()
if args.type == "ig":
video_batch = video_batch.permute(1, 0, 2, 3).unsqueeze(0)
batch_features = model(video_batch)
#print(video_batch.shape, batch_features.shape, features.shape, video.shape)
else:
batch_features = model(video_batch)
if args.l2_normalize:
batch_features = F.normalize(batch_features, dim=1)
features[min_ind:max_ind] = batch_features
features = features.cpu().numpy()
if args.half_precision:
features = features.astype('float16')
np.save(output_file, features)
else:
print('Video {} already processed.'.format(input_file))
def main():
"""
Main function to extract features.
"""
args = parse_args()
failed_log = open(args.csv.split(".csv")[0]+"_failed.txt", "w")
preprocess = Preprocessing(
target_fps=args.target_framerate,
size=224, clip_len=args.clip_len, padding_mode='tile')
if args.dataflow:
readvideo = ReadVideo(
preprocess,
framerate=args.target_framerate,
size=224,
centercrop=True,
pix_fmt=args.pix_fmt,
overwrite=args.overwrite
)
dataset = VideoDataFlow(args.csv)
loader = MultiProcessMapDataZMQ(
dataset, num_proc=args.num_decoding_thread,
map_func=readvideo, strict=True)
loader.reset_state()
n_dataset = len(dataset)
else:
dataset = VideoLoader(
args.csv,
preprocess,
framerate=args.target_framerate,
size=224,
centercrop=True,
pix_fmt=args.pix_fmt,
overwrite=args.overwrite
)
n_dataset = len(dataset)
sampler = RandomSequenceSampler(n_dataset, 10)
loader = DataLoader(
dataset,
batch_size=1,
shuffle=False,
num_workers=args.num_decoding_thread,
sampler=sampler if n_dataset > 10 else None,
)
model = build_model(args)
perform_test(
loader, model, args, failed_log, n_dataset)
help='l2 normalize feature')
parser.add_argument('--resnext101_model_path',
type=str,
default='./model/resnext101.pth',
help='Resnext model path')
args = parser.parse_args()
for csv_path in csv_paths:
print(csv_path + ' began ...')
start_time = time.time()
dataset = VideoLoader(
csv=csv_path,
src_folder=src_folder,
dst_folder=dst_folder,
framerate=1 if args.type == '2d' else 24,
size=224 if args.type == '2d' else 112,
centercrop=(args.type == '3d'),
)
n_dataset = len(dataset)
sampler = RandomSequenceSampler(n_dataset, 10)
loader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=args.num_decoding_thread,
sampler=sampler if n_dataset > 10 else None,
pin_memory=True)
preprocess = Preprocessing(args.type)
model = get_model(args)
with th.no_grad():
help='Num parallel thread for video decoding')
parser.add_argument('--l2_normalize',
type=int,
default=1,
help='l2 normalize feature')
parser.add_argument('--resnext101_model_path',
type=str,
default='./3d_resnets/model/resnext101.pth',
help='Resnext model path')
args = parser.parse_args()
assert args.type == '2d', "Only 2d feature extraction is tested under this release"
dataset = VideoLoader(args.csv,
framerate=1 / args.clip_len if args.type == '2d' else 24,
size=224 if args.type == '2d' else 112,
centercrop=(args.type == '3d'),
overwrite=args.overwrite)
n_dataset = len(dataset)
sampler = RandomSequenceSampler(n_dataset, 10)
loader = DataLoader(
dataset,
batch_size=1,
shuffle=False,
num_workers=args.num_decoding_thread,
sampler=sampler if n_dataset > 10 else None,
)
preprocess = Preprocessing(args.type)
model = get_model(args)
totatl_num_frames = 0
type=int,
default=6,
help='Num parallel thread for video decoding')
parser.add_argument('--l2_normalize',
type=int,
default=1,
help='l2 normalize feature')
parser.add_argument('--resnext101_model_path',
type=str,
default='model/resnext101.pth',
help='Resnext model path')
args = parser.parse_args()
dataset = VideoLoader(
args.csv,
framerate=1 if args.type == '2d' else 24,
size=224 if args.type == '2d' else 112,
centercrop=(args.type == '3d'),
)
n_dataset = len(dataset)
sampler = RandomSequenceSampler(n_dataset, 10)
loader = DataLoader(
dataset,
batch_size=1,
shuffle=False,
num_workers=args.num_decoding_thread,
sampler=sampler if n_dataset > 10 else None,
)
preprocess = Preprocessing(args.type)
model = get_model(args)
feat_root = args.feat_root
with th.no_grad():
def __init__(self, video_file_name, note_file_name):
self.video_loader = VideoLoader(video_file_name)
self.w, self.h = self.video_loader.get_size()
note_loader = NoteLoader(note_file_name)
notes = note_loader.get_note_list()
self.note_evaluator = NoteEvaluator(notes)
def video_to_frame(self, video_path, total_frame_num):
loader = VideoLoader(video_path)
loader.set_frame_property(total_frame_num=total_frame_num)
frames_matrix = loader.video_to_frame(to_gray=self.to_gray)
return frames_matrix
help='allow overwrite output files')
parser.add_argument('--half_precision', type=int, default=1,
help='output half precision float')
parser.add_argument('--num_decoding_thread', type=int, default=4,
help='Num parallel thread for video decoding')
parser.add_argument('--model_version', type=str, default="ViT-B/32",
choices=["ViT-B/32", "RN50x4"],
help='Num parallel thread for video decoding')
args = parser.parse_args()
# model_version = "RN50x4" # "RN50x4" # "ViT-B/32"
output_feat_size = 512 if args.model_version == "ViT-B/32" else 640
dataset = VideoLoader(
args.csv,
framerate=1/args.clip_len,
size=224 if args.model_version == "ViT-B/32" else 288,
centercrop=True,
overwrite=args.overwrite,
model_version=args.model_version
)
n_dataset = len(dataset)
sampler = RandomSequenceSampler(n_dataset, 10)
loader = DataLoader(
dataset,
batch_size=1,
shuffle=False,
num_workers=args.num_decoding_thread,
sampler=sampler if n_dataset > 10 else None,
)
preprocess = Preprocessing()
model, _ = clip.load(args.model_version, device="cuda")
