def __getitem__(self, index): if self._representation == 'mv': representation_idx = 1 elif self._representation == 'residual': representation_idx = 2 else: representation_idx = 0 if self._is_train: video_path, label, num_frames = random.choice(self._video_list) else: video_path, label, num_frames = self._video_list[index] frames = [] for seg in range(self._num_segments): if self._is_train: gop_index, gop_pos = self._get_train_frame_index(num_frames, seg) else: gop_index, gop_pos = self._get_test_frame_index(num_frames, seg) img = load(video_path, gop_index, gop_pos, representation_idx, self._accumulate) if img is None: print('Error: loading video %s failed.' % video_path) img = np.zeros((256, 256, 2)) if self._representation == 'mv' else np.zeros((256, 256, 3)) else: if self._representation == 'mv': img = clip_and_scale(img, 20) img += 128 img = (np.minimum(np.maximum(img, 0), 255)).astype(np.uint8) elif self._representation == 'residual': img += 128 img = (np.minimum(np.maximum(img, 0), 255)).astype(np.uint8) if self._representation == 'iframe': img = color_aug(img) # BGR to RGB. (PyTorch uses RGB according to doc.) img = img[..., ::-1] frames.append(img) frames = self._transform(frames) frames = np.array(frames) frames = np.transpose(frames, (0, 3, 1, 2)) input = torch.from_numpy(frames).float() / 255.0 if self._representation == 'iframe': input = (input - self._input_mean) / self._input_std elif self._representation == 'residual': input = (input - 0.5) / self._input_std elif self._representation == 'mv': input = (input - 0.5) return input, label
def load_segment(is_train, num_frames, seg, representation, num_segments, video_path, representation_idx, accumulate): if is_train: gop_index, gop_pos = get_train_frame_index(num_frames, seg, representation, num_segments) else: gop_index, gop_pos = get_test_frame_index(num_frames, seg, representation, num_segments) img = load(video_path, gop_index, gop_pos, representation_idx, accumulate) if img is None: print('Error: loading video %s failed.' % video_path) img = np.zeros((256, 256, 2)) if representation == 'mv' else np.zeros( (256, 256, 3)) else: if representation == 'mv': img = clip_and_scale(img, 20) img += 128 img = (np.minimum(np.maximum(img, 0), 255)).astype(np.uint8) elif representation == 'residual': img += 128 img = (np.minimum(np.maximum(img, 0), 255)).astype(np.uint8) if representation == 'iframe': if is_train: img = color_aug(img) # BGR to RGB. (PyTorch uses RGB according to doc.) img = img[..., ::-1] return img
def process_segment_consecutive(self, frames, gop_index, gop_pos, video_path,representation_idx): # if self._is_train: # gop_index, gop_pos = self._get_train_frame_index(num_frames, seg) # else: # gop_index, gop_pos = self._get_test_frame_index(num_frames, seg) # returns image of the specified frame img = load(video_path, gop_index, gop_pos, representation_idx, self._accumulate) if img is None: print('Error: loading video %s failed.' % video_path) img = np.zeros((256, 256, 2)) if self._representation == 'mv' else np.zeros((256, 256, 3)) else: if self._representation == 'mv': img = clip_and_scale(img, 20) img += 128 img = (np.minimum(np.maximum(img, 0), 255)).astype(np.uint8) elif self._representation == 'residual': img += 128 img = (np.minimum(np.maximum(img, 0), 255)).astype(np.uint8) if self._representation == 'iframe': img = color_aug(img) # BGR to RGB. (PyTorch uses RGB according to doc.) img = img[..., ::-1] frames.append(img) return frames
def __getitem__(self, index): video_path = self.data['video_path'][index] gop_index = self.data['gop_index'][index] target = self.data['targets'][index] # print(video_path, gop_index, target) if self._representation == 'iframe': frames_i = [] img_i = load(video_path, gop_index, 0, 0, self._accumulate) img_i = color_aug(img_i) img_i = img_i[..., ::-1] frames_i.append(img_i) frames_i = self._transform(frames_i) frames_i = np.array(frames_i) frames_i = np.transpose(frames_i, (0, 3, 1, 2)) input_i = torch.from_numpy(frames_i).float() / 255.0 input_i = (input_i - self._input_mean) / self._input_std input = input_i if self._representation == 'mv': frames_m = [] img_m = load(video_path, gop_index, 6, 1, self._accumulate) img_m = clip_and_scale(img_m, 20) img_m += 128 img_m = (np.minimum(np.maximum(img_m, 0), 255)).astype(np.uint8) frames_m.append(img_m) frames_m = self._transform(frames_m) frames_m = np.array(frames_m) frames_m = np.transpose(frames_m, (0, 3, 1, 2)) input_m = torch.from_numpy(frames_m).float() / 255.0 input_m = (input_m - 0.5) input = input_m if self._representation == 'r': frames_r = [] img_r = load(video_path, gop_index, 6, 2, self._accumulate) img_r += 128 img_r = (np.minimum(np.maximum(img_r, 0), 255)).astype(np.uint8) frames_r.append(img_r) frames_r = self._transform(frames_r) frames_r = np.array(frames_r) frames_r = np.transpose(frames_r, (0, 3, 1, 2)) input_r = torch.from_numpy(frames_r).float() / 255.0 input_r = (input_r - 0.5) / self._input_std input = input_r # print(input.shape) # target = target.long() # print(target.shape) return input, target
def __getitem__(self, index): frames = [] tmp = self._frames[index] for _, img in enumerate(tmp): if img is None: print('Error: loading video failed.') img = np.zeros((256, 256, 2)) if self._representation == 'mv' else np.zeros((256, 256, 3)) else: if self._representation == 'mv': img = clip_and_scale(img, 20) img += 128 img = (np.minimum(np.maximum(img, 0), 255)).astype(np.uint8) elif self._representation == 'residual': img += 128 img = (np.minimum(np.maximum(img, 0), 255)).astype(np.uint8) if self._representation == 'iframe': img = color_aug(img) # BGR to RGB. (PyTorch uses RGB according to doc.) img = img[..., ::-1] frames.append(img) frames = self._transform(frames) frames = np.array(frames) frames = np.transpose(frames, (0, 3, 1, 2)) input = torch.from_numpy(frames).float() / 255.0 if self._representation == 'iframe': input = (input - self._input_mean) / self._input_std elif self._representation == 'residual': input = (input - 0.5) / self._input_std elif self._representation == 'mv': input = (input - 0.5) return input
def __getitem__(self, index): if self._representation == 'mv': representation_idx = 1 elif self._representation == 'residual': representation_idx = 2 else: representation_idx = 0 if self._is_train: video_path, label, num_frames = random.choice(self._video_list) else: video_path, label, num_frames = self._video_list[index] frames = [] idx_first = -99999 for seg in range(self._num_segments): if self._is_train: gop_index, gop_pos = self._get_train_frame_index( num_frames, seg) else: gop_index, gop_pos = self._get_test_frame_index( num_frames, seg) flow_path = video_path_to_flow_path(self._flow_root, video_path) if self._flow_folder == 'tvl1': flow_tmpl = 'flow_{0}_{1:05d}.jpg' idx = gop_index * GOP_SIZE + gop_pos + 1 if idx_first == -99999: idx_first = idx # read the corresponding pre-computed optical flow along x and y dimension x_img = np.array( Image.open(os.path.join(flow_path, flow_tmpl.format('x', idx))).convert('L')) y_img = np.array( Image.open(os.path.join(flow_path, flow_tmpl.format('y', idx))).convert('L')) flow = np.stack([x_img, y_img], axis=-1) if flow is None: print('Error: loading flow %s failed.' % video_path) # load MV and data pre-processing mv = load(video_path, gop_index, gop_pos, representation_idx, self._accumulate) if mv is None: print('Error: loading video %s failed.' % video_path) mv = np.zeros( (256, 256, 2)) if self._representation == 'mv' else np.zeros( (256, 256, 3)) else: if self._representation == 'mv': if self._mv_minmaxnorm == 1: mv = clip_and_scale( mv, 20) # scale values from +-20 to +-127.5 mv += 128 mv = (np.minimum(np.maximum(mv, 0), 255)).astype(np.uint8) elif self._representation == 'residual': mv += 128 mv = (np.minimum(np.maximum(mv, 0), 255)).astype(np.uint8) if self._representation == 'iframe': mv = color_aug(mv) # BGR to RGB. (PyTorch uses RGB according to doc.) mv = mv[..., ::-1] # load residual and data pre-processing residual = load(video_path, gop_index, gop_pos, 2, self._accumulate) residual += 128 residual = (np.minimum(np.maximum(residual, 0), 255)).astype(np.uint8) frames.append(np.concatenate((flow, mv, residual), axis=2)) frames = self._transform(frames) frames = np.array(frames) frames = np.transpose(frames, (0, 3, 1, 2)) # print('frames shape in dataloader:') # print(frames.shape) # (num_crops*num_segments, 5, 224, 224) # split input into input_mv and input_flow input_flow = frames[:, 0:2, :, :] input_mv = frames[:, 2:4, :, :] input_residual = frames[:, 4:, :, :] if self._flow_ds_factor is not 0: # downsample to make OF blocky factor = self._flow_ds_factor w_max = input_flow.shape[2] h_max = input_flow.shape[3] input_flow = block_reduce(input_flow, block_size=(1, 1, factor, factor), func=np.mean) # resize to original size by repeating or interpolation if self._upsample_interp is False: input_flow = input_flow.repeat(factor, axis=2).repeat(factor, axis=3) else: # interpolate along certain dimension? only interp1d can do so w_max_ds = input_flow.shape[2] h_max_ds = input_flow.shape[3] f_out = interpolate.interp1d(np.linspace(0, 1, w_max_ds), input_flow, kind='linear', axis=2) input_flow = f_out(np.linspace(0, 1, w_max_ds * factor)) f_out = interpolate.interp1d(np.linspace(0, 1, h_max_ds), input_flow, kind='linear', axis=3) input_flow = f_out(np.linspace(0, 1, h_max_ds * factor)) input_flow = input_flow[:, :, :w_max, :h_max] """load data from numpy to torch and pre-processing""" # print('input_flow shape in dataloader:') # print(input_flow.shape) # (num_crops*num_segments, 2, 224, 224) input_flow = torch.from_numpy(input_flow).float() / 255.0 input_mv = torch.from_numpy(input_mv).float() / 255.0 input_residual = torch.from_numpy(input_residual).float() / 255.0 # print('input_flow after torch shape in dataloader:') # print(input_flow.shape) # torch.Size([num_crops*num_segments, 2, 224, 224]) if self._representation == 'iframe': input_mv = (input_mv - self._input_mean) / self._input_std elif self._representation == 'mv': input_mv = (input_mv - 0.5) / torch.mean(self._input_std) input_flow = (input_flow - 0.5) / torch.mean(self._input_std) input_residual = (input_residual - 0.5) / self._input_std # print('Input flow shape %s:' % str(input_flow.shape)) # torch.Size([1, num_crops*num_segments, 2, 224, 224]) # print('Input mv shape %s:' % str(input_mv.shape)) # print('Input residual shape %s:' % str(input_residual.shape)) # print('Input mv scope min %s:' % str(input_mv.min())) # print('Input mv scope max %s:' % str(input_mv.max())) # print('Input flow scope min %s:' % str(input_flow.min())) # print('Input flow scope max %s:' % str(input_flow.max())) if (self._viz == True) and (self._is_train == False): classname = flow_path.split('/')[-2] img_tmpl = 'img_{:05d}.jpg' # idx is the index of the first frame/segment of the current video return input_flow, input_mv, input_residual, label, os.path.join( flow_path, flow_tmpl.format('x', idx)), os.path.join( flow_path, flow_tmpl.format('y', idx)), os.path.join( flow_path, img_tmpl.format(idx_first)), classname else: return input_flow, input_mv, input_residual, label
def __getitem__(self, index): if self._is_train: video_path, label, num_frames = random.choice(self._video_list) else: video_path, label, num_frames = self._video_list[index] num_gop = num_frames // GOP_SIZE for gop in range(num_gop): frames_i = [] frames_m = [] frames_r = [] img_i = load(video_path, gop, 0, 0, self._accumulate) img_i = color_aug(img_i) img_i = img_i[..., ::-1] img_m = load(video_path, gop, 6, 1, self._accumulate) img_m = clip_and_scale(img_m, 20) img_m += 128 img_m = (np.minimum(np.maximum(img_m, 0), 255)).astype(np.uint8) img_r = load(video_path, gop, 6, 2, self._accumulate) img_r += 128 img_r = (np.minimum(np.maximum(img_r, 0), 255)).astype(np.uint8) frames_i.append(img_i) frames_m.append(img_m) frames_r.append(img_r) frames_i = self._transform_i(frames_i) frames_m = self._transform_m(frames_m) frames_r = self._transform_r(frames_r) frames_i = np.array(frames_i) frames_m = np.array(frames_m) frames_r = np.array(frames_r) frames_i = np.transpose(frames_i, (0, 3, 1, 2)) frames_m = np.transpose(frames_m, (0, 3, 1, 2)) frames_r = np.transpose(frames_r, (0, 3, 1, 2)) input_i = torch.from_numpy(frames_i).float() / 255.0 input_m = torch.from_numpy(frames_m).float() / 255.0 input_r = torch.from_numpy(frames_r).float() / 255.0 input_i = (input_i - self._input_mean) / self._input_std input_m = (input_m - 0.5) input_r = (input_r - 0.5) / self._input_std # print(input_i.shape) # a=input_i.view((-1, ) + input_i.size()[-3:]) # print(a.shape) # print(input_m.shape) # print(input_r.shape) input1 = torch.cat((input_i, input_m, input_r), 1) # print(input1.shape) if gop == 0: input = input1 else: input = torch.cat((input, input1), 0) # print(input.shape) # a=input.view((-1, ) + input.size()[-3:]) # print(a.shape) # print(input) return input, label
def __getitem__(self, index): # siamese label, '0' means same, '1' means diff siamese_label = 0 if self._representation == 'mv': representation_idx = 1 elif self._representation == 'residual': representation_idx = 2 else: representation_idx = 0 index_1 = 0 index_2 = 0 if self._is_train: # 我们需要构建一个正负样本 index_1 = random.randint(0, self._size - 1) index_2 = 0 if index % 2 == 0: # construct a POSTIVE pair indices = np.squeeze( np.argwhere( self._labels_list == self._labels_list[index_1])) index_2 = random.choice(indices) siamese_label = 0 else: # construct a NEG pair indices = np.squeeze( np.argwhere( self._labels_list != self._labels_list[index_1])) index_2 = random.choice(indices) siamese_label = 1 else: # 为了检测性能,这里还是需要用均衡的数据来验证,但是要确保每次生成的对是一样的 index_1 = index index_2 = 0 if index % 2 == 0: # construct a POSTIVE pair indices = np.squeeze( np.argwhere( self._labels_list == self._labels_list[index_1])) random.seed(index_1) index_2 = random.choice(indices) siamese_label = 0 else: # construct a NEG pair indices = np.squeeze( np.argwhere( self._labels_list != self._labels_list[index_1])) random.seed(index_1) index_2 = random.choice(indices) siamese_label = 1 frames_all = [] # divide into segments, then fetch a frame in every seg for i in (index_1, index_2): frames_per_sample = [] # TODO summary: bug a few time , convert i to int video_path = self._videos_list[int(i)] for seg in range(self._num_segments): if self._is_train: gop_index, gop_pos = self._get_train_frame_index( self._frames_list[i], seg) else: gop_index, gop_pos = self._get_test_frame_index( self._frames_list[i], seg) # load(input.mp4, 3, 8, 1, True) # returns the accumulated motion vectors of the 9th frame of the 4th GOP. img = load(video_path, gop_index, gop_pos, representation_idx, self._accumulate) if img is None: print('Error: loading video %s failed.' % video_path) img = np.zeros( (256, 256, 2)) if self._representation == 'mv' else np.zeros( (256, 256, 3)) else: if self._representation == 'mv': img = clip_and_scale(img, 20) img += 128 img = (np.minimum(np.maximum(img, 0), 255)).astype(np.uint8) elif self._representation == 'residual': img += 128 img = (np.minimum(np.maximum(img, 0), 255)).astype(np.uint8) if self._representation == 'iframe': img = color_aug(img) # BGR to RGB. (PyTorch uses RGB according to doc.) img = img[..., ::-1] frames_per_sample.append(img) frames_per_sample = self._transform(frames_per_sample) frames_per_sample = np.array(frames_per_sample) frames_per_sample = np.transpose(frames_per_sample, (0, 3, 1, 2)) input = torch.from_numpy(frames_per_sample).float() / 255.0 # when mv, the num_segments=3 # The input shape is: # torch.Size([3, 2, 224, 224]) # The input label is: 37 if self._representation == 'iframe': input = (input - self._input_mean) / self._input_std # was a bug here, input_mean,std is a tensor ,so we must convert input a tensor elif self._representation == 'residual': input = (input - 0.5) / self._input_std elif self._representation == 'mv': input = (input - 0.5) frames_all.append(input) return frames_all, siamese_label
def __getitem__(self, index): if self._representation == 'mv': representation_idx = 1 elif self._representation == 'residual': representation_idx = 2 else: representation_idx = 0 if self._is_train: video_path, label, num_frames = random.choice(self._video_list) else: video_path, label, num_frames = self._video_list[index] frames = [] for seg in range(self._num_segments): if self._is_train: gop_index, gop_pos = self._get_train_frame_index( num_frames, seg) else: gop_index, gop_pos = self._get_test_frame_index( num_frames, seg) img = load(video_path, gop_index, gop_pos, representation_idx, self._accumulate) if img is None: print('Error: loading video %s failed.' % video_path) img = np.zeros( (256, 256, 2)) if self._representation == 'mv' else np.zeros( (256, 256, 3)) else: if self._representation == 'mv': img = clip_and_scale(img, 20) img += 128 img = (np.minimum(np.maximum(img, 0), 255)).astype(np.uint8) elif self._representation == 'residual': img += 128 img = (np.minimum(np.maximum(img, 0), 255)).astype(np.uint8) if self._representation == 'iframe': img = color_aug(img) # BGR to RGB. (PyTorch uses RGB according to doc.) img = img[..., ::-1] frames.append(img) frames = self._transform(frames) frames = np.array(frames) frames = np.transpose(frames, (0, 3, 1, 2)) input = torch.from_numpy(frames).float() / 255.0 if self._representation == 'iframe': input = (input - self._input_mean) / self._input_std elif self._representation == 'residual': input = (input - 0.5) / self._input_std elif self._representation == 'mv': input = (input - 0.5) return input, label