def __init__(self, dataset_name='SumMe',split='train', clip_size=50, output3=False): if dataset_name.lower() not in ['summe', 'tvsum']: print('Unrecognized dataset {:s}'.format(dataset_name)) sys.exit(-1) self.dataset_name = dataset_name self.split = split self.filenames = datasetpaths[self.dataset_name.lower()].file_names self.data_loader = data_loaders[self.dataset_name.lower()] n_files = len(self.filenames) if self.split == 'train': self.filenames = self.filenames[:int(0.8*n_files)] elif self.split == 'val': self.filenames = self.filenames[int(0.8*n_files):] else: print("Unrecognized split:{:s}".format(self.split)) sys.exit(-1) self.clip_size = clip_size self.output3 = output3 Kydataset = KyLoader.loadKyDataset(self.dataset_name.lower()) conversion = KyLoader.loadConversion(self.dataset_name.lower()) self.raw2Ky = conversion[0] self.Ky2raw = conversion[1] self.features = {} self.avg_summary = {} self.annotations = [] pbar = progressbar.ProgressBar(max_value=len(self.filenames)) print("Processing {:s}\t{:s} data".format(self.dataset_name, self.split)) for file_dix, s_filename in enumerate(self.filenames): pbar.update(file_dix) Kykey = self.raw2Ky[s_filename] s_features, _, _ = self.data_loader.load_by_name(s_filename) s_usersummaries = Kydataset[Kykey]['user_summary'][...] s_usersummaries = s_usersummaries.transpose() self.features[s_filename] = s_features s_avg_summary = np.mean(s_usersummaries, axis=1) self.avg_summary[s_filename] = s_avg_summary s_video_len = len(s_avg_summary) s_segments = LoaderUtils.convertlabels2segs(s_usersummaries) for s_seg in s_segments: sSegment = LoaderUtils.Segment(s_filename) s_score = np.mean(s_avg_summary[s_seg[0]:s_seg[1]]) sSegment.initId(s_seg[0], s_seg[1], s_video_len, score=s_score) self.annotations.append(sSegment) self.augmented_ratio = 10 self.dataset_size = len(self.annotations) print("{:s}\t{:s}\t{:d}".format(self.dataset_name, self.split, self.dataset_size))
def __init__(self, dataset_name='SumMe', split='train', clip_size=100, sum_budget=0.15, sample_rate=None, overlap_ratio=0.25): if dataset_name.lower() not in ['summe', 'tvsum']: print('Unrecognized dataset {:s}'.format(dataset_name)) self.dataset_name = dataset_name self.eval_metrics = eval_metrics[self.dataset_name.lower()] self.split = split self.sum_budget = sum_budget self.filenames = datasetpaths[self.dataset_name.lower()].file_names self.data_loader = data_loaders[self.dataset_name.lower()] n_files = len(self.filenames) if self.split == 'train': self.filenames = self.filenames[:int(0.8 * n_files)] elif self.split == 'val': self.filenames = self.filenames[int(0.8 * n_files):] else: print("Unrecognized split:{:s}".format(self.split)) self.overlap_ratio = overlap_ratio self.clip_size = clip_size if sample_rate is None: self.sample_rate = [1, 5, 10] else: if isinstance(sample_rate, list): self.sample_rate = sample_rate else: self.sample_rate = [sample_rate] self.videofeatures = [] self.groundtruthscores = [] self.segments = [] self.videonames = [] Kydataset = KyLoader.loadKyDataset(self.dataset_name.lower()) conversion = KyLoader.loadConversion(self.dataset_name.lower()) self.raw2Ky = conversion[0] self.Ky2raw = conversion[1] for s_video_idx, s_filename in enumerate(self.filenames): KyKey = self.raw2Ky[s_filename] s_video_features, _, _ = self.data_loader.load_by_name(s_filename) s_scores = Kydataset[KyKey]['user_summary'][...] s_scores = s_scores.transpose() s_segments = LoaderUtils.convertlabels2segs(s_scores) self.groundtruthscores.append(s_scores) self.videofeatures.append(s_video_features) self.segments.append(s_segments) self.videonames.append(s_filename) self.dataset_size = len(self.videofeatures) print("{:s}\tEvaluator: {:s}\t{:d} Videos".format( self.dataset_name, self.split, self.dataset_size))
def __init__(self, dataset_name='TVSum', split='train', seq_length=90, overlap=0.9, sample_rate=None, feature_file_ext='npy', sum_budget=0.15, train_val_perms=None, eval_metrics='avg', data_path=None): if dataset_name.lower() not in ['summe', 'tvsum']: print('Unrecognized dataset {:s}'.format(dataset_name)) self.dataset_name = dataset_name self.eval_metrics = eval_metrics #[self.dataset_name.lower()] self.split = split self.sum_budget = sum_budget self.feature_file_ext = feature_file_ext self.feature_directory = os.path.join( data_path, '%s/features/c3dd-red500' % (dataset_name)) self.filenames = os.listdir(self.feature_directory) self.filenames = [f.split('.', 1)[0] for f in self.filenames] self.filenames.sort() n_files = len(self.filenames) self.filenames = [self.filenames[i] for i in train_val_perms] if sample_rate is None: self.sample_rate = [1, 2, 4] else: self.sample_rate = sample_rate self.seq_len = seq_length self.overlap = overlap self.videofeatures = [] self.groundtruthscores = [] self.groundtruth01scores = [] # self.segments = [] self.videonames = [] KY_dataset_path = os.path.join(data_path, 'KY_AAAI18/datasets') Kydataset = KyLoader.loadKyDataset( self.dataset_name.lower(), file_path=os.path.join( KY_dataset_path, 'eccv16_dataset_{:s}_google_pool5.h5'.format( dataset_name.lower()))) conversion = KyLoader.loadConversion( self.dataset_name.lower(), file_path=os.path.join( KY_dataset_path, '{:s}_name_conversion.pkl'.format(dataset_name.lower()))) self.raw2Ky = conversion[0] self.Ky2raw = conversion[1] for s_video_idx, s_filename in enumerate(self.filenames): KyKey = self.raw2Ky[s_filename] s_scores = Kydataset[KyKey]['gtscore'][...] s_scores = s_scores.reshape([-1, 1]) n_frames = s_scores.shape[0] s_segments, s_segment_scores = LoaderUtils.convertscores2segs( s_scores) selected_segments = rep_conversions.selecteTopSegments( s_segments, s_segment_scores, n_frames) s_frame01scores = rep_conversions.keyshots2frame01scores( selected_segments, n_frames) # s_frame01scores = rep_conversions.framescore2frame01score_inteval(s_scores.reshape([-1]), s_segments, lratio=self.sum_budget) # the size of s_features is: [length, fea_dim] # s_video_features = np.load( # os.path.join(self.feature_directory, '{:s}.{:s}'.format(s_filename, self.feature_file_ext))) s_video_features = Kydataset[KyKey]['features'] s_features_len = len(s_video_features) # the length of c3d feature is larger than annotation, choose middles to match assert abs( n_frames - s_features_len ) < 6, 'annotation and feature length not equal! {:d}, {:d}'.format( n_frames, s_features_len) offset = abs(s_features_len - n_frames) / 2 s_video_features = s_video_features[offset:offset + n_frames] self.groundtruthscores.append(s_scores) self.groundtruth01scores.append(s_frame01scores) self.videofeatures.append(s_video_features) # self.segments.append(s_segments) self.videonames.append(s_filename) self.dataset_size = len(self.videofeatures) print("{:s}\tEvaluator: {:s}\t{:d} Videos".format( self.dataset_name, self.split, self.dataset_size))
def __init__(self, dataset_name='TVSum', split='train', seq_length=90, overlap=0.9, sample_rate=None, feature_file_ext='npy', sum_budget=0.15, train_val_perms=None, eval_metrics='max', data_path=None): if dataset_name.lower() not in ['summe', 'tvsum']: print('Unrecognized dataset {:s}'.format(dataset_name)) self.dataset_name = dataset_name self.eval_metrics = eval_metrics#[self.dataset_name.lower()] self.split = split self.sum_budget = sum_budget self.feature_file_ext = feature_file_ext # self.feature_directory = os.path.join(user_root, 'datasets/%s/features/c3dd-red500' % (dataset_name)) self.feature_directory = os.path.join(data_path, '%s/features/c3dd-red500' % (dataset_name)) self.filenames = os.listdir(self.feature_directory) self.filenames = [f.split('.', 1)[0] for f in self.filenames] self.filenames.sort() n_files = len(self.filenames) # selected_perms = range(n_files) # if self.split == 'train': # selected_perms = train_val_perms[:int(0.8 * n_files)] # elif self.split == 'val': # selected_perms = train_val_perms[int(0.8 * n_files):] # else: # print("Unrecognized split:{:s}".format(self.split)) # self.filenames = [self.filenames[i] for i in selected_perms] self.filenames = [self.filenames[i] for i in train_val_perms] if sample_rate is None: self.sample_rate = [1, 2, 4] else: self.sample_rate = sample_rate self.seq_len = seq_length self.overlap = overlap self.videofeatures = [] self.groundtruthscores = [] self.combinegroundtruth01scores = [] # self.segments = [] self.videonames = [] KY_dataset_path = os.path.join(data_path, 'KY_AAAI18/datasets') Kydataset = KyLoader.loadKyDataset(self.dataset_name.lower(), file_path=os.path.join(KY_dataset_path, 'eccv16_dataset_{:s}_google_pool5.h5'.format(dataset_name.lower()))) conversion = KyLoader.loadConversion(self.dataset_name.lower(), file_path=os.path.join(KY_dataset_path, '{:s}_name_conversion.pkl'.format(dataset_name.lower()))) self.raw2Ky = conversion[0] self.Ky2raw = conversion[1] # project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # raw_data_path = os.path.join(project_root, 'Devs_vsSum/datasets/TVSum/TVSumRaw.pkl') # raw_annotation_data = pkl.load(open(raw_data_path, 'rb')) for s_video_idx, s_filename in enumerate(self.filenames): KyKey = self.raw2Ky[s_filename] s_scores = Kydataset[KyKey]['user_summary'][...] s_scores = s_scores.transpose() n_frames = len(s_scores) # s_segments = LoaderUtils.convertlabels2segs(s_scores) # raw_user_summaris = raw_annotation_data[s_filename] # raw_user_summaris_01 = [] # for s_raw_user_summary in raw_user_summaris: # assert len(s_raw_user_summary) == n_frames # s_raw_user_summary = np.expand_dims(np.array(s_raw_user_summary), -1) # s_summary_segments, s_summary_scores = LoaderUtils.convertscores2segs(s_raw_user_summary) # s_selected_segments = rep_conversions.selecteTopSegments(s_summary_segments, s_summary_scores, n_frames) # # raw_user_summaris_01.append(s_segments) # s_frame01scores = rep_conversions.keyshots2frame01scores(s_selected_segments, n_frames) # raw_user_summaris_01.append(s_frame01scores) # raw_user_summaris_01 = np.stack(raw_user_summaris_01, axis=1) # raw_user_summaris = np.array(raw_user_summaris) # raw_user_summaris = raw_user_summaris.transpose() ky_combine_summaris = np.mean(s_scores, 1, keepdims=True) s_combine_segments, s_combine_segment_scores = LoaderUtils.convertscores2segs(ky_combine_summaris) s_combine_selected_segments = rep_conversions.selecteTopSegments(s_combine_segments, s_combine_segment_scores, n_frames) s_combine_frame01scores = rep_conversions.keyshots2frame01scores(s_combine_selected_segments, n_frames) # the size of s_features is: [length, fea_dim] s_video_features = np.load( os.path.join(self.feature_directory, '{:s}.{:s}'.format(s_filename, self.feature_file_ext))) s_features_len = len(s_video_features) # the length of c3d feature is larger than annotation, choose middles to match assert abs(n_frames - s_features_len) < 6, 'annotation and feature length not equal! {:d}, {:d}'.format( n_frames, s_features_len) offset = abs(s_features_len - n_frames) / 2 s_video_features = s_video_features[offset:offset + n_frames] self.groundtruthscores.append(s_scores) self.videofeatures.append(s_video_features) self.combinegroundtruth01scores.append(s_combine_frame01scores) # self.segments.append(s_segments) self.videonames.append(s_filename) self.dataset_size = len(self.videofeatures) print("{:s}\tEvaluator: {:s}\t{:d} Videos".format(self.dataset_name, self.split, self.dataset_size))
def __init__(self, dataset_name='TVSum', split='train', max_input_len=130, maximum_outputs=26, feature_file_ext='npy', sum_budget=0.15, train_val_perms=None, eval_metrics='avg', data_path=None): if dataset_name.lower() not in ['summe', 'tvsum']: print('Unrecognized dataset {:s}'.format(dataset_name)) self.dataset_name = dataset_name self.eval_metrics = eval_metrics #[self.dataset_name.lower()] self.split = split self.sum_budget = sum_budget self.feature_file_ext = feature_file_ext # self.feature_directory = os.path.join(user_root, 'datasets/%s/features/c3dd-red500' % (dataset_name)) self.feature_directory = os.path.join( data_path, '%s/features/c3dd-red500' % (dataset_name)) self.filenames = os.listdir(self.feature_directory) self.filenames = [f.split('.', 1)[0] for f in self.filenames] self.filenames.sort() n_files = len(self.filenames) self.filenames = [self.filenames[i] for i in train_val_perms] self.max_input_len = max_input_len self.maximum_outputs = maximum_outputs self.segment_features = [] self.groundtruthscores = [] self.video_nfps = [] self.video_n_frames = [] self.video_cps = [] self.videonames = [] KY_dataset_path = os.path.join(data_path, 'KY_AAAI18/datasets') Kydataset = KyLoader.loadKyDataset( self.dataset_name.lower(), file_path=os.path.join( KY_dataset_path, 'eccv16_dataset_{:s}_google_pool5.h5'.format( dataset_name.lower()))) conversion = KyLoader.loadConversion( self.dataset_name.lower(), file_path=os.path.join( KY_dataset_path, '{:s}_name_conversion.pkl'.format(dataset_name.lower()))) self.raw2Ky = conversion[0] self.Ky2raw = conversion[1] # project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # raw_data_path = os.path.join(project_root, 'Devs_vsSum/datasets/TVSum/TVSumRaw.pkl') # raw_annotation_data = pkl.load(open(raw_data_path, 'rb')) for s_video_idx, s_filename in enumerate(self.filenames): KyKey = self.raw2Ky[s_filename] s_scores = Kydataset[KyKey]['user_summary'][...] s_scores = s_scores.transpose() n_frames = len(s_scores) # load raw scores # # the size of raw_user_summaris is [len, num_users] # raw_user_summaris = np.array(raw_annotation_data[s_filename]).transpose() # change_points, each segment is [cps[i,0]: cps[i,1]+1] cps = Kydataset[KyKey]['change_points'][...] # total number of frames num_frames = Kydataset[KyKey]['n_frames'][()] assert n_frames == num_frames, 'frame length should be the same' # num of frames per segment nfps = Kydataset[KyKey]['n_frame_per_seg'][...] num_segments = cps.shape[0] # the size of s_features is: [length, fea_dim] s_video_features = np.load( os.path.join( self.feature_directory, '{:s}.{:s}'.format(s_filename, self.feature_file_ext))) s_features_len = len(s_video_features) # the length of c3d feature is larger than annotation, choose middles to match assert abs( n_frames - s_features_len ) < 6, 'annotation and feature length not equal! {:d}, {:d}'.format( n_frames, s_features_len) offset = abs(s_features_len - n_frames) / 2 s_video_features = s_video_features[offset:offset + n_frames] # get average features s_segment_features = LoaderUtils.get_avg_seg_features( s_video_features, cps, num_segments) self.groundtruthscores.append(s_scores) self.segment_features.append(s_segment_features) self.videonames.append(s_filename) self.video_cps.append(cps) self.video_nfps.append(nfps) self.video_n_frames.append(n_frames) self.dataset_size = len(self.segment_features) print("{:s}\tEvaluator: {:s}\t{:d} Videos".format( self.dataset_name, self.split, self.dataset_size))