def __init__(self, dataset_name='SumMe',split='train', clip_size=50, output3=False):
if dataset_name.lower() not in ['summe', 'tvsum']:
print('Unrecognized dataset {:s}'.format(dataset_name))
sys.exit(-1)
self.dataset_name = dataset_name
self.split = split
self.filenames = datasetpaths[self.dataset_name.lower()].file_names
self.data_loader = data_loaders[self.dataset_name.lower()]
n_files = len(self.filenames)
if self.split == 'train':
self.filenames = self.filenames[:int(0.8*n_files)]
elif self.split == 'val':
self.filenames = self.filenames[int(0.8*n_files):]
else:
print("Unrecognized split:{:s}".format(self.split))
sys.exit(-1)
self.clip_size = clip_size
self.output3 = output3
Kydataset = KyLoader.loadKyDataset(self.dataset_name.lower())
conversion = KyLoader.loadConversion(self.dataset_name.lower())
self.raw2Ky = conversion[0]
self.Ky2raw = conversion[1]
self.features = {}
self.avg_summary = {}
self.annotations = []
pbar = progressbar.ProgressBar(max_value=len(self.filenames))
print("Processing {:s}\t{:s} data".format(self.dataset_name, self.split))
for file_dix, s_filename in enumerate(self.filenames):
pbar.update(file_dix)
Kykey = self.raw2Ky[s_filename]
s_features, _, _ = self.data_loader.load_by_name(s_filename)
s_usersummaries = Kydataset[Kykey]['user_summary'][...]
s_usersummaries = s_usersummaries.transpose()
self.features[s_filename] = s_features
s_avg_summary = np.mean(s_usersummaries, axis=1)
self.avg_summary[s_filename] = s_avg_summary
s_video_len = len(s_avg_summary)
s_segments = LoaderUtils.convertlabels2segs(s_usersummaries)
for s_seg in s_segments:
sSegment = LoaderUtils.Segment(s_filename)
s_score = np.mean(s_avg_summary[s_seg[0]:s_seg[1]])
sSegment.initId(s_seg[0], s_seg[1], s_video_len, score=s_score)
self.annotations.append(sSegment)
self.augmented_ratio = 10
self.dataset_size = len(self.annotations)
print("{:s}\t{:s}\t{:d}".format(self.dataset_name, self.split, self.dataset_size))
def __init__(self,
dataset_name='SumMe',
split='train',
clip_size=100,
sum_budget=0.15,
sample_rate=None,
overlap_ratio=0.25):
if dataset_name.lower() not in ['summe', 'tvsum']:
print('Unrecognized dataset {:s}'.format(dataset_name))
self.dataset_name = dataset_name
self.eval_metrics = eval_metrics[self.dataset_name.lower()]
self.split = split
self.sum_budget = sum_budget
self.filenames = datasetpaths[self.dataset_name.lower()].file_names
self.data_loader = data_loaders[self.dataset_name.lower()]
n_files = len(self.filenames)
if self.split == 'train':
self.filenames = self.filenames[:int(0.8 * n_files)]
elif self.split == 'val':
self.filenames = self.filenames[int(0.8 * n_files):]
else:
print("Unrecognized split:{:s}".format(self.split))
self.overlap_ratio = overlap_ratio
self.clip_size = clip_size
if sample_rate is None:
self.sample_rate = [1, 5, 10]
else:
if isinstance(sample_rate, list):
self.sample_rate = sample_rate
else:
self.sample_rate = [sample_rate]
self.videofeatures = []
self.groundtruthscores = []
self.segments = []
self.videonames = []
Kydataset = KyLoader.loadKyDataset(self.dataset_name.lower())
conversion = KyLoader.loadConversion(self.dataset_name.lower())
self.raw2Ky = conversion[0]
self.Ky2raw = conversion[1]
for s_video_idx, s_filename in enumerate(self.filenames):
KyKey = self.raw2Ky[s_filename]
s_video_features, _, _ = self.data_loader.load_by_name(s_filename)
s_scores = Kydataset[KyKey]['user_summary'][...]
s_scores = s_scores.transpose()
s_segments = LoaderUtils.convertlabels2segs(s_scores)
self.groundtruthscores.append(s_scores)
self.videofeatures.append(s_video_features)
self.segments.append(s_segments)
self.videonames.append(s_filename)
self.dataset_size = len(self.videofeatures)
print("{:s}\tEvaluator: {:s}\t{:d} Videos".format(
self.dataset_name, self.split, self.dataset_size))
def __init__(self,
dataset_name='TVSum',
split='train',
seq_length=90,
overlap=0.9,
sample_rate=None,
feature_file_ext='npy',
sum_budget=0.15,
train_val_perms=None,
eval_metrics='avg',
data_path=None):
if dataset_name.lower() not in ['summe', 'tvsum']:
print('Unrecognized dataset {:s}'.format(dataset_name))
self.dataset_name = dataset_name
self.eval_metrics = eval_metrics #[self.dataset_name.lower()]
self.split = split
self.sum_budget = sum_budget
self.feature_file_ext = feature_file_ext
self.feature_directory = os.path.join(
data_path, '%s/features/c3dd-red500' % (dataset_name))
self.filenames = os.listdir(self.feature_directory)
self.filenames = [f.split('.', 1)[0] for f in self.filenames]
self.filenames.sort()
n_files = len(self.filenames)
self.filenames = [self.filenames[i] for i in train_val_perms]
if sample_rate is None:
self.sample_rate = [1, 2, 4]
else:
self.sample_rate = sample_rate
self.seq_len = seq_length
self.overlap = overlap
self.videofeatures = []
self.groundtruthscores = []
self.groundtruth01scores = []
# self.segments = []
self.videonames = []
KY_dataset_path = os.path.join(data_path, 'KY_AAAI18/datasets')
Kydataset = KyLoader.loadKyDataset(
self.dataset_name.lower(),
file_path=os.path.join(
KY_dataset_path, 'eccv16_dataset_{:s}_google_pool5.h5'.format(
dataset_name.lower())))
conversion = KyLoader.loadConversion(
self.dataset_name.lower(),
file_path=os.path.join(
KY_dataset_path,
'{:s}_name_conversion.pkl'.format(dataset_name.lower())))
self.raw2Ky = conversion[0]
self.Ky2raw = conversion[1]
for s_video_idx, s_filename in enumerate(self.filenames):
KyKey = self.raw2Ky[s_filename]
s_scores = Kydataset[KyKey]['gtscore'][...]
s_scores = s_scores.reshape([-1, 1])
n_frames = s_scores.shape[0]
s_segments, s_segment_scores = LoaderUtils.convertscores2segs(
s_scores)
selected_segments = rep_conversions.selecteTopSegments(
s_segments, s_segment_scores, n_frames)
s_frame01scores = rep_conversions.keyshots2frame01scores(
selected_segments, n_frames)
# s_frame01scores = rep_conversions.framescore2frame01score_inteval(s_scores.reshape([-1]), s_segments, lratio=self.sum_budget)
# the size of s_features is: [length, fea_dim]
# s_video_features = np.load(
# os.path.join(self.feature_directory, '{:s}.{:s}'.format(s_filename, self.feature_file_ext)))
s_video_features = Kydataset[KyKey]['features']
s_features_len = len(s_video_features)
# the length of c3d feature is larger than annotation, choose middles to match
assert abs(
n_frames - s_features_len
) < 6, 'annotation and feature length not equal! {:d}, {:d}'.format(
n_frames, s_features_len)
offset = abs(s_features_len - n_frames) / 2
s_video_features = s_video_features[offset:offset + n_frames]
self.groundtruthscores.append(s_scores)
self.groundtruth01scores.append(s_frame01scores)
self.videofeatures.append(s_video_features)
# self.segments.append(s_segments)
self.videonames.append(s_filename)
self.dataset_size = len(self.videofeatures)
print("{:s}\tEvaluator: {:s}\t{:d} Videos".format(
self.dataset_name, self.split, self.dataset_size))
def __init__(self, dataset_name='TVSum', split='train', seq_length=90, overlap=0.9, sample_rate=None,
feature_file_ext='npy', sum_budget=0.15, train_val_perms=None, eval_metrics='max', data_path=None):
if dataset_name.lower() not in ['summe', 'tvsum']:
print('Unrecognized dataset {:s}'.format(dataset_name))
self.dataset_name = dataset_name
self.eval_metrics = eval_metrics#[self.dataset_name.lower()]
self.split = split
self.sum_budget = sum_budget
self.feature_file_ext = feature_file_ext
# self.feature_directory = os.path.join(user_root, 'datasets/%s/features/c3dd-red500' % (dataset_name))
self.feature_directory = os.path.join(data_path, '%s/features/c3dd-red500' % (dataset_name))
self.filenames = os.listdir(self.feature_directory)
self.filenames = [f.split('.', 1)[0] for f in self.filenames]
self.filenames.sort()
n_files = len(self.filenames)
# selected_perms = range(n_files)
# if self.split == 'train':
# selected_perms = train_val_perms[:int(0.8 * n_files)]
# elif self.split == 'val':
# selected_perms = train_val_perms[int(0.8 * n_files):]
# else:
# print("Unrecognized split:{:s}".format(self.split))
# self.filenames = [self.filenames[i] for i in selected_perms]
self.filenames = [self.filenames[i] for i in train_val_perms]
if sample_rate is None:
self.sample_rate = [1, 2, 4]
else:
self.sample_rate = sample_rate
self.seq_len = seq_length
self.overlap = overlap
self.videofeatures = []
self.groundtruthscores = []
self.combinegroundtruth01scores = []
# self.segments = []
self.videonames = []
KY_dataset_path = os.path.join(data_path, 'KY_AAAI18/datasets')
Kydataset = KyLoader.loadKyDataset(self.dataset_name.lower(), file_path=os.path.join(KY_dataset_path, 'eccv16_dataset_{:s}_google_pool5.h5'.format(dataset_name.lower())))
conversion = KyLoader.loadConversion(self.dataset_name.lower(), file_path=os.path.join(KY_dataset_path, '{:s}_name_conversion.pkl'.format(dataset_name.lower())))
self.raw2Ky = conversion[0]
self.Ky2raw = conversion[1]
# project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
# raw_data_path = os.path.join(project_root, 'Devs_vsSum/datasets/TVSum/TVSumRaw.pkl')
# raw_annotation_data = pkl.load(open(raw_data_path, 'rb'))
for s_video_idx, s_filename in enumerate(self.filenames):
KyKey = self.raw2Ky[s_filename]
s_scores = Kydataset[KyKey]['user_summary'][...]
s_scores = s_scores.transpose()
n_frames = len(s_scores)
# s_segments = LoaderUtils.convertlabels2segs(s_scores)
# raw_user_summaris = raw_annotation_data[s_filename]
# raw_user_summaris_01 = []
# for s_raw_user_summary in raw_user_summaris:
# assert len(s_raw_user_summary) == n_frames
# s_raw_user_summary = np.expand_dims(np.array(s_raw_user_summary), -1)
# s_summary_segments, s_summary_scores = LoaderUtils.convertscores2segs(s_raw_user_summary)
# s_selected_segments = rep_conversions.selecteTopSegments(s_summary_segments, s_summary_scores, n_frames)
# # raw_user_summaris_01.append(s_segments)
# s_frame01scores = rep_conversions.keyshots2frame01scores(s_selected_segments, n_frames)
# raw_user_summaris_01.append(s_frame01scores)
# raw_user_summaris_01 = np.stack(raw_user_summaris_01, axis=1)
# raw_user_summaris = np.array(raw_user_summaris)
# raw_user_summaris = raw_user_summaris.transpose()
ky_combine_summaris = np.mean(s_scores, 1, keepdims=True)
s_combine_segments, s_combine_segment_scores = LoaderUtils.convertscores2segs(ky_combine_summaris)
s_combine_selected_segments = rep_conversions.selecteTopSegments(s_combine_segments, s_combine_segment_scores, n_frames)
s_combine_frame01scores = rep_conversions.keyshots2frame01scores(s_combine_selected_segments, n_frames)
# the size of s_features is: [length, fea_dim]
s_video_features = np.load(
os.path.join(self.feature_directory, '{:s}.{:s}'.format(s_filename, self.feature_file_ext)))
s_features_len = len(s_video_features)
# the length of c3d feature is larger than annotation, choose middles to match
assert abs(n_frames - s_features_len) < 6, 'annotation and feature length not equal! {:d}, {:d}'.format(
n_frames, s_features_len)
offset = abs(s_features_len - n_frames) / 2
s_video_features = s_video_features[offset:offset + n_frames]
self.groundtruthscores.append(s_scores)
self.videofeatures.append(s_video_features)
self.combinegroundtruth01scores.append(s_combine_frame01scores)
# self.segments.append(s_segments)
self.videonames.append(s_filename)
self.dataset_size = len(self.videofeatures)
print("{:s}\tEvaluator: {:s}\t{:d} Videos".format(self.dataset_name, self.split, self.dataset_size))
def __init__(self,
dataset_name='TVSum',
split='train',
max_input_len=130,
maximum_outputs=26,
feature_file_ext='npy',
sum_budget=0.15,
train_val_perms=None,
eval_metrics='avg',
data_path=None):
if dataset_name.lower() not in ['summe', 'tvsum']:
print('Unrecognized dataset {:s}'.format(dataset_name))
self.dataset_name = dataset_name
self.eval_metrics = eval_metrics #[self.dataset_name.lower()]
self.split = split
self.sum_budget = sum_budget
self.feature_file_ext = feature_file_ext
# self.feature_directory = os.path.join(user_root, 'datasets/%s/features/c3dd-red500' % (dataset_name))
self.feature_directory = os.path.join(
data_path, '%s/features/c3dd-red500' % (dataset_name))
self.filenames = os.listdir(self.feature_directory)
self.filenames = [f.split('.', 1)[0] for f in self.filenames]
self.filenames.sort()
n_files = len(self.filenames)
self.filenames = [self.filenames[i] for i in train_val_perms]
self.max_input_len = max_input_len
self.maximum_outputs = maximum_outputs
self.segment_features = []
self.groundtruthscores = []
self.video_nfps = []
self.video_n_frames = []
self.video_cps = []
self.videonames = []
KY_dataset_path = os.path.join(data_path, 'KY_AAAI18/datasets')
Kydataset = KyLoader.loadKyDataset(
self.dataset_name.lower(),
file_path=os.path.join(
KY_dataset_path, 'eccv16_dataset_{:s}_google_pool5.h5'.format(
dataset_name.lower())))
conversion = KyLoader.loadConversion(
self.dataset_name.lower(),
file_path=os.path.join(
KY_dataset_path,
'{:s}_name_conversion.pkl'.format(dataset_name.lower())))
self.raw2Ky = conversion[0]
self.Ky2raw = conversion[1]
# project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
# raw_data_path = os.path.join(project_root, 'Devs_vsSum/datasets/TVSum/TVSumRaw.pkl')
# raw_annotation_data = pkl.load(open(raw_data_path, 'rb'))
for s_video_idx, s_filename in enumerate(self.filenames):
KyKey = self.raw2Ky[s_filename]
s_scores = Kydataset[KyKey]['user_summary'][...]
s_scores = s_scores.transpose()
n_frames = len(s_scores)
# load raw scores
# # the size of raw_user_summaris is [len, num_users]
# raw_user_summaris = np.array(raw_annotation_data[s_filename]).transpose()
# change_points, each segment is [cps[i,0]: cps[i,1]+1]
cps = Kydataset[KyKey]['change_points'][...]
# total number of frames
num_frames = Kydataset[KyKey]['n_frames'][()]
assert n_frames == num_frames, 'frame length should be the same'
# num of frames per segment
nfps = Kydataset[KyKey]['n_frame_per_seg'][...]
num_segments = cps.shape[0]
# the size of s_features is: [length, fea_dim]
s_video_features = np.load(
os.path.join(
self.feature_directory,
'{:s}.{:s}'.format(s_filename, self.feature_file_ext)))
s_features_len = len(s_video_features)
# the length of c3d feature is larger than annotation, choose middles to match
assert abs(
n_frames - s_features_len
) < 6, 'annotation and feature length not equal! {:d}, {:d}'.format(
n_frames, s_features_len)
offset = abs(s_features_len - n_frames) / 2
s_video_features = s_video_features[offset:offset + n_frames]
# get average features
s_segment_features = LoaderUtils.get_avg_seg_features(
s_video_features, cps, num_segments)
self.groundtruthscores.append(s_scores)
self.segment_features.append(s_segment_features)
self.videonames.append(s_filename)
self.video_cps.append(cps)
self.video_nfps.append(nfps)
self.video_n_frames.append(n_frames)
self.dataset_size = len(self.segment_features)
print("{:s}\tEvaluator: {:s}\t{:d} Videos".format(
self.dataset_name, self.split, self.dataset_size))