def early_gen_new_val(self, list_files, batch_size, mode="val", stride=1):
""" stride 50% sul su tutti i file """
c = 0
labels = features = []
clip_ids = list(list_files.keys())
while True:
for clip_id in tqdm(clip_ids):
video_info = list_files[clip_id]
ground_truth = video_info[0][0]
for start in range(0, len(video_info) - self.time_step, self.time_step // stride):
if c == 0:
labels = []
features = [np.zeros((batch_size, self.time_step, self.feature_num)).astype('float'),
np.zeros((batch_size, self.time_step, 1024)).astype('float')]
for index, elem in enumerate(video_info[start:self.time_step + start]):
_, frame_path, audio_path = elem
frame_feature = np.load(frame_path)
features[0][c][index] = np.array(from_arff_to_feture(audio_path)).reshape(
self.feature_num, )
features[1][c][index] = frame_feature.reshape(1024, )
labels.append(ground_truth)
c += 1
if c == batch_size:
c = 0
labels = self.lb.transform(np.array(labels)).reshape((batch_size, 7))
yield features, labels
if mode == "eval":
break
def early_gen_train(self, list_files, batch_size):
c = 0
clip_ids = list(self.csv_fusion["train"].keys())
random.shuffle(clip_ids)
while True:
labels = []
features = [np.zeros((batch_size, self.time_step, self.feature_num)).astype('float'),
np.zeros((batch_size, self.time_step, 1024)).astype('float')]
for i in range(c, c + batch_size):
clip_id = clip_ids[i]
video_info = self.csv_fusion["train"][clip_id]
ground_truth = video_info[0][0]
# first_frame_num = int(video_info[0][1].split("_")[-1].split(".")[0])
start = random.randint(0, len(video_info) - self.time_step)
for index, elem in enumerate(video_info[start:self.time_step + start]):
_, frame_path, audio_path = elem
if not isfile(frame_path):
start += 1
if start >= len(video_info):
raise
continue
frame_feature = np.load(frame_path)
features[0][i - c][index] = np.array(from_arff_to_feture(audio_path)).reshape(self.feature_num, )
features[1][i - c][index] = frame_feature.reshape(1024, )
labels.append(ground_truth)
c += batch_size
if c + batch_size > len(clip_ids):
c = 0
random.shuffle(clip_ids)
labels = self.lb.transform(np.array(labels)).reshape((batch_size, 7))
yield features, labels
def early_gen_new_val(self, list_files, batch_size, mode="val", stride=1):
""" stride 50% sul su tutti i file """
c = 0
labels = features = []
clip_ids = list(list_files.keys())
while True:
for clip_id in clip_ids:
video_info = list_files[clip_id]
ground_truth = video_info[0][0]
csv_path = '/user/vlongobardi/AFEW/aligned/Val/GroundTruth/ID.csv'
csv_path = csv_path.replace("GroundTruth",
ground_truth).replace(
"ID", clip_id)
images = DataGen(csv_path,
'',
1,
31,
NoAug(),
16,
1,
12,
test=True)[0][0][0]
first_frame_num = int(
video_info[0][1].split("_")[-1].split(".")[0])
for start in range(0,
len(video_info) - self.time_step,
self.time_step // stride):
if c == 0:
labels = []
features = [
np.zeros((batch_size, self.time_step,
self.feature_num)).astype('float'),
np.zeros((batch_size, self.time_step, 224, 224,
3)).astype('float')
]
for index, elem in enumerate(
video_info[start:self.time_step + start]):
audio_path = elem[2]
features[0][c][index] = np.array(
from_arff_to_feture(audio_path)).reshape(
self.feature_num, )
features[1][c][index] = images[first_frame_num +
start + index]
labels.append(ground_truth)
c += 1
if c == batch_size:
c = 0
labels = self.lb.transform(np.array(labels)).reshape(
(batch_size, 7))
yield features, labels
if mode == "eval":
break
def early_gen_train(self, list_files, batch_size):
c = 0
clip_ids = list(list_files.keys())
random.shuffle(clip_ids)
while True:
labels = []
features = [
np.zeros((batch_size, self.time_step,
self.feature_num)).astype('float'),
np.zeros(
(batch_size, self.time_step, 224, 224, 3)).astype('float')
]
for i in range(c, c + batch_size):
clip_id = clip_ids[i]
video_info = list_files[clip_id]
ground_truth = video_info[0][0]
csv_path = '/user/vlongobardi/AFEW/aligned/Train/GroundTruth/ID.csv'
csv_path = csv_path.replace("GroundTruth",
ground_truth).replace(
"ID", clip_id)
images = DataGen(csv_path,
'',
1,
31,
NoAug(),
16,
1,
12,
test=True)[0][0][0]
first_frame_num = int(
video_info[0][1].split("_")[-1].split(".")[0])
start = random.randint(0, len(video_info) - self.time_step)
for index, elem in enumerate(video_info[start:self.time_step +
start]):
ground_truth, _, audio_path = elem
features[0][i - c][index] = np.array(
from_arff_to_feture(audio_path)).reshape(
self.feature_num, )
features[1][i - c][index] = images[first_frame_num +
start + index]
labels.append(ground_truth)
c += batch_size
if c + batch_size > len(clip_ids):
c = 0
random.shuffle(clip_ids)
labels = self.lb.transform(np.array(labels)).reshape(
(batch_size, 7))
yield features, labels
def early_gen_test_clip(self, list_files, clip_id, stride=1):
""" stride su singolo file, quindi va richiamato per ogni file """
ground_truth = list_files[0][0]
start = 0
end = len(list_files) - self.time_step
while True:
labels = []
features = [np.zeros((1, self.time_step, self.feature_num)).astype('float'),
np.zeros((1, self.time_step, 1024)).astype('float')]
for index, elem in enumerate(list_files[start:start + self.time_step]):
_, frame_path, audio_path = elem
frame_feature = np.load(frame_path)
features[0][0][index] = np.array(from_arff_to_feture(audio_path)).reshape(self.feature_num, )
features[1][0][index] = frame_feature.reshape(1024, )
labels.append(ground_truth)
start += self.time_step // stride
if start >= end:
break
labels = self.lb.transform(np.array(labels)).reshape((1, 7))
yield features, labels
def early_gen_test_clip(self, list_files, clip_id, stride=1):
""" stride su singolo file, quindi va richiamato per ogni file """
ground_truth = list_files[0][0]
csv_path = '/user/vlongobardi/AFEW/aligned/Val/GroundTruth/ID.csv'
csv_path = csv_path.replace("GroundTruth",
ground_truth).replace("ID", clip_id)
first_frame_num = int(list_files[0][1].split("_")[-1].split(".")[0])
start = 0
end = len(list_files) - self.time_step
while True:
labels = []
features = [
np.zeros(
(1, self.time_step, self.feature_num)).astype('float'),
np.zeros((1, self.time_step, 224, 224, 3)).astype('float')
]
images = DataGen(csv_path,
'',
1,
31,
NoAug(),
16,
1,
12,
test=True)[0][0][0]
for index, elem in enumerate(list_files[start:start +
self.time_step]):
audio_path = elem[2]
features[0][0][index] = np.array(
from_arff_to_feture(audio_path)).reshape(
self.feature_num, )
features[1][0][index] = images[first_frame_num + start + index]
labels.append(ground_truth)
start += self.time_step // stride
if start >= end:
break
labels = self.lb.transform(np.array(labels)).reshape((1, 7))
yield features, labels