def main():
# compose data for xgboost
train, train_target, val, val_target = load_dataset() # np arrays [X 2048] [X 15] (large X)
# train = np.array([[1,2,3,4,5], [2,3,4,5,6], [4,5,6,7,8], [5,6,7,8,9]])
# train_target = np.array([3,4,5,6])
print(train.shape, train_target.shape, val[0].shape, val_target[0].shape)
regressor = xgb.XGBRegressor(tree_method='gpu_hist', predictor='gpu_predictor')
#
regressor.fit(train, train_target)
score = regressor.score(train, train_target)
print('Score:', score)
# print(regressor.predict(np.array([[6,7,8,9,10]])))
video_count = len(val)
corr_mean = 0.0
for i in range(video_count):
preds = regressor.predict(val[i])
preds = torch.from_numpy(preds).cuda()
preds = torch.cat([preds, preds[-1:]])
preds = preds.unsqueeze(1) # T -> T 1
preds = interpolate_output(preds, 1 , 6)
pred_len = val_target[i].shape[0]
preds = preds[:pred_len]
val_ti = torch.from_numpy(val_target[i]).cuda()
val_ti = val_ti.unsqueeze(1) # T -> T 1
corr, _ = correlation(preds, val_ti)
corr_mean += corr.item()
print('Correlation:', corr_mean/video_count)
def test_result(filt_emp):
global best_conf
# do a correlation evalutation
filt_emp = np.concatenate((filt_emp, filt_emp[-1:]))
output = interpolate_output(torch.from_numpy(filt_emp), 1, 6)
cor, _ = correlation(output[:vid_labels.shape[0]],
torch.from_numpy(vid_labels))
print('Correlation:', cor.item())
return cor.item()
def main_test():
print('Running test...')
torch.multiprocessing.set_sharing_strategy('file_system')
model = Baseline()
if args.use_swa:
model = torch.optim.swa_utils.AveragedModel(model)
model = torch.nn.DataParallel(model).cuda()
# ckpt structure {epoch, state_dict, optimizer, best_corr}
if args.resume and os.path.isfile(args.resume):
print('Load checkpoint:', args.resume)
ckpt = torch.load(args.resume)
args.start_epoch = ckpt['epoch']
best_corr = ckpt['best_corr']
model.load_state_dict(ckpt['state_dict'])
print('Loaded ckpt at epoch:', args.start_epoch)
else:
print('No model given. Abort!')
exit(1)
test_loader = torch.utils.data.DataLoader(
dataset=EEV_Dataset(
csv_path=None,
vidmap_path=args.test_vidmap,
image_feat_path=args.image_features,
audio_feat_path=args.audio_features,
mode='test',
test_freq=args.test_freq
),
batch_size=None, shuffle=False,
num_workers=args.workers, pin_memory=False
)
model.eval()
batch_time = AverageMeter()
t_start = time.time()
outputs = []
with torch.no_grad():
for i, (img_feat, au_feat, frame_count, vid) in enumerate(test_loader):
img_feat = torch.stack(img_feat).cuda()
au_feat = torch.stack(au_feat).cuda()
assert len(au_feat.size()) == 3, 'bad auf %s' % (vid)
output = model(img_feat, au_feat) # [Clip S 15]
# rearrange and remove extra padding in the end
output = rearrange(output, 'Clip S C -> (Clip S) C')
output = torch.cat([output, output[-1:]]) # repeat the last frame to avoid missing
if args.train_freq < args.test_freq:
# print('interpolating:', output.size()[0], frame_count)
output = interpolate_output(output, args.train_freq, 6)
# print('Interpolated:', output.size()[0], frame_count)
# truncate extra frames
assert output.size(0) >= frame_count, '{}/{}'.format(output.size(0), frame_count)
output = output[:frame_count]
outputs.append((vid, frame_count, output.cpu().detach().numpy()))
# update statistics
batch_time.update(time.time() - t_start)
t_start = time.time()
if i % args.print_freq == 0:
output = ('Test: [{0}/{1}]\t'
'Time: {batch_time.val:.3f} ({batch_time.avg:.3f})'.format(
i, len(test_loader), batch_time=batch_time))
print(output)
time_stamps = [0, 166666, 333333, 500000, 666666, 833333]
time_step = 1000000 # time starts at 0
header = 'Video ID,Timestamp (milliseconds),amusement,anger,awe,concentration,confusion,contempt,contentment,disappointment,doubt,elation,interest,pain,sadness,surprise,triumph\n'
final_res = {}
for vid, frame_count, out in outputs:# videos
video_time = frame_count // 6 + 1
# print('video', vid, video_time)
entry_count = 0
for t in range(video_time): # seconds
for i in range(6): # frames
timestamp = time_step * t + time_stamps[i]
fcc = t * 6 + i
if fcc >= frame_count:
continue
# print('Frame count', frame_count)
frame_output = out[fcc]
frame_output = [str(x) for x in frame_output]
temp = '{vid},{timestamp},'.format(vid=vid,timestamp=timestamp) + ','.join(frame_output) + '\n'
# file.write(temp)
if vid in final_res:
final_res[vid].append(temp)
else:
final_res[vid] = [temp]
entry_count += 1
assert entry_count == frame_count
# fixed for now
missing = [('WKXrnB7alT8', 2919), ('o0ooW14pIa4', 3733), ('GufMoL_MuNE',2038), ('Uee0Tv1rTz8', 1316), ('ScvvOWtb04Q', 152), ('R9kJlLungmo', 3609),('QMW3GuohzzE', 822), ('fjJYTW2n6rk', 4108), ('rbTIMt0VcLw', 1084),('L9cdaj74kLo', 3678), ('l-ka23gU4NA', 1759)]
for vid, length in missing:
video_time = length // 6 + 1
# print('video', vid, video_time)
for t in range(video_time): # seconds
for i in range(6): # frames
timestamp = time_step * t + time_stamps[i]
fcc = t * 6 + i
if fcc >= length:
continue
frame_output = ',0'*15
temp = '{vid},{timestamp}'.format(vid=vid, timestamp=timestamp) + frame_output + '\n'
# file.write(temp)
if vid in final_res:
final_res[vid].append(temp)
else:
final_res[vid] = [temp]
print('Write test outputs...')
with open('test_output.csv', 'w') as file:
file.write(header)
temp_vidmap = [x.strip().split(' ') for x in open(args.test_vidmap)]
temp_vidmap = [x[0] for x in temp_vidmap]
for vid in tqdm(temp_vidmap):
for entry in final_res[vid]:
file.write(entry)
def validate(val_loader, model, accuracy, epoch, log=None, tb_writer=None):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
correlations = AverageMeter()
model.eval()
t_start = time.time()
# load 1 video at a time for now (loader is not batched when batch_size == None)
# but since we split one vid into multiple small clips, the input shape is still batch-like
with torch.no_grad():
for i, (img_feat, au_feat, labels, frame_count) in enumerate(val_loader):
data_time.update(time.time() - t_start)
# print(type(img_feat), len(img_feat), img_feat[0].size())
img_feat = torch.stack(img_feat).cuda()
au_feat = torch.stack(au_feat).cuda()
labels = torch.stack(labels).cuda()
output = model(img_feat, au_feat) # [Clip S 15]
# rearrange and remove extra padding in the end
output = rearrange(output, 'Clip S C -> (Clip S) C')
output = torch.cat([output, output[-1:]]) # repeat the last frame to avoid missing
if args.train_freq < args.val_freq:
output = interpolate_output(output, args.train_freq, args.val_freq)
output = output[:frame_count]
labels = rearrange(labels, 'Clip S C -> (Clip S) C')[:frame_count]
loss = loss_function(output, labels, validate=True)
mean_cor, cor = accuracy(output, labels) # mean and per-class correlation
# update statistics
losses.update(loss.item())
assert not math.isnan(mean_cor.item()), 'at epoch %d' % (epoch)
correlations.update(mean_cor.item())
batch_time.update(time.time() - t_start)
t_start = time.time()
if i % args.print_freq == 0:
output = ('Val: [{0}/{1}]\t'
'Time: {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss: {loss.val:.4f} ({loss.avg:.4f})\t'
'Corr: {corr.val:.4f} ({corr.avg:.4f}, {corr.count})'.format(
i, len(val_loader), batch_time=batch_time, loss=losses,
corr=correlations))
print(output)
if log is not None:
log.write(output + '\n')
log.flush()
output = ('Validate Results: Corr:{corr.avg:.4f} Loss {loss.avg:.5f}'
.format(corr=correlations, loss=losses))
print(output)
if log is not None:
log.write(output + '\n')
log.flush()
if tb_writer is not None:
tb_writer.add_scalar('loss/validate', losses.avg, epoch)
tb_writer.add_scalar('acc/validate_corr', correlations.avg, epoch)
return correlations.avg, losses.avg
def validate(val_loader, model, accuracy, epoch, log, tb_writer):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
correlations = AverageMeter()
pc_corr = []
model.eval()
t_start = time.time()
# load 1 video at a time for now (loader is not batched when batch_size == None)
# but since we split one vid into multiple small clips, the input shape is still batch-like
with torch.no_grad():
for i, (img_feat, au_feat, labels,
frame_count) in enumerate(val_loader):
data_time.update(time.time() - t_start)
# print(type(img_feat), len(img_feat), img_feat[0].size())
img_feat = torch.stack(img_feat).cuda()
au_feat = torch.stack(au_feat).cuda()
labels = torch.stack(labels).cuda()
# normalize
# img_feat = F.normalize(img_feat, 2, dim=2)
# au_feat = F.normalize(au_feat, 2, dim=2)
if args.model == 'EmoBase':
output, _, _ = model(img_feat, au_feat)
else:
if args.repeat_sample:
img_feat = rearrange(
img_feat, 'Clip R S C -> (Clip R) S C') # Clip R S C
output = model_output(model, img_feat, au_feat)
# rearrange and remove extra padding in the end
if args.repeat_sample:
output = rearrange(output, '(Clip R) S C -> (Clip S R) C', R=6)
else:
output = rearrange(output, 'Clip S C -> (Clip S) C')
output = torch.cat([output, output[-1:]
]) # repeat the last frame to avoid missing
if not args.repeat_sample and args.train_freq < args.val_freq:
output = interpolate_output(output, args.train_freq,
args.val_freq)
output = output[:frame_count]
labels = rearrange(labels, 'Clip S C -> (Clip S) C')[:frame_count]
loss, c_loss, t_loss = loss_function(output,
labels,
args,
validate=True)
mean_cor, cor = accuracy(output,
labels) # mean and per-class correlation
if args.cls_mask != None:
# mask = [1 if x in args.cls_mask else 0 for x in range(0, 15)]
mean_cor = torch.mean(cor[args.cls_mask])
pc_corr.append(cor)
# update statistics
losses.update(loss.item())
assert not math.isnan(mean_cor.item()), 'at epoch %d' % (epoch)
correlations.update(mean_cor.item())
batch_time.update(time.time() - t_start)
t_start = time.time()
if i % args.print_freq == 0:
output = ('Val: [{0}/{1}]\t'
'Time: {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss: {loss.val:.4f} ({loss.avg:.4f})\t'
'Corr: {corr.val:.4f} ({corr.avg:.4f})'.format(
i,
len(val_loader),
batch_time=batch_time,
loss=losses,
corr=correlations))
print(output)
if log is not None:
log.write(output + '\n')
log.flush()
output = (
'Validate Results: Corr:{corr.avg:.4f} Loss {loss.avg:.5f}'.format(
corr=correlations, loss=losses))
print(output)
pc_corr = torch.stack(pc_corr, dim=0)
pc_corr = torch.mean(pc_corr, dim=0).cpu().numpy()
print('Per-Class Corr:', pc_corr)
if log is not None:
log.write(output + '\n')
log.flush()
if tb_writer is not None:
tb_writer.add_scalar('loss/validate', losses.avg, epoch)
tb_writer.add_scalar('acc/validate_corr', correlations.avg, epoch)
return correlations.avg