def __init__(self, options):
self.usecudnn = options["general"]["usecudnn"]
self.batchsize = options["input"]["batchsize"]
self.statsfrequency = options["training"]["statsfrequency"]
self.learningrate = options["training"]["learningrate"]
self.modelType = options["training"]["learningrate"]
self.weightdecay = options["training"]["weightdecay"]
self.momentum = options["training"]["momentum"]
self.save_prefix = options["training"]["save_prefix"]
self.trainingdataset = LipreadingDataset(
options["training"]["data_root"],
options["training"]["index_root"], options["training"]["padding"],
True)
self.trainingdataloader = DataLoader(
self.trainingdataset,
batch_size=options["input"]["batchsize"],
shuffle=options["input"]["shuffle"],
num_workers=options["input"]["numworkers"],
drop_last=True)
def __init__(self, args):
augment = False
shuffle = False
self.channel = args.channel
self.batchsize = args.batch_size
self.validationdataset = LipreadingDataset(args.dataset,
"val",
augment=augment,
channel=self.channel)
self.validationdataloader = DataLoader(self.validationdataset,
batch_size=self.batchsize,
shuffle=shuffle,
num_workers=args.workers,
drop_last=True)
# self.usecudnn = options["general"]["usecudnn"]
self.statsfrequency = args.statsfrequency
# self.gpuid = options["general"]["gpuid"]
self.log_file = args.logfile
self.savedir = args.save_dir
self.num_batches = int(len(self.validationdataset) / self.batchsize)
self.num_samples = int(len(self.validationdataset))
self.num_frames = args.num_frames
self.modelname = args.modelname
print_log('loaded validation dataset with %d data' %
len(self.validationdataset),
log=self.log_file)
def __init__(self, options, mode):
self.usecudnn = options["general"]["usecudnn"]
self.batchsize = options["input"]["batchsize"]
self.validationdataset_ori = LipreadingDataset(
options[mode]["data_root"], options[mode]["index_root"],
options[mode]["padding"], False)
self.validationdataset = LipreadingDataset_val(
options[mode]["data_root"], options[mode]["index_root"],
options[mode]["padding"], False)
self.tot_data = len(self.validationdataset)
self.validationdataloader = DataLoader(
self.validationdataset,
batch_size=options["input"]["batchsize"],
shuffle=False,
num_workers=options["input"]["numworkers"],
drop_last=False)
self.mode = mode
self.video_dist_label = convert_task3_label_to_video_label()
self.average_IOU = 0
self.video_id_cnt = 0
self.time0 = time.time()
def __init__(self, args):
augment = True
shuffle = True
self.channel = args.channel
self.batchsize = args.batch_size
self.trainingdataset = LipreadingDataset(args.dataset,
"train",
augment=augment,
channel=self.channel)
self.trainingdataloader = DataLoader(self.trainingdataset,
batch_size=self.batchsize,
shuffle=shuffle,
num_workers=args.workers,
drop_last=True)
# self.usecudnn = options["general"]["usecudnn"]
self.statsfrequency = args.statsfrequency
# self.gpuid = options["general"]["gpuid"]
self.learningrate = args.lr
self.weightdecay = args.weight_decay
self.momentum = args.momentum
self.log_file = args.logfile
self.modelsavedir = args.save_dir
# _, self.savename, _ = fileparts(self.modelsavedir)
self.num_batches = int(len(self.trainingdataset) / self.batchsize)
self.num_samples = int(len(self.trainingdataset))
self.num_frames = args.num_frames
self.modelname = args.modelname
print_log('loaded training dataset with %d data' %
len(self.trainingdataset),
log=self.log_file)
if augment: print_log('using augmentation', log=self.log_file)
else: print_log('no data augmentation', log=self.log_file)
def __init__(self, options):
self.dataset = LipreadingDataset(options['validation']['data_path'],
"val", False, options['model']['landmark'])
self.dataloader = DataLoader(
self.dataset,
batch_size=options["input"]["batch_size"],
shuffle=options["input"]["shuffle"],
num_workers=options["input"]["num_worker"],
drop_last=True
)
self.batch_size = options["input"]["batch_size"]
def __init__(self, options, mode):
self.usecudnn = options["general"]["usecudnn"]
self.batchsize = options["input"]["batchsize"]
self.validationdataset = LipreadingDataset(options[mode]["data_root"],
options[mode]["index_root"],
options[mode]["padding"],
False)
self.tot_data = len(self.validationdataset)
self.validationdataloader = DataLoader(
self.validationdataset,
batch_size=options["input"]["batchsize"],
shuffle=options["input"]["shuffle"],
num_workers=options["input"]["numworkers"],
drop_last=False)
self.mode = mode
def __init__(self, options, save_dir):
self.validationdataset = LipreadingDataset(
options["validation"]["dataset"],
"val",
augment=False,
use_frames=options['training']['use_frames'])
self.validationdataloader = DataLoader(
self.validationdataset,
batch_size=options["input"]["batchsize"],
shuffle=options["input"]["shuffle"],
num_workers=options["input"]["numworkers"],
drop_last=True)
self.usecudnn = options["general"]["usecudnn"]
self.batchsize = options["input"]["batchsize"]
self.save_dir = save_dir
def __init__(self, options):
self.validationdataset = LipreadingDataset(
"/udisk/pszts-ssd/AV-ASR-data/BBC_Oxford/lipread_mp4", "val",
False)
self.validationdataloader = DataLoader(
self.validationdataset,
batch_size=options["input"]["batchsize"],
shuffle=options["input"]["shuffle"],
num_workers=options["input"]["numworkers"],
drop_last=True)
self.usecudnn = options["general"]["usecudnn"]
self.batchsize = options["input"]["batchsize"]
self.statsfrequency = options["training"]["statsfrequency"]
self.gpuid = options["general"]["gpuid"]
def __init__(self, options):
self.batchsize = options["input"]["batchsize"]
self.validationdataset = LipreadingDataset(
options["general"]["dataset"], "val", False)
self.validationdataloader = DataLoader(
self.validationdataset,
batch_size=self.batchsize,
shuffle=False,
num_workers=options["input"]["numworkers"],
drop_last=True)
self.usecudnn = options["general"]["usecudnn"]
self.statsfrequency = options["training"]["statsfrequency"]
self.gpuid = options["general"]["gpuid"]
self.log_file = options["general"]["logfile"]
self.savedir = options["general"]["modelsavedir"]
self.num_batches = int(len(self.validationdataset) / self.batchsize)
print_log('loaded validation dataset with %d data' %
len(self.validationdataset),
log=self.log_file)
def __init__(self, options):
self.batchsize = options["input"]["batchsize"]
self.trainingdataset = LipreadingDataset(options["general"]["dataset"],
"train")
self.trainingdataloader = DataLoader(
self.trainingdataset,
batch_size=self.batchsize,
shuffle=options["training"]["shuffle"],
num_workers=options["input"]["numworkers"],
drop_last=True)
self.usecudnn = options["general"]["usecudnn"]
self.statsfrequency = options["training"]["statsfrequency"]
self.gpuid = options["general"]["gpuid"]
self.learningrate = options["training"]["learningrate"]
# self.modelType = options["training"]["learningrate"]
self.weightdecay = options["training"]["weightdecay"]
self.momentum = options["training"]["momentum"]
self.log_file = options["general"]["logfile"]
self.modelsavedir = options["general"]["modelsavedir"]
_, self.time_str, _ = fileparts(self.modelsavedir)
print_log('loaded training dataset with %d data' %
len(self.trainingdataset),
log=options["general"]["logfile"])
def __init__(self, options):
self.trainingdataset = LipreadingDataset(options["training"]["dataset"], "train", use_frames=options['training']['use_frames'])
self.trainingdataloader = DataLoader(
self.trainingdataset,
batch_size=options["input"]["batchsize"],
shuffle=options["input"]["shuffle"],
num_workers=options["input"]["numworkers"],
drop_last=True
)
if len(self.trainingdataset) == 0:
print("WARN: no data for training", file=sys.stderr)
self.usecudnn = options["general"]["usecudnn"]
self.batchsize = options["input"]["batchsize"]
self.learningrate = options["training"]["learningrate"]
self.modelType = options["training"]["learningrate"]
self.weightdecay = options["training"]["weightdecay"]
self.momentum = options["training"]["momentum"]
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-c", "--config", type=str, help="configuration file")
parser.add_argument("-s",
"--start",
type=int,
help="start epoch if reload",
default=1)
parser.add_argument("-t",
"--test",
type=bool,
help="test mode",
default=False)
parser.add_argument("-b",
"--best",
type=float,
help="best epoch val accuracy",
default=0.)
parser.add_argument("-e",
"--bepoch",
type=int,
help="the number of bad epoch",
default=0)
args = parser.parse_args()
writer = SummaryWriter()
print("Loading options...")
with open(args.config, 'r') as optionsFile:
options = yaml.load(optionsFile.read())
options['training']['start_epoch'] = args.start - 1
if (options["general"]["usecudnnbenchmark"]
and options["general"]["usecudnn"]):
print("Running cudnn benchmark...")
torch.backends.cudnn.benchmark = True
writer.add_text('name', options['name'])
#Create the model.
model = LipRead(options).cuda()
print('lr : ', options['training']['learning_rate'])
print('weight_decay : ', options['training']['weight_decay'])
print('landmark : ', options['input']['landmark'])
print('landmark channel concat : ',
not options['input']['landmark_seperate'])
print('coord conv : ', options['model']['coord'])
print('attention : ', options['model']['attention'])
optimizer = optim.Adam(model.parameters(),
lr=options['training']['learning_rate'],
weight_decay=options['training']['weight_decay'])
if options['training']['schedule'] == 'plateau':
if args.start > 1 and args.best == 0. and not args.test:
print("must have best accuracy")
raise
plat = True
scheduler = optim.lr_scheduler.ReduceLROnPlateau(
optimizer,
mode='max',
factor=options['training']['lr_decay'],
patience=3,
verbose=True,
threshold=0.003,
threshold_mode='abs',
)
scheduler.best = args.best
scheduler.num_bad_epochs = args.bepoch
else:
plat = False
scheduler = optim.lr_scheduler.MultiStepLR(
optimizer,
milestones=options['training']['schedule'],
gamma=options['training']['lr_decay'])
criterion = model.loss()
if not os.path.isdir(
os.path.join(options["general"]["save_path"], options['name'])):
os.mkdir(os.path.join(options["general"]["save_path"],
options['name']))
if not os.path.isdir(
os.path.join(options["general"]["save_path"], options['name'],
'optimizers')):
os.mkdir(
os.path.join(options["general"]["save_path"], options['name'],
'optimizers'))
if not os.path.isdir(
os.path.join(options["general"]["save_path"], options['name'],
'models')):
os.mkdir(
os.path.join(options["general"]["save_path"], options['name'],
'models'))
if (options["general"]['model_load']):
path = glob(
os.path.join(options["general"]["save_path"], options['name'],
'models', 'model{}.pth'.format(args.start - 1)))
#path = sorted(glob(os.path.join(options["general"]["save_path"], options['name'], 'models', '*.pth')), key=lambda name : int(name.split('/')[-1].replace('.pth', '').replace('model', '')))
if path:
print('load {} model..'.format(path[-1]))
model.load_state_dict(torch.load(path[-1]))
#path = sorted(glob(os.path.join(options["general"]["save_path"], options['name'], 'optimizers', '*.pth')), key=lambda name : int(name.split('/')[-1].replace('.pth', '').replace('optimizer', '')))
path = glob(
os.path.join(options["general"]["save_path"], options['name'],
'optimizers',
'optimizer{}.pth'.format(args.start - 1)))
if path:
print('load {} optimizer..'.format(path[-1]))
optimizer.load_state_dict(torch.load(path[-1]))
train_dataset = LipreadingDataset(options["training"]["data_path"],
"train", options['input']['aug'],
options['input']['landmark'],
options['input']['landmark_seperate'])
train_dataloader = DataLoader(train_dataset,
batch_size=options["input"]["batch_size"],
shuffle=options["input"]["shuffle"],
num_workers=options["input"]["num_worker"],
drop_last=True)
val_dataset = LipreadingDataset(options['validation']['data_path'], "val",
False, options['input']['landmark'],
options['input']['landmark_seperate'])
val_dataloader = DataLoader(val_dataset,
batch_size=options["input"]["batch_size"],
shuffle=options["input"]["shuffle"],
num_workers=options["input"]["num_worker"],
drop_last=True)
train_size = len(train_dataset)
batch_size = options["input"]["batch_size"]
stats_frequency = options["training"]["stats_frequency"]
if args.test:
test_dataset = LipreadingDataset(options['validation']['data_path'],
"test", False,
options['input']['landmark'],
options['input']['landmark_seperate'])
test_dataloader = DataLoader(
test_dataset,
batch_size=options["input"]["batch_size"],
shuffle=options["input"]["shuffle"],
num_workers=options["input"]["num_worker"],
drop_last=True)
model.eval()
with torch.no_grad():
print("Starting testing...")
count = 0
validator_function = model.validator_function()
for i_batch, sample_batched in enumerate(test_dataloader):
input = sample_batched[0]
labels = sample_batched[1]
input = input.cuda()
labels = labels.cuda()
outputs = model(input)
count += validator_function(outputs, labels)
accuracy = count / len(val_dataset)
print('#############test result################')
print('correct count: {}, total count: {}, accu: {}'.format(
count, len(test_dataset), accuracy))
# with open(os.path.join('./', options['name']+'.txt'), "a") as outputfile:
# outputfile.write("\ncorrect count: {}, total count: {} accuracy: {}" .format(count, len(test_dataset), accuracy ))
return
for epoch in range(options["training"]["start_epoch"],
options["training"]["max_epoch"]):
model.train()
if (options["training"]["train"]):
if options['training']['schedule'] and not plat:
scheduler.step(epoch)
running_loss = 0.0
count = 0
count_bs = 0
startTime = datetime.now()
print("Starting training...")
for i_batch, sample_batched in enumerate(train_dataloader):
optimizer.zero_grad()
if options['input']['landmark_seperate']:
x = sample_batched[0].cuda()
labels = sample_batched[1].cuda()
dot_labels = sample_batched[2].float().cuda()
else:
x = sample_batched[0].cuda()
labels = sample_batched[1].cuda()
if not options['input']['landmark_seperate']:
outputs = model(x)
else:
outputs = model(x, dot_labels)
loss = criterion(outputs, labels)
count += model.validator_function()(outputs, labels)
count_bs += labels.shape[0]
running_loss += loss.item()
loss.backward()
optimizer.step()
if (i_batch % stats_frequency == 0 and i_batch != 0):
print('[%d, %5d] loss: %.8f, acc: %f' %
(epoch + 1, i_batch + 1,
running_loss / stats_frequency, count / count_bs))
writer.add_scalar(
'train/loss', running_loss / stats_frequency,
(epoch - 1) * train_size + i_batch * batch_size)
writer.add_scalar('train/accuracy', count / count_bs,
(epoch - 1) * train_size +
i_batch * batch_size)
writer.add_scalar('train/true', count,
(epoch - 1) * train_size +
i_batch * batch_size)
writer.add_scalar('train/false', count_bs - count,
(epoch - 1) * train_size +
i_batch * batch_size)
try:
print('lr {}, name {}'.format(
optimizer.param_groups[-1]['lr'], options['name']))
writer.add_scalar(
'train/lr', optimizer.param_groups[-1]['lr'],
(epoch - 1) * train_size + i_batch * batch_size)
except Exception as e:
print('lr {}, name {}'.format(
options['training']['learning_rate'],
options['name']))
writer.add_scalar(
'train/lr', options['training']['learning_rate'],
(epoch - 1) * train_size + i_batch * batch_size)
running_loss = 0.0
count = 0
count_bs = 0
currentTime = datetime.now()
output_iteration(i_batch * batch_size,
currentTime - startTime, train_size)
print("Epoch completed")
if options['general']['model_save'] and options['training']['train']:
print("saving state..")
torch.save(
model.state_dict(),
os.path.join(options["general"]["save_path"], options['name'],
'models', 'model{}.pth'.format(epoch + 1)))
torch.save(
optimizer.state_dict(),
os.path.join(options["general"]["save_path"], options['name'],
'optimizers',
'optimizer{}.pth'.format(epoch + 1)))
model.eval()
with torch.no_grad():
if (options["validation"]["validate"]):
print("Starting validation...")
count = 0
validator_function = model.validator_function()
for i_batch, sample_batched in enumerate(val_dataloader):
if options['input']['landmark_seperate']:
x = sample_batched[0].cuda()
labels = sample_batched[1].cuda()
dot_labels = sample_batched[2].float().cuda()
else:
x = sample_batched[0].cuda()
labels = sample_batched[1].cuda()
if not options['input']['landmark_seperate']:
outputs = model(x)
else:
outputs = model(x, dot_labels)
count += validator_function(outputs, labels)
accuracy = count / len(val_dataset)
if options['training']['schedule'] and plat:
scheduler.step(accuracy)
writer.add_scalar('val/accuracy', accuracy, epoch)
writer.add_scalar('val/true', count, epoch)
writer.add_scalar('val/false', len(val_dataset) - count, epoch)
print('correct count: {}, total count: {}, accu: {}'.format(
count, len(val_dataset), accuracy))
with open(os.path.join('./', options['name'] + '.txt'),
"a") as outputfile:
outputfile.write(
"\ncorrect count: {}, total count: {} accuracy: {}".
format(count, len(val_dataset), accuracy))
import math
import sys
import torch
import time
import numpy as np
with open(sys.argv[1], 'r') as optFile:
opt = toml.loads(optFile.read())
if (opt["general"]["usecudnnbenchmark"] and opt["general"]["usecudnn"]):
print("Running cudnn benchmark...")
torch.backends.cudnn.benchmark = True
writer = SummaryWriter(comment=opt["general"]["comment"])
train_dataset = LipreadingDataset(opt["training"]["dataset"], "train")
train_dataloader = DataLoader(train_dataset,
batch_size=opt["input"]["batchsize"],
shuffle=opt["input"]["shuffle"],
num_workers=opt["input"]["numworkers"],
drop_last=False)
val_dataset = LipreadingDataset(opt["validation"]["dataset"], "val")
val_dataloader = DataLoader(val_dataset,
batch_size=opt["input"]["batchsize"],
shuffle=opt["input"]["shuffle"],
num_workers=opt["input"]["numworkers"],
drop_last=False)
model = LRGANModel()
model.initialize(opt)
from data import LipreadingDataset_LR as LipreadingDataset
from torch.utils.data import DataLoader
from tensorboardX import SummaryWriter
import torch.nn as nn
import os
import pdb
import math
import sys
import torch
import time
import numpy as np
with open(sys.argv[1], 'r') as optFile:
opt = toml.loads(optFile.read())
val_dataset = LipreadingDataset(opt["validation"]["dataset"], "val")
val_dataloader = DataLoader(val_dataset,
batch_size=opt["input"]["batchsize"],
shuffle=opt["input"]["shuffle"],
num_workers=opt["input"]["numworkers"],
drop_last=False)
model = LRGANModel()
model.initialize(opt)
print('Start Testing...')
model.set_eval()
count = np.array([0, 0, 0, 0])
acc = np.array([0.0, 0.0, 0.0, 0.0])
val_loss = np.array([0.0, 0.0, 0.0, 0.0])
len_dataset = len(val_dataloader)