def __init__(self, args):
augment = False
shuffle = False
self.channel = args.channel
self.batchsize = args.batch_size
self.validationdataset = LipreadingDataset(args.dataset,
"val",
augment=augment,
channel=self.channel)
self.validationdataloader = DataLoader(self.validationdataset,
batch_size=self.batchsize,
shuffle=shuffle,
num_workers=args.workers,
drop_last=True)
# self.usecudnn = options["general"]["usecudnn"]
self.statsfrequency = args.statsfrequency
# self.gpuid = options["general"]["gpuid"]
self.log_file = args.logfile
self.savedir = args.save_dir
self.num_batches = int(len(self.validationdataset) / self.batchsize)
self.num_samples = int(len(self.validationdataset))
self.num_frames = args.num_frames
self.modelname = args.modelname
print_log('loaded validation dataset with %d data' %
len(self.validationdataset),
log=self.log_file)
def __init__(self, args, input_dims=256):
super(C3D_CONV_CONV, self).__init__()
channel = args.channel
print_log('channel is %d' % channel, log=args.logfile)
self.frontend = ConvFrontend(channel)
self.resnet = ResNetBBC(input_dims, args.batch_size)
self.backend = ConvBackend(args.num_classes)
# self.frontend.apply(freeze)
# self.resnet.apply(freeze)
#function to initialize the weights and biases of each module. Matches the
#classname with a regular expression to determine the type of the module, then
#initializes the weights for it.
def weights_init(m):
classname = m.__class__.__name__
if re.search("Conv[123]d", classname):
m.weight.data.normal_(0.0, 0.02)
elif re.search("BatchNorm[123]d", classname):
m.weight.data.fill_(1.0)
m.bias.data.fill_(0)
elif re.search("Linear", classname):
m.bias.data.fill_(0)
#Apply weight initialization to every module in the model.
self.apply(weights_init)
def test_print_log():
str_to_print = '123'
log_file = 'log.txt'
log_file = open(log_file, 'w')
print_log(str_to_print, log=log_file, same_line=True)
print_log(str_to_print, log=log_file, same_line=True)
print_log(str_to_print, log=log_file, same_line=True)
print_log(str_to_print, log=log_file, same_line=True)
log_file.close()
def __init__(self, options):
self.batchsize = options["input"]["batchsize"]
self.validationdataset = LipreadingDataset(
options["general"]["dataset"], "val", False)
self.validationdataloader = DataLoader(
self.validationdataset,
batch_size=self.batchsize,
shuffle=False,
num_workers=options["input"]["numworkers"],
drop_last=True)
self.usecudnn = options["general"]["usecudnn"]
self.statsfrequency = options["training"]["statsfrequency"]
self.gpuid = options["general"]["gpuid"]
self.log_file = options["general"]["logfile"]
self.savedir = options["general"]["modelsavedir"]
self.num_batches = int(len(self.validationdataset) / self.batchsize)
print_log('loaded validation dataset with %d data' %
len(self.validationdataset),
log=self.log_file)
def display(self, log_file):
"""Display Configuration values."""
print_log('\nConfigurations:', log=log_file)
for a in dir(self):
if not a.startswith("__") and not callable(getattr(self, a)):
print_log('{:30} {}'.format(a, getattr(self, a)), log_file)
print_log('\n', log=log_file)
def __init__(self, args):
augment = True
shuffle = True
self.channel = args.channel
self.batchsize = args.batch_size
self.trainingdataset = LipreadingDataset(args.dataset,
"train",
augment=augment,
channel=self.channel)
self.trainingdataloader = DataLoader(self.trainingdataset,
batch_size=self.batchsize,
shuffle=shuffle,
num_workers=args.workers,
drop_last=True)
# self.usecudnn = options["general"]["usecudnn"]
self.statsfrequency = args.statsfrequency
# self.gpuid = options["general"]["gpuid"]
self.learningrate = args.lr
self.weightdecay = args.weight_decay
self.momentum = args.momentum
self.log_file = args.logfile
self.modelsavedir = args.save_dir
# _, self.savename, _ = fileparts(self.modelsavedir)
self.num_batches = int(len(self.trainingdataset) / self.batchsize)
self.num_samples = int(len(self.trainingdataset))
self.num_frames = args.num_frames
self.modelname = args.modelname
print_log('loaded training dataset with %d data' %
len(self.trainingdataset),
log=self.log_file)
if augment: print_log('using augmentation', log=self.log_file)
else: print_log('no data augmentation', log=self.log_file)
def __init__(self, options):
self.batchsize = options["input"]["batchsize"]
self.trainingdataset = LipreadingDataset(options["general"]["dataset"],
"train")
self.trainingdataloader = DataLoader(
self.trainingdataset,
batch_size=self.batchsize,
shuffle=options["training"]["shuffle"],
num_workers=options["input"]["numworkers"],
drop_last=True)
self.usecudnn = options["general"]["usecudnn"]
self.statsfrequency = options["training"]["statsfrequency"]
self.gpuid = options["general"]["gpuid"]
self.learningrate = options["training"]["learningrate"]
# self.modelType = options["training"]["learningrate"]
self.weightdecay = options["training"]["weightdecay"]
self.momentum = options["training"]["momentum"]
self.log_file = options["general"]["logfile"]
self.modelsavedir = options["general"]["modelsavedir"]
_, self.time_str, _ = fileparts(self.modelsavedir)
print_log('loaded training dataset with %d data' %
len(self.trainingdataset),
log=options["general"]["logfile"])
def epoch(self, model, epoch):
print_log("Starting validation...", log=self.log_file)
count = 0
validator_function = model.validator_function()
for i_batch, (sample_batched,
filename_batch) in enumerate(self.validationdataloader):
with torch.no_grad():
input = Variable(sample_batched['temporalvolume'])
labels = sample_batched['label']
if (self.usecudnn):
input = input.cuda(self.gpuid)
labels = labels.cuda(self.gpuid) # num_batch x 1
outputs = model(
input
) # num_batch x 500 for temp-conv num_batch x 29 x 500
count_tmp, predict_index_list = validator_function(
outputs, labels)
count += count_tmp
for batch_index in range(self.batchsize):
filename_tmp = filename_batch[batch_index]
_, filename_tmp, _ = fileparts(filename_tmp)
filename_tmp = filename_tmp.split('_')[0]
prediction_tmp = self.validationdataset.label_list[
predict_index_list[batch_index]]
print_log(
'Evaluation: val set, batch index %d/%d, filename: %s, prediction: %s'
% (batch_index + 1, self.batchsize, filename_tmp,
prediction_tmp),
log=self.log_file)
print_log(
'Evaluation: val set, batch %d/%d, correct so far %d/%d' %
(i_batch + 1, self.num_batches, count, self.batchsize *
(i_batch + 1)),
log=self.log_file)
accuracy = count / len(self.validationdataset)
accu_savepath = os.path.join(self.savedir,
'accuracy_epoch%03d.txt' % epoch)
print_log('saving the accuracy file to %s' % accu_savepath,
log=self.log_file)
with open(accu_savepath, "a") as outputfile:
outputfile.write(
"\ncorrect count: {}, total count: {} accuracy: {}".format(
count, len(self.validationdataset), accuracy))
def epoch(self, model, epoch):
print_log("Starting validation...", log=self.log_file)
criterion = model.loss()
validator_function = model.validator_function()
sum_loss_so_far, corrects_so_far, sum_samples_so_far = 0., 0., 0.
for i_batch, (sample_batched,
filename_batch) in enumerate(self.validationdataloader):
with torch.no_grad():
inputs = Variable(sample_batched['temporalvolume']).cuda()
labels = sample_batched['label'].cuda()
# if(self.usecudnn):
# inputs = inputs.cuda(self.gpuid)
# labels = labels.cuda(self.gpuid) # num_batch x 1
outputs = model(
inputs
) # num_batch x 500 for temp-conv num_batch x 29 x 500
loss = criterion(outputs, labels.squeeze(1))
# ave_loss_per_batch = loss.item() / float(self.num_frames)
# ave_loss_per_batch = loss.item() / 7. # TODO only true for lstm model
ave_loss_per_batch = loss.item()
sum_loss_so_far += ave_loss_per_batch * inputs.size(0)
corrects_per_batch, predict_index_list = validator_function(
outputs, labels)
corrects_so_far += corrects_per_batch
sum_samples_so_far += self.batchsize
print_log(
'%s, val, Epoch: %d, %d/%d (%.f%%), Loss: %.4f, Accu: %.4f'
% (self.modelname, epoch, sum_samples_so_far,
self.num_samples, 100. * i_batch /
(self.num_batches - 1), ave_loss_per_batch,
corrects_per_batch / float(self.batchsize)),
log=self.log_file)
ave_loss_per_epoch = sum_loss_so_far / sum_samples_so_far
ave_accu_per_epoch = corrects_so_far / sum_samples_so_far # debug: to test the number is the same
print_log(
'val, Epoch: {}, Average Loss: {:.4f}, Average Accuracy: {:.4f}'.
format(epoch, ave_loss_per_epoch, ave_accu_per_epoch) + '\n',
log=self.log_file)
return ave_loss_per_epoch, ave_accu_per_epoch
def epoch(self, model, epoch):
#set up the loss function.
criterion = model.loss()
optimizer = optim.SGD(model.parameters(),
lr=self.learningRate(epoch),
momentum=self.learningrate,
weight_decay=self.weightdecay)
#transfer the model to the GPU.
if self.usecudnn: criterion = criterion.cuda(self.gpuid)
startTime = datetime.now()
print_log("Starting training...", log=self.log_file)
for i_batch, (sample_batched, _) in enumerate(self.trainingdataloader):
optimizer.zero_grad()
input = Variable(sample_batched['temporalvolume'])
labels = Variable(sample_batched['label'])
if (self.usecudnn):
input = input.cuda(self.gpuid)
labels = labels.cuda(self.gpuid)
outputs = model(input)
loss = criterion(outputs, labels.squeeze(1))
print_log('Training: {}, Epoch: {}, loss is {}'.format(
self.time_str, epoch, loss.item()),
log=self.log_file)
loss.backward()
optimizer.step()
sampleNumber = i_batch * self.batchsize
if sampleNumber % self.statsfrequency == 0:
currentTime = datetime.now()
output_iteration(sampleNumber, currentTime - startTime,
len(self.trainingdataset), self.log_file)
print_log("Epoch completed, saving state...", log=self.log_file)
torch.save(
model.state_dict(),
os.path.join(self.modelsavedir,
'trained_model_epoch%03d.pt' % epoch))
def output_iteration(i, time, totalitems, log_file):
avgBatchTime = time / (i + 1)
estTime = avgBatchTime * (totalitems - i)
print_log("Iteration: {}\nElapsed Time: {} \nEstimated Time Remaining: {}".
format(i, timedelta_string(time), timedelta_string(estTime)),
log=log_file)
def compute_metrics(gt_json, pred_json, split='test', total_sample=20):
results_dir = fileparts(pred_json)[0]
log_file = os.path.join(results_dir, 'eval_traj_log.txt')
log_file = open(log_file, 'w')
# data loading
print_log('loading GT from %s' % gt_json, log_file)
with open(gt_json, 'r') as file:
gt_data = json.load(file)
print_log('loading results from %s' % pred_json, log_file)
with open(pred_json, 'r') as file:
pred_data = json.load(file)
# 4 metrics x 3 settings x 4 classes x N windows
metrics = ['ADE', 'FDE', 'APD', 'FPD', 'MsR'] # set up metrics
stats_meter = {}
for pred_len in [10, 20, 50]: # 1s, 2s, 5s prediction settings
skip = len2skip[pred_len]
for obj_class in ['Car', 'Ped', 'Cyc', 'Mot']:
for metric_tmp in metrics:
stats_meter['%s_%d_%s' % (metric_tmp, pred_len,
obj_class)] = AverageMeter()
gt_obj = gt_data[obj_class]
if str(pred_len) not in pred_data.keys(): continue
pred_data_len = pred_data[str(
pred_len)] # get prediction at a specific pred len
if obj_class not in pred_data_len.keys(): continue
pred_data_obj = pred_data_len[
obj_class] # get prediction for a specific class
# loop through based on GT data so that missing prediction can be handled
num_total_obj_valid, num_miss = 0, 0
for seqname, gt_seq in gt_obj.items():
gt_seq = np.array(gt_seq) # N x 4
# frame-centric evaluation, go through every possible window of frames, e.g., frame 1-20, 2-21 for 10->10
frames = np.unique(gt_seq[:, 0]).tolist()
min_frame, max_frame = frames[0], frames[-1]
num_windows = int(max_frame - min_frame + 1 - pred_len * 2 +
skip) # this additional + 1 is correct
for window_index in range(num_windows):
start_frame = int(window_index + min_frame + pred_len)
end_frame = int(start_frame + pred_len)
# check which window to be dropped
check_pass = check_eval_windows(start_frame, pred_len,
split)
if not check_pass: continue
####### filter frame windows, reserver any window with past data >= 1 frame, future data >= 1 frame
# filter out frame window where there is no GT inside
gt_window = []
for frame in range(start_frame, end_frame, skip):
gt_window.append(gt_seq[frame == gt_seq[:, 0], :])
gt_window = np.concatenate(
gt_window,
axis=0) # N x 4, within a particular time window
if gt_window.shape[0] == 0: continue
ID_list = np.unique(gt_window[:, 1]).tolist() # get GT IDs
# filter out frame window where there is no past observations at all
past_start = start_frame - pred_len
gt_window_past = []
for frame in range(past_start, start_frame, skip):
gt_window_past.append(gt_seq[frame == gt_seq[:, 0], :])
gt_window_past = np.concatenate(
gt_window_past,
axis=0) # N x 4, within a particular time window
if gt_window_past.shape[0] == 0: continue
# filter out ID in this frame window when this ID has no past observations
for ID_tmp in ID_list:
gt_past = copy.copy(
gt_window_past[gt_window_past[:, 1] ==
float(ID_tmp), :]
) # (<=pred_len/skip) x 2, might have missing data
num_past_frames = gt_past.shape[0]
if num_past_frames == 0:
ID_list, _ = remove_unique_item_from_list(
ID_list, ID_tmp)
if len(ID_list) == 0: continue
num_total_obj_valid += len(ID_list)
# check prediction if have data in this seq and frame, otherwise all objects are missed
try:
pred_window = pred_data_obj[seqname][str(
start_frame)] # {sample: {ID: {state, prob}}}
except:
try:
pred_window = pred_data_obj[seqname][
start_frame] # {sample: {ID: {state, prob}}}
except:
print('No prediction data found for len of %d, %s, seq %s, frame %s' % \
(pred_len, obj_class, seqname, start_frame))
num_miss += len(ID_list)
continue
######## get predictions within this window
sample_count = 0
ade_list, fde_list = [], []
pred_all_sample, gt_all_sample = [], []
# sample must be in the outer loop to compute scene-specific min of ADE
for sample_index, pred_tmp in pred_window.items():
best_k_pred, best_k_gt = [], []
gt_mask = np.zeros(
(len(ID_list),
int(pred_len /
skip))) # initialize mask on missing frames
keep_ID_index = []
for ID_count in range(len(ID_list)):
ID_tmp = ID_list[ID_count]
# check if prediction has data for this ID, otherwise this object is missed
try:
pred_ID = pred_tmp[str(int(ID_tmp))]
except:
try:
pred_ID = pred_tmp[int(ID_tmp)]
except:
if int(sample_index) == 0:
num_miss += 1
print('\nID %d missied in prediction data for len of %d, %s, seq %s, frame %s\n' % \
(ID_tmp, pred_len, obj_class, seqname, start_frame))
continue
# get prediction states
state, prob = pred_ID['state'], pred_ID['prob']
state = np.array(state) # pred_len/skip x 2
assert state.shape[0] == pred_len / skip, 'error'
# get GT states
gt_ID = copy.copy(
gt_window[gt_window[:, 1] == float(ID_tmp), :]
) # (<=pred_len/skip) x 2, might have missing data
assert gt_ID.shape[
0] > 0, 'error' # this is guaranteed, as this ID exists so there is at least one frame of data for this ID
frame_exist = gt_ID[:, 0].tolist(
) # get existing frames before completion
if gt_ID.shape[0] < pred_len / skip:
gt_ID = complete_data_window(
gt_ID, start_frame, end_frame,
skip) # pred_len/skip x 2
# get GT mask
frame_exist_index = np.array([
frame_tmp - start_frame
for frame_tmp in frame_exist
])
frame_exist_index = (frame_exist_index /
skip).astype('uint8')
gt_mask[ID_count, frame_exist_index] = 1
# store both prediction and GT
keep_ID_index.append(ID_count)
best_k_pred.append(state)
best_k_gt.append(
gt_ID[:, 2:]
) # take only the state (last two columns)
# compute ADE/FDE in this sample
gt_mask = gt_mask[keep_ID_index, :]
best_k_pred = np.stack(
best_k_pred, axis=0) # obj x pred_len/skip x 2
best_k_gt = np.stack(best_k_gt,
axis=0) # obj x pred_len/skip x 2
ade_tmp = compute_ADE(best_k_pred,
best_k_gt,
mask=gt_mask)
fde_tmp = compute_FDE(best_k_pred,
best_k_gt,
mask=gt_mask)
ade_list.append(ade_tmp)
fde_list.append(fde_tmp)
# aggregate all samples
pred_all_sample.append(best_k_pred)
gt_all_sample.append(best_k_gt)
# evaluation allows up to the first 20 samples
sample_count += 1
if sample_count > total_sample: break
# check if every sample has the same shape, i.e., has the same number of predictions of IDs
try:
ade_array = np.stack(ade_list, axis=1) # obj x samples
fde_array = np.stack(fde_list, axis=1) # obj x samples
pred_all_sample = np.stack(
pred_all_sample,
axis=0) # samples x obj x frames x 2
gt_all_sample = np.stack(gt_all_sample, axis=0)
assert pred_all_sample.shape == gt_all_sample.shape, 'error'
except ValueError:
assert False, 'each sample has predictions with different set of IDs, please make sure every sample has the same shape of prediction (same set of IDs)'
# compute best of K ADE/FDE
ade_best, fde_best = best_of_K(ade_array, fde_array)
num_obj = ade_array.shape[
0] # might less than len(ID_list) as some IDs are not contained in predictions
stats_meter['ADE_%s_%s' % (pred_len, obj_class)].update(
ade_best, n=num_obj)
stats_meter['FDE_%s_%s' % (pred_len, obj_class)].update(
fde_best, n=num_obj)
# compute APD/FPD for diversity
apd = compute_APD(pred_all_sample)
fpd = compute_FPD(pred_all_sample)
stats_meter['APD_%s_%s' % (pred_len, obj_class)].update(
apd, n=num_obj)
stats_meter['FPD_%s_%s' % (pred_len, obj_class)].update(
fpd, n=num_obj)
# only copying the ones under the current class when logging
stats_meter_show = {}
for metric_tmp, values in stats_meter.items():
if (str(pred_len)
in metric_tmp) and obj_class in metric_tmp:
stats_meter_show[metric_tmp] = values
# logging
stats_str = ' '.join([
f'{x[:3]}: {y.val:6.3f} ({y.avg:6.3f})'
for x, y in stats_meter_show.items()
])
print_str = 'Len: %d, %s, %s, %03d-%03d, %s\r' % \
(pred_len, obj_class, seqname, start_frame, end_frame-1, stats_str)
print_log(print_str, log_file, display=False)
sys.stdout.write(print_str)
sys.stdout.flush()
# compute missing rate metric
missrate = num_miss * 1.0 / num_total_obj_valid
stats_meter['MsR_%s_%s' % (pred_len, obj_class)].update(
missrate, n=num_total_obj_valid)
assert missrate == stats_meter['MsR_%s_%s' %
(pred_len, obj_class)].avg, 'error'
if num_miss > 0:
print('\n\nMissing IDs: %d, total IDs: %d, miss rate: %.3f for len of %d, %s\n' % \
(num_miss, num_total_obj_valid, missrate, pred_len, obj_class))
################ logging
print_log('', log_file)
metric_all = dict()
for pred_len in [10, 20, 50]:
# compute average ADE/FDE/APD/FPD/MsR
for metric in ['ADE', 'FDE', 'APD', 'FPD', 'MsR']:
metric_all['%s_%d' % (metric, pred_len)] = list()
for obj_class in ['Car', 'Ped', 'Cyc', 'Mot']:
key_tmp = '%s_%d_%s' % (metric, pred_len, obj_class)
num_value = stats_meter[key_tmp].count
if num_value == 0: tmp_value = np.nan
else: tmp_value = stats_meter[key_tmp].avg
metric_all['%s_%d' % (metric, pred_len)].append(tmp_value)
metric_all['%s_%d' % (metric, pred_len)] = np.array(
metric_all['%s_%d' % (metric, pred_len)]).mean()
# logging header
print_log('', log_file)
print_log('-' * 30 + ' STATS for length of %d ' % pred_len + '-' * 30,
log_file)
print_log(' ' * 5, log_file, same_line=True)
print_log('ADE_%d'%pred_len+' '*2+'FDE_%d'%pred_len+' '*2+'APD_%d'%pred_len+' '*2+'FPD_%d'%pred_len+' '*2+'MsR_%d'%pred_len+' '*2, \
log_file, same_line=True)
print_log('', log_file)
# logging table: row class, column metrics
cur_class = 'Car'
cur_index = 0
for name, meter in stats_meter.items():
num_value = meter.count
if num_value == 0: final_value = np.nan
else: final_value = meter.avg
if str(pred_len) not in name: continue
# check to see if switch from one class to the other
if cur_class not in name:
print_log('', log_file)
cur_class = name[-3:]
cur_index = 0
# print all errors for this class
if cur_index == 0:
print_log('%s: ' % name[-3:], log_file, same_line=True)
print_log('%6.3f ' % final_value, log_file, same_line=True)
cur_index += 1
# print average value
print_log('', log_file)
print_log('Ave: ', log_file, same_line=True)
for metric in ['ADE', 'FDE', 'APD', 'FPD', 'MsR']:
ave_value = metric_all['%s_%d' % (metric, pred_len)]
print_log('%6.3f ' % ave_value, log_file, same_line=True)
# print end of the logging
print_log('', log_file)
print_log('-' * 84, log_file)
log_file.close()
# filter out metrics other than len of 20
output = dict()
for key, value in metric_all.items():
if '20' in key: # only add length of 20 to output metric for EvalAI leaderboard
output[key] = value
pprint.pprint(output) # print output metrics
return output
parser.add_argument('--seed', type=int, default=0, help='random seed')
args = parser.parse_args()
torch.backends.cudnn.benchmark = True
prepare_seed(args.seed)
print("Loading options...")
with open('options.toml', 'r') as optionsFile:
options = toml.loads(optionsFile.read())
args.save_dir = os.path.join(options["general"]["modelsavedir"],
args.modelname + '_' + get_timestring())
mkdir_if_missing(args.save_dir)
args.dataset = options["general"]["dataset"]
args.logfile = os.path.join(args.save_dir, 'log.txt')
args.logfile = open(args.logfile, 'w')
# print_log(options, args.logfile)
print_log(args, args.logfile)
print_log('\n\nsaving to %s' % args.save_dir, log=args.logfile)
print_log('creating the model\n\n', log=args.logfile)
if args.modelname == 'C3D_CONV_BLSTM': model = C3D_CONV_BLSTM(args)
elif args.modelname == 'C3D_CONV_BLSTM_frontfix': model = C3D_CONV_BLSTM(args)
elif args.modelname == 'C3D_CONV_CONV': model = C3D_CONV_CONV(args)
elif args.modelname == 'I3D_BLSTM': model = I3D_BLSTM()
elif args.modelname == 'I3D': model = I3D()
elif args.modelname == 'I3D_BLSTM_mini': model = I3D_BLSTM_mini()
elif args.modelname == 'I3D_BLSTM_mini2': model = I3D_BLSTM_mini2()
elif args.modelname == 'I3D_BLSTM_mini3': model = I3D_BLSTM_mini3()
print_log(model, log=args.logfile)
model = reload_model(model, args.logfile, args.path) # reload model
model = model.cuda() # move the model to the GPU.
# model_path = os.path.join(root_dir, 'resnet50_imagenet.pth') # Path to trained weights file
if train_dataset == 'coco':
model_path = os.path.join(
root_dir, 'models/mask_rcnn_coco.pth') # Path to trained weights file
else:
assert False, 'error'
model = MaskRCNN(model_dir=save_dir, config=config) # Create model object.
if config.GPU_COUNT: model = model.cuda()
model.load_weights(model_path) # Load weights
##--------------------------------- Testing ----------------------------------##
image_list, num_list = load_list_from_folder(images_dir,
ext_filter=['.png', '.jpg'],
depth=2)
print_log('testing results on %d images' % num_list, log=log_file)
count = 1
timer = Timer()
timer.tic()
for image_file_tmp in image_list:
parent_dir, filename, _ = fileparts(image_file_tmp)
video_dir = parent_dir.split('/')[-1]
# print(video_dir)
# zxc
image = load_image(image_file_tmp)
results = model.detect([image]) # inference, results is a dictionary
if len(results) == 0:
count += 1
print_log('Mask-RCNN demo: testing %d/%d, no detected results!!!!!' %
(count, num_list),
def epoch(self, model, epoch):
#set up the loss function.
criterion = model.loss().cuda()
optimizer = optim.SGD(model.parameters(),
lr=self.learningRate(epoch),
momentum=self.momentum,
weight_decay=self.weightdecay)
validator_function = model.validator_function()
print_log('\n\nlearning args: lr, momentum and weight decay are',
log=self.log_file)
for param_group in optimizer.param_groups:
print_log(param_group['lr'], log=self.log_file)
print_log(param_group['momentum'], log=self.log_file)
print_log(param_group['weight_decay'], log=self.log_file)
# zxc
#transfer the model to the GPU.
# if self.usecudnn: criterion = criterion.cuda()
# startTime = datetime.now()
print_log("\n\nStarting training...", log=self.log_file)
sum_loss_so_far, corrects_so_far, sum_samples_so_far = 0., 0., 0.
for i_batch, (sample_batched, _) in enumerate(self.trainingdataloader):
optimizer.zero_grad()
inputs = Variable(sample_batched['temporalvolume']).cuda()
labels = Variable(sample_batched['label']).cuda()
# if(self.usecudnn):
# inputs = inputs.cuda()
# labels = labels.cuda()
outputs = model(inputs)
loss = criterion(outputs, labels.squeeze(1))
loss.backward()
optimizer.step()
# ave_loss_per_batch = loss.item() / float(self.num_frames) # TODO only true for lstm model
# ave_loss_per_batch = loss.item() / 7. # TODO only true for lstm model
ave_loss_per_batch = loss.item()
sum_loss_so_far += ave_loss_per_batch * inputs.size(0)
corrects_per_batch, predict_index_list = validator_function(
outputs, labels)
corrects_so_far += corrects_per_batch
sum_samples_so_far += self.batchsize
if i_batch == 0: since = time.time()
elif i_batch % self.statsfrequency == 0 or (i_batch
== self.num_batches -
1):
print_log(
'%s, train, Epoch: %d, %d/%d (%.f%%), Loss: %.4f, Accu: %.4f, EP: %s, ETA: %s'
% (self.modelname, epoch, sum_samples_so_far,
self.num_samples, 100. * i_batch /
(self.num_batches - 1), ave_loss_per_batch,
corrects_per_batch / float(self.batchsize),
convert_secs2time(time.time() - since),
convert_secs2time((time.time() - since) *
(self.num_batches - 1) / i_batch -
(time.time() - since))),
log=self.log_file)
ave_loss_per_epoch = sum_loss_so_far / sum_samples_so_far
ave_accu_per_epoch = corrects_so_far / sum_samples_so_far # debug: to test the number is the same
print_log(
'train, Epoch: {}, Average Loss: {:.4f}, Average Accuracy: {:.4f}'.
format(epoch, ave_loss_per_epoch, ave_accu_per_epoch) + '\n',
log=self.log_file)
print_log("Epoch completed, saving state...", log=self.log_file)
torch.save(
model.state_dict(),
os.path.join(self.modelsavedir,
'%s_epoch%03d.pt' % (self.modelname, epoch)))
return ave_loss_per_epoch, ave_accu_per_epoch
vis_dir = os.path.join(save_dir, 'visualization'); mkdir_if_missing(vis_dir)
log_file = os.path.join(save_dir, 'log.txt'); log_file = open(log_file, 'w')
bbox_eval_folder = os.path.join(save_dir, 'data'); mkdir_if_missing(bbox_eval_folder)
mask_dir = os.path.join(save_dir, 'masks'); mkdir_if_missing(mask_dir)
label_bbox_match_dir = os.path.join(save_dir, 'label_bbox_matching'); mkdir_if_missing(label_bbox_match_dir)
detection_result_filepath = os.path.join(save_dir, 'mask_results.txt'); detection_results_file = open(detection_result_filepath, 'w')
##--------------------------------- Model Directory ----------------------------------##
if train_dataset == 'coco': model_path = os.path.join(root_dir, '../models/mask_rcnn_coco.pth') # Path to trained weights file
elif train_dataset == 'cityscape': model_path = '/media/xinshuo/Data/models/mask_rcnn_pytorch/%s/mask_rcnn_cityscape_%04d.pth' % (model_folder, epoch)
elif train_dataset == 'kitti': model_path = '/media/xinshuo/Data/models/mask_rcnn_pytorch/%s/mask_rcnn_kitti_%04d.pth' % (model_folder, epoch)
else: model_path = os.path.join(root_dir, 'resnet50_imagenet.pth') # Path to trained weights from Imagenet
model = MaskRCNN(model_dir=save_dir, config=config) # Create model object.
if config.GPU_COUNT: model = model.cuda()
model.load_weights(model_path) # Load weights
print_log('load weights from %s' % model_path, log=log_file)
##--------------------------------- Build KITTI Evaluation Results ----------------------------------##
id_list, num_list = load_list_from_file(split_file)
print_log('KITTI Evaluation: testing results on %d images' % num_list, log=log_file)
count = 1
timer = Timer(); timer.tic()
# count_skipped = 0
for id_tmp in id_list:
image_file_tmp = os.path.join(images_dir, id_tmp+'.png')
_, filename, _ = fileparts(image_file_tmp)
bbox_eval_file_tmp = os.path.join(bbox_eval_folder, filename+'.txt'); bbox_eval_file_tmp = open(bbox_eval_file_tmp, 'w')
label_matching_file_tmp = os.path.join(label_bbox_match_dir, filename+'.txt')
label_matching_file_tmp = open(label_matching_file_tmp, 'w')
image = load_image(image_file_tmp)
from validation import Validator
from xinshuo_miscellaneous import get_timestring, print_log
from xinshuo_io import mkdir_if_missing
print("Loading options...")
with open('options.toml', 'r') as optionsFile:
options = toml.loads(optionsFile.read())
if options["general"]["usecudnnbenchmark"] and options["general"]["usecudnn"]:
torch.backends.cudnn.benchmark = True
options["general"]["modelsavedir"] = os.path.join(
options["general"]["modelsavedir"], 'trained_model_' + get_timestring())
mkdir_if_missing(options["general"]["modelsavedir"])
options["general"]["logfile"] = open(
os.path.join(options["general"]["modelsavedir"], 'log.txt'), 'w')
print_log('saving to %s' % options["general"]["modelsavedir"],
log=options["general"]["logfile"])
print_log('creating the model', log=options["general"]["logfile"])
model = LipRead(options)
print_log('loading model', log=options["general"]["logfile"])
if options["general"]["loadpretrainedmodel"]:
print_log('loading the pretrained model at %s' %
options["general"]["pretrainedmodelpath"],
log=options["general"]["logfile"])
model.load_state_dict(torch.load(
options["general"]["pretrainedmodelpath"])) #Create the model.
if options["general"]["usecudnn"]:
model = model.cuda(
options["general"]["gpuid"]) #Move the model to the GPU.
random.seed(args.manualSeed)
np.random.seed(args.manualSeed)
torch.manual_seed(args.manualSeed)
torch.cuda.manual_seed_all(args.manualSeed)
assert args.dataset == 'coco' or args.dataset == 'cityscapes' or args.dataset == 'kitti', 'wrong dataset name'
# Configurations
if args.dataset == 'coco': config = CocoConfig()
elif args.dataset == 'cityscapes': config = CityscapeConfig()
elif args.dataset == 'kitti': config = KITTIConfig()
else: assert False, 'error'
model = MaskRCNN(config=config, model_dir=args.save_dir)
if config.GPU_COUNT: model = model.cuda()
config.display(model.log_file)
print_log('Seed: %d' % args.manualSeed, model.log_file)
print_log('Model: %s' % args.model, model.log_file)
print_log('Dataset: %s' % args.dataset, model.log_file)
print_log('Save Directory: %s' % args.save_dir, model.log_file)
if args.model:
if args.model.lower() == 'imagenet':
model_path = config.IMAGENET_MODEL_PATH # Start from ImageNet trained weights
elif args.model.lower() == 'last':
model_path = model.find_last()[1] # Find last trained weights
else:
model_path = args.model
else:
model_path = ''
# train and evaluate
print_log('Loading weights from %s' % model_path, model.log_file)