def eval(model, split, seq_length, n_cpu, disp):
dataset = GolfDB(data_file='data/val_split_{}.pkl'.format(split),
vid_dir='data/videos_160/',
seq_length=seq_length,
transform=transforms.Compose([ToTensor(),
Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])]),
train=False)
data_loader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=n_cpu,
drop_last=False)
correct = []
delta = []
tolerance = []
predictions = []
predictions_original = []
ground_truth = []
for i, sample in enumerate(data_loader):
images, labels = sample['images'][:,::step,:,:,:], sample['labels']
# full samples do not fit into GPU memory so evaluate sample in 'seq_length' batches
seq_length_new = int(np.ceil(seq_length/step))
batch = 0
while batch * (seq_length_new-1) < images.shape[1]-1:
if (batch + 1) * (seq_length_new-1) + 1 > images.shape[1]:
image_batch = images[:, batch * (seq_length_new-1):, :, :, :]
else:
image_batch = images[:, batch * (seq_length_new-1):(batch + 1) * (seq_length_new-1)+1, :, :, :]
logits = model(image_batch.to(device))
if batch == 0:
probs = F.softmax(logits.data, dim=1).cpu().numpy()
else:
probs = np.append(probs[:-1], F.softmax(logits.data, dim=1).cpu().numpy(), 0)
batch += 1
gt, pp, deltas, tol, c, original = correct_preds(probs, labels.squeeze())
if disp:
print(i, c)
correct.append(c)
tolerance.append(tol)
delta.append(deltas)
predictions.append(pp)
ground_truth.append(gt)
predictions_original.append(original)
np.savez_compressed('results/' + version_name + '.npz', array1=np.array(delta), array2=np.array(predictions),
array3=np.array(tolerance), array4=np.array(ground_truth),
array5=np.array(predictions_original))
print(np.round(np.mean(np.array(correct),0),3))
print(np.round(np.sqrt(np.mean(np.square(np.array(delta)/np.array(tolerance)[:,np.newaxis]),0)),3))
print(np.round(np.std(np.array(delta)/np.array(tolerance)[:,np.newaxis],0),3))
PCE = np.mean(correct)
return PCE
def eval(model, split, seq_length, n_cpu, disp):
dataset = StsqDB(data_file='val_split_{}.pkl'.format(split),
vid_dir='data/videos_40/',
seq_length=seq_length,
transform=transforms.Compose([
ToTensor(),
Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
]),
train=False)
data_loader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=n_cpu,
drop_last=False)
correct = []
for i, sample in enumerate(data_loader):
images, labels = sample['images'], sample['labels']
# full samples do not fit into GPU memory so evaluate sample in 'seq_length' batches
batch = 0
while batch * seq_length < images.shape[1]:
if (batch + 1) * seq_length > images.shape[1]:
image_batch = images[:, batch * seq_length:, :, :, :]
else:
image_batch = images[:, batch * seq_length:(batch + 1) *
seq_length, :, :, :]
logits = model(image_batch.to(device))
if batch == 0:
probs = F.softmax(logits.data, dim=1).to(device).numpy()
else:
probs = np.append(
probs,
F.softmax(logits.data, dim=1).to(device).numpy(), 0)
batch += 1
_, _, _, _, c = correct_preds(probs, labels.squeeze())
if disp:
print(i, c)
correct.append(c)
PCE = np.mean(correct)
return PCE
def main():
global args, writer
args = parser.parse_args()
logger.info('Arguments:')
logger.info(json.dumps(vars(args), indent=1))
# Dataset
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# device = torch.device('cpu')
if device.type != 'cpu':
cudnn.benchmark = True
logger.info(f"device:{device}")
logger.info('=> setting data loader')
reduction = args.level - args.depth_level
fisheye_transform = transforms.Compose([ColorJitter(), RandomShift()]) if args.aug_real else None
# Dataset
root_train_dataset = OmniStereoDataset(args.root_dir, args.train_list, fisheye_transform, fov=args.fov)
ocam_dict = root_train_dataset.ocams
# camera poses world <- T cam
pose_dict = root_train_dataset.pose_dict
transform = transforms.Compose([ToTensor(), Normalize()])
trainset = FisheyeToIcoDataset(root_train_dataset, ocam_dict, pose_dict, level=args.level, reduction=reduction,
transform=transform)
logger.info(f'trainset:{len(trainset)} samples were found.')
train_loader = DataLoader(trainset, args.batch_size, shuffle=True, num_workers=args.workers)
root_val_dataset = OmniStereoDataset(args.root_dir, args.val_list, fov=args.fov)
val_dataset = FisheyeToIcoDataset(root_val_dataset, root_val_dataset.ocams, root_val_dataset.pose_dict,
level=args.level, reduction=reduction, transform=transform)
logger.info(f'{len(val_dataset)} samples were found.')
val_loader = DataLoader(val_dataset, args.batch_size, shuffle=False, num_workers=args.workers)
logger.info('=> setting model')
model = IcoSweepNet(args.root_dir, args.ndisp, args.level, fov=args.fov)
total_params = 0
for param in model.parameters():
total_params += np.prod(param.shape)
logger.info(f"Total model parameters: {total_params:,}.")
model = model.to(device)
invd_0 = model.inv_depths[0]
invd_max = model.inv_depths[-1]
converter = InvDepthConverter(model.ndisp, invd_0, invd_max)
# setup solver scheduler
logger.info('=> setting optimizer')
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
logger.info('=> setting scheduler')
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=int(2 * args.epochs / 3), gamma=0.1)
start_epoch = 0
# Load pretrained model
if args.pretrained:
checkpoint = torch.load(args.pretrained)
param_check = {
'ndisp': model.ndisp,
'min_depth': model.min_depth,
'level': model.level,
}
for key, val in param_check.items():
if not checkpoint[key] == val:
logger.error(f'Error! Key:{key} is not the same as the checkpoints')
logger.info("=> using pre-trained weights")
model.load_state_dict(checkpoint['state_dict'])
start_epoch = checkpoint['epoch']
optimizer.load_state_dict(checkpoint['optimizer'])
scheduler.load_state_dict(checkpoint['scheduler'])
logger.info("=> Resume training from epoch {}".format(start_epoch))
#
model = nn.DataParallel(model)
# setup writer
timestamp = datetime.now().strftime("%m%d-%H%M")
log_folder = join('checkpoints', f'{args.arch}_{timestamp}')
logger.info(f'=> create log folder: {log_folder}')
os.makedirs(log_folder, exist_ok=True)
with open(join(log_folder, 'args.json'), 'w') as f:
json.dump(vars(args), f, indent=1)
writer = SummaryWriter(log_dir=log_folder)
writer.add_text('args', json.dumps(vars(args), indent=1).replace('\n', ' \n'))
logger.info('=> copy models folder to log folder')
shutil.copytree('./models', join(log_folder, 'models'))
# setup logger file handler
handler = FileHandler(join(log_folder, 'train.log'))
handler.setLevel(INFO)
logger.addHandler(handler)
logger.info('Start training')
for epoch in range(start_epoch, args.epochs):
# ----------------------------
# training
mode = 'train'
ave_loss = run(epoch, mode, model, train_loader, converter, device, optimizer)
# ----------------------------
# evaluation
mode = 'val'
ave_loss = run(epoch, mode, model, val_loader, converter, device, optimizer=None, show_metrics=True)
scheduler.step()
# save
save_data = {
'epoch': epoch + 1,
'state_dict': model.module.state_dict(),
'optimizer': optimizer.state_dict(),
'scheduler': scheduler.state_dict(),
'ave_loss': ave_loss,
'ndisp': model.module.ndisp,
'min_depth': model.module.min_depth,
'level': model.module.level,
}
torch.save(save_data, join(log_folder, f'checkpoints_{epoch}.pth'))
writer.close()
logger.info('Finish training.')
net = FCN8s(vgg_model)
checkpoint = torch.load('baseline.pth')
net.load_state_dict(checkpoint['model_state_dict'])
device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device(
'cpu')
net.to(device)
# train dataloader
scale = Rescale(int(1.5 * 230))
crop = RandomCrop(224)
rotate = RandomRotate(20.0)
norm = Normalize()
tupled = ToTupleTensor()
tupled_with_roi_align = ToRoIAlignTensor()
composed_for_tracking = transforms.Compose(
[Rescale(224), norm, tupled_with_roi_align])
dataset = SISSDataset(num_slices=153,
num_scans=2,
root_dir=Path.cwd().parents[0],
transform=composed_for_tracking,
train=True)
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=1,
shuffle=True,
#print('model.py, class EventDetector()')
freeze_layers(k, model)
#print('utils.py, func freeze_laters()')
model.train()
model.to(device)
print('Loading Data')
# TODO: vid_dirのpathをかえる。stsqの動画を切り出したimage全部が含まれているdirにする
if use_no_element == False:
dataset = StsqDB(data_file='data/no_ele/seq_length_{}/train_split_{}.pkl'.format(args.seq_length, args.split),
vid_dir='/home/akiho/projects/golfdb/data/videos_40/',
seq_length=int(seq_length),
transform=transforms.Compose([ToTensor(),
Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])]),
train=True)
else:
dataset = StsqDB(data_file='data/seq_length_{}/train_split_{}.pkl'.format(args.seq_length, args.split),
vid_dir='/home/akiho/projects/golfdb/data/videos_40/',
seq_length=int(seq_length),
transform=transforms.Compose([ToTensor(),
Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])]),
train=True)
print('dataloader.py, class StsqDB()')
# dataset.__len__() : 1050
data_loader = DataLoader(dataset,
batch_size=int(bs),
shuffle=True,
def main():
global args, writer, logger
args = parser.parse_args()
logger.info('Arguments:')
logger.info(json.dumps(vars(args), indent=1))
# Dataset
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# device = torch.device('cpu')
if device.type != 'cpu':
cudnn.benchmark = True
logger.info(f"device:{device}")
# Load pretrained model
logger.info("=> loading checkpoints")
checkpoint = torch.load(args.pretrained)
ndisp = checkpoint['ndisp']
# min_depth = checkpoint['min_depth']
level = checkpoint['level']
logger.info(f'ndisp:{ndisp}')
logger.info(f'level:{level}')
logger.info('=> setting model')
model = IcoSweepNet(args.root_dir, ndisp, level, fov=args.fov)
model = model.to(device)
invd_0 = model.inv_depths[0]
invd_max = model.inv_depths[-1]
converter = InvDepthConverter(model.ndisp, invd_0, invd_max)
model.load_state_dict(checkpoint['state_dict'])
epoch = checkpoint['epoch'] - 1
logger.info("=> Pretrained model epoch {}".format(epoch))
logger.info('=> setting data loader')
transform = transforms.Compose([ToTensor(), Normalize()])
reduction = model.level - model.idepth_level
root_val_dataset = OmniStereoDataset(args.root_dir,
args.val_list,
fov=args.fov)
val_dataset = FisheyeToIcoDataset(root_val_dataset,
root_val_dataset.ocams,
root_val_dataset.pose_dict,
level=model.level,
reduction=reduction,
transform=transform)
logger.info(f'{len(val_dataset)} samples were found.')
val_loader = DataLoader(val_dataset,
args.batch_size,
shuffle=False,
num_workers=args.workers)
model = nn.DataParallel(model)
# setup writer
log_folder = os.path.dirname(args.pretrained)
logger.info(f'=> save in checkpoint folder: {log_folder}')
base = os.path.splitext(os.path.basename(args.pretrained))[0]
root_dirname = args.root_dir.strip('/').split('/')[-1]
# setup logger file handler
handler = FileHandler(join(log_folder, f'eval_{root_dirname}_{base}.log'))
handler.setLevel(INFO)
logger.addHandler(handler)
logger.info('Start evaluation')
# ----------------------------
# evaluation
mode = 'eval'
depth_folder = join(
log_folder,
f'depth_{root_dirname}_{base}') if args.save_depth else None
ave_loss = run(epoch,
mode,
model,
val_loader,
converter,
device,
optimizer=None,
show_metrics=True,
depth_folder=depth_folder)
logger.info('Finish training.')
def eval(model, split, seq_length, bs, n_cpu, disp):
if use_no_element == False:
dataset = StsqDB(
data_file='data/no_ele/seq_length_{}/val_split_{}.pkl'.format(
int(seq_length), split),
vid_dir='data/videos_40/',
seq_length=int(seq_length),
transform=transforms.Compose([
ToTensor(),
Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
train=False)
else:
dataset = StsqDB(
data_file='data/seq_length_{}/train_split_{}.pkl'.format(
args.seq_length, args.split),
vid_dir='data/videos_40/',
seq_length=int(seq_length),
transform=transforms.Compose([
ToTensor(),
Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
train=True)
data_loader = DataLoader(dataset,
batch_size=int(bs),
shuffle=False,
num_workers=n_cpu,
drop_last=True)
correct = []
if use_no_element == False:
element_correct = [[] for i in range(12)]
element_sum = [[] for i in range(12)]
confusion_matrix = np.zeros([12, 12], int)
else:
element_correct = [[] for i in range(13)]
element_sum = [[] for i in range(13)]
confusion_matrix = np.zeros([13, 13], int)
for i, sample in enumerate(data_loader):
images, labels = sample['images'].to(device), sample['labels'].to(
device)
logits = model(images)
probs = F.softmax(logits.data, dim=1) ##確率
labels = labels.view(int(bs) * int(seq_length))
_, c, element_c, element_s, conf = correct_preds(
probs, labels.squeeze())
if disp:
print(i, c)
correct.append(c)
for j in range(len(element_c)):
element_correct[j].append(element_c[j])
for j in range(len(element_s)):
element_sum[j].append(element_s[j])
confusion_matrix = confusion_matrix + conf
PCE = np.mean(correct)
all_element_correct = np.sum(element_correct, axis=1)
all_element_sum = np.sum(element_sum, axis=1)
element_PCE = all_element_correct / all_element_sum
return PCE, element_PCE, all_element_correct, all_element_sum, confusion_matrix
def eval(model, split, seq_length, n_cpu, disp, steps=1):
print("------in function")
if not _video_interpolation:
dataset = GolfDB(data_file='data/val_split_{}.pkl'.format(split),
vid_dir='data/videos_160/',
seq_length=seq_length,
transform=transforms.Compose([
ToTensor(),
Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
]),
train=False)
else:
dataset = GolfDB(data_file='data/val_split_{}.pkl'.format(split),
vid_dir='data/videos_160/'.replace(
'videos_160',
'videos_downsampled_' + str(steps) + 'x'),
seq_length=seq_length,
transform=transforms.Compose([
ToTensor(),
Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
]),
train=False)
steps = 1
data_loader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=n_cpu,
drop_last=False)
idx_keep = np.arange(0, seq_length, steps)
idx_erase = np.delete(np.arange(0, seq_length), idx_keep)
correct = []
for i, sample in enumerate(data_loader):
images, labels = sample['images'], sample['labels']
if steps > 1:
#### Downsample video (temporally)
images[:,
idx_erase, :, :, :] = images[:,
np.repeat(idx_keep, steps -
1)[:len(idx_erase
)], :, :, :]
# full samples do not fit into GPU memory so evaluate sample in 'seq_length' batches
batch = 0
while batch * seq_length < images.shape[1]:
if (batch + 1) * seq_length > images.shape[1]:
image_batch = images[:, batch * seq_length:, :, :, :]
else:
image_batch = images[:, batch * seq_length:(batch + 1) *
seq_length, :, :, :]
logits = model(image_batch.cuda())
if batch == 0:
probs = F.softmax(logits.data, dim=1).cpu().numpy()
else:
probs = np.append(probs,
F.softmax(logits.data, dim=1).cpu().numpy(),
0)
batch += 1
if i == 176:
print('hello')
_, _, _, _, c = correct_preds(probs, labels.squeeze())
if disp:
print(i, c)
correct.append(c)
PCE = np.mean(correct)
return PCE
def main():
args = parser.parse_args()
print('Arguments:')
print(json.dumps(vars(args), indent=1))
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# device = torch.device('cpu')
if device.type != 'cpu':
cudnn.benchmark = True
print("device:", device)
print('=> setting data loader')
erp_shape = (60, 120)
transform = transforms.Compose(
[ToTensor(), Normalize((0.0645, ), (0.2116, ))])
trainset = UnfoldIcoDataset(datasets.MNIST(root='raw_data',
train=True,
download=True),
erp_shape,
args.level,
rotate=args.train_rot,
transform=transform)
testset = UnfoldIcoDataset(datasets.MNIST(root='raw_data',
train=False,
download=True),
erp_shape,
args.level,
rotate=args.test_rot,
transform=transform)
train_loader = DataLoader(trainset,
args.batch_size,
shuffle=True,
num_workers=args.workers)
test_loader = DataLoader(testset,
args.batch_size,
shuffle=False,
num_workers=args.workers)
print('=> setting model')
start_epoch = 0
model = HexRUNet_C(1)
total_params = 0
for param in model.parameters():
total_params += np.prod(param.shape)
print(f"Total model parameters: {total_params:,}.")
model = model.to(device)
# Loss function
print('=> setting loss function')
criterion = nn.CrossEntropyLoss()
# setup solver scheduler
print('=> setting optimizer')
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
print('=> setting scheduler')
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=100,
gamma=0.1)
if args.pretrained:
checkpoint = torch.load(args.pretrained)
print("=> using pre-trained weights")
model.load_state_dict(checkpoint['state_dict'])
start_epoch = checkpoint['epoch']
optimizer.load_state_dict(checkpoint['optimizer'])
scheduler.load_state_dict(checkpoint['scheduler'])
print("=> Resume training from epoch {}".format(start_epoch))
timestamp = datetime.now().strftime("%m%d-%H%M")
log_folder = join('checkpoints', f'{args.arch}_{timestamp}')
print(f'=> create log folder: {log_folder}')
os.makedirs(log_folder, exist_ok=True)
with open(join(log_folder, 'args.json'), 'w') as f:
json.dump(vars(args), f, indent=1)
writer = SummaryWriter(log_dir=log_folder)
writer.add_text('args', json.dumps(vars(args), indent=1))
# Training
for epoch in range(start_epoch, args.epochs):
# --------------------------
# training
# --------------------------
model.train()
losses = []
pbar = tqdm(train_loader)
total = 0
correct = 0
mode = 'train'
for idx, batch in enumerate(pbar):
# to cuda
for key in batch.keys():
batch[key] = batch[key].to(device)
outputs = model(batch)
labels = batch['label']
# Loss function
loss = criterion(outputs, labels)
losses.append(loss.item())
# update parameters
optimizer.zero_grad()
loss.backward()
optimizer.step()
# accuracy
_, predicted = outputs.max(1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
# update progress bar
display = OrderedDict(mode=f'{mode}',
epoch=f"{epoch:>2}",
loss=f"{losses[-1]:.4f}")
pbar.set_postfix(display)
# tensorboard log
if idx % args.log_interval == 0:
niter = epoch * len(train_loader) + idx
writer.add_scalar(f'{mode}/loss', loss.item(), niter)
# End of one epoch
scheduler.step()
ave_loss = sum(losses) / len(losses)
ave_acc = 100 * correct / total
writer.add_scalar(f'{mode}/loss_ave', ave_loss, epoch)
writer.add_scalar(f'{mode}/acc_ave', ave_acc, epoch)
print(
f"Epoch:{epoch}, Train Loss average:{ave_loss:.4f}, Accuracy average:{ave_acc:.2f}"
)
save_data = {
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
'scheduler': scheduler.state_dict(),
'ave_loss': ave_loss,
}
torch.save(save_data, join(log_folder, f'checkpoints_{epoch}.pth'))
# --------------------------
# evaluation
# --------------------------
model.eval()
losses = []
pbar = tqdm(test_loader)
total = 0
correct = 0
mode = 'test'
for idx, batch in enumerate(pbar):
with torch.no_grad():
# to cuda
for key in batch.keys():
batch[key] = batch[key].to(device)
outputs = model(batch)
labels = batch['label']
# Loss function
loss = criterion(outputs, labels)
losses.append(loss.item())
# accuracy
_, predicted = outputs.max(1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
# update progress bar
display = OrderedDict(mode=f'{mode}',
epoch=f"{epoch:>2}",
loss=f"{losses[-1]:.4f}")
pbar.set_postfix(display)
# tensorboard log
if idx % args.log_interval == 0:
niter = epoch * len(test_loader) + idx
writer.add_scalar(f'{mode}/loss', loss.item(), niter)
# End of one epoch
ave_loss = sum(losses) / len(losses)
ave_acc = 100 * correct / total
writer.add_scalar(f'{mode}/loss_ave', ave_loss, epoch)
writer.add_scalar(f'{mode}/acc_ave', ave_acc, epoch)
print(
f"Epoch:{epoch}, Test Loss average:{ave_loss:.4f}, Accuracy average:{ave_acc:.2f}"
)
writer.close()
print("Finish")