def run(init_lr=0.01, max_steps=200, mode='rgb', root='/media/pritesh/Entertainment/Visual-Tactile_Dataset/dataset/',\
train_split='train.txt', test_split='test.txt', batch_size=1, save_model=''):
print(train_split, test_split)
writer = tensorboardX.SummaryWriter()
# setup dataset
test_transforms = transforms.Compose([videotransforms.CenterCrop(224)])
dataset = Dataset(train_split, root, mode, test_transforms)
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
shuffle=True,
num_workers=0,
pin_memory=True)
val_dataset = Dataset(test_split, root, mode, test_transforms)
val_dataloader = torch.utils.data.DataLoader(val_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=0,
pin_memory=True)
dataloaders = {'train': dataloader, 'val': val_dataloader}
datasets = {'train': dataset, 'val': val_dataset}
# setup the model
sm = InceptionI3d(400, in_channels=3)
sm.replace_logits(1)
#add your network here
fusedNet = FusionNet(sm)
if torch.cuda.is_available():
fusedNet.cuda()
fusedNet = nn.DataParallel(fusedNet)
lr = init_lr
optimizer = optim.SGD(fusedNet.parameters(),
lr=lr,
momentum=0.9,
weight_decay=0.0000001)
lr_sched = optim.lr_scheduler.MultiStepLR(optimizer, [50, 100, 150, 200])
if torch.cuda.is_available():
data = torch.load(save_model)
else:
data = torch.load(save_model,
map_location=lambda storage, loc: storage)
fusedNet.load_state_dict(data['model_state'])
optimizer.load_state_dict(data['optimizer_state'])
lr_sched.load_state_dict(data['scheduler_state'])
steps = 0
with open('inference_V.txt', 'w') as file:
file.write("train and validation loss file\n")
# train it
# Each epoch has a training and validation phase
fusedNet.train(False) # Set model to evaluate mode
for phase in ['train', 'val']:
print('phase : {}'.format(phase))
tot_cls_loss = 0.0
num_iter = 0
count = 0
# optimizer.zero_grad()
with open('inference_V.txt', 'a') as file:
file.write("---------------\n")
# Iterate over data.
for data in dataloaders[phase]:
num_iter += 1
# get the inputs
f_vid, l_vid, tactile, pos, labels = data
if torch.cuda.is_available():
rgb_inputs = Variable(f_vid.cuda())
t = rgb_inputs.size(2)
labels = Variable(labels.cuda())
else:
rgb_inputs = Variable(f_vid)
t = rgb_inputs.size(2)
labels = Variable(labels)
out = fusedNet(rgb_inputs.float())
#print('prediction output = ', per_frame_logits.shape)
#print('labels = ',labels.shape)
# compute classification loss (with max-pooling along time B x C x T)
out = out.squeeze(1)
cls_loss = F.binary_cross_entropy_with_logits(
out.double(), labels.double())
tot_cls_loss += cls_loss.item()
# cls_loss.backward()
print('{} Loss: {:.4f} and lr: {}'.format(phase,
tot_cls_loss / num_iter,
init_lr))
with open('inference_V.txt', 'a') as file:
file.write("%f\n" % (tot_cls_loss / num_iter))
# optimizer.step()
# optimizer.zero_grad()
if phase == 'val':
writer.add_scalar('inference_error/' + phase,
(tot_cls_loss / num_iter), num_iter)
else:
writer.add_scalar('inference_error/' + phase,
(tot_cls_loss / num_iter), num_iter)
def run(init_lr=0.001, max_steps=30, mode='rgb', root='/media/pritesh/Entertainment/Visual-Tactile_Dataset/dataset/',\
train_split='trainv2.txt', test_split='testv2.txt', batch_size=100, save_model='', save_filename=''):
writer = tensorboardX.SummaryWriter()
# setup dataset
dataset = Dataset(train_split, root, mode)
dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True, num_workers=3, pin_memory=True)
val_dataset = Dataset(test_split, root, mode)
val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, shuffle=True, num_workers=3, pin_memory=True)
dataloaders = {'train': dataloader, 'val': val_dataloader}
datasets = {'train': dataset, 'val': val_dataset}
# setup the model
#add your network here
# fusedNet = FusionNet(sm,tm)
fuseNet1 = tactileNet()
fuseNet2 = tactileNet()
fuseNet3 = tactileNet()
fuseNet4 = tactileNet()
masterNet = tactileFusionNet(fuseNet1,fuseNet2,fuseNet3,fuseNet4)
if torch.cuda.is_available():
fuseNet1.cuda()
fuseNet2.cuda()
fuseNet3.cuda()
fuseNet4.cuda()
masterNet.cuda()
fusedNet1 = nn.DataParallel(fuseNet1)
fusedNet2 = nn.DataParallel(fuseNet2)
fusedNet3 = nn.DataParallel(fuseNet3)
fusedNet4 = nn.DataParallel(fuseNet4)
masterNet = nn.DataParallel(masterNet)
lr = init_lr
optimizer = optim.SGD(masterNet.parameters(), lr=lr, momentum=0.9, weight_decay=0.0000001)
#optimizer = optim.Adam(masterNet.parameters(), lr=lr, weight_decay=0.0000001)
lr_sched = optim.lr_scheduler.MultiStepLR(optimizer, [10,20,25])
steps = 0
with open(save_filename, 'w') as file:
file.write("train and validation loss file\n")
# train it
while steps < max_steps:#for epoch in range(num_epochs):
print ('Step {}/{}'.format(steps, max_steps))
print ('-' * 10)
# Each epoch has a training and validation phase
for phase in ['train', 'val']:
print('phase : {}'.format(phase))
if phase == 'train':
masterNet.train(True)
else:
masterNet.train(False) # Set model to evaluate mode
tot_cls_loss = 0.0
num_iter = 0
count = 0
optimizer.zero_grad()
# Iterate over data.
for data in dataloaders[phase]:
num_iter += 1
# get the inputs
tactile, labels = data
if torch.cuda.is_available():
t1 = Variable(tactile[0].cuda())
t2 = Variable(tactile[1].cuda())
t3 = Variable(tactile[2].cuda())
t4 = Variable(tactile[3].cuda())
labels = Variable(labels.cuda())
else:
t1 = Variable(tactile[0])
t2 = Variable(tactile[1])
t3 = Variable(tactile[2])
t4 = Variable(tactile[3])
labels = Variable(labels)
out = masterNet(t1.float(),t2.float(),t2.float(),t2.float())
#print('prediction output = ', per_frame_logits.shape)
#print('labels = ',labels.shape)
# compute classification loss (with max-pooling along time B x C x T)
out = out.squeeze(1)
cls_loss = F.binary_cross_entropy_with_logits(out.double(), labels.double())
tot_cls_loss += cls_loss.item()
cls_loss.backward()
print('{} Loss: {:.4f} and lr: {}'.format(phase,tot_cls_loss/num_iter,init_lr))
with open(save_filename, 'a') as file:
file.write('epoch: ({}) {}: {} Loss: {:.7f} \n'.format(steps+1, num_iter,phase,tot_cls_loss/num_iter))
optimizer.step()
optimizer.zero_grad()
if phase == 'val':
writer.add_scalar('error/'+ phase , (tot_cls_loss/num_iter), num_iter)
else:
writer.add_scalar('error/'+ phase, (tot_cls_loss/num_iter), num_iter)
if((steps+1)%30 == 0):
save_checkpoint(masterNet, optimizer, lr_sched, steps,save_model)
#save error at every epoch
writer.add_scalar('errorAtEpoch/'+phase, (tot_cls_loss/num_iter), steps)
tot_cls_loss = 0.
with open(save_filename, 'a') as file:
file.write("-"*50)
file.write("\n")
#if(steps%50 == 0):
# torch.save(fusedNet.module.state_dict(), save_model+phase+str(steps).zfill(6)+'.pt')
# save_checkpoint(fusedNet, optimizer, lr_sched, steps)
steps+=1
lr_sched.step()
def run(init_lr=0.01, max_steps=200, mode='rgb', root='/media/pritesh/Entertainment/Visual-Tactile_Dataset/dataset/',\
train_split='train.txt', test_split='test.txt', batch_size=5, save_model=''):
writer = tensorboardX.SummaryWriter()
# setup dataset
train_transforms = transforms.Compose([
videotransforms.RandomCrop(224),
videotransforms.RandomHorizontalFlip(),
])
test_transforms = transforms.Compose([videotransforms.CenterCrop(224)])
dataset = Dataset(train_split, root, mode, train_transforms)
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
shuffle=True,
num_workers=3,
pin_memory=True)
val_dataset = Dataset(test_split, root, mode, test_transforms)
val_dataloader = torch.utils.data.DataLoader(val_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=3,
pin_memory=True)
dataloaders = {'train': dataloader, 'val': val_dataloader}
datasets = {'train': dataset, 'val': val_dataset}
# setup the model
sm = InceptionI3d(400, in_channels=3)
sm.load_state_dict(torch.load('models/rgb_imagenet.pt'))
#tm = InceptionI3d(400, in_channels=2)
#tm.load_state_dict(torch.load('models/flow_imagenet.pt'))
sm.replace_logits(1)
sm = freeze_network_layer(sm)
#add your network here
fusedNet = FusionNet(sm)
if torch.cuda.is_available():
fusedNet.cuda()
fusedNet = nn.DataParallel(fusedNet)
lr = init_lr
optimizer = optim.SGD(fusedNet.parameters(),
lr=lr,
momentum=0.9,
weight_decay=0.0000001)
lr_sched = optim.lr_scheduler.MultiStepLR(optimizer, [50, 100, 150, 200])
steps = 0
with open('i3d_video.txt', 'w') as file:
file.write("train and validation loss file\n")
# train it
while steps < max_steps: #for epoch in range(num_epochs):
print('Step {}/{}'.format(steps, max_steps))
print('-' * 10)
# Each epoch has a training and validation phase
for phase in ['train', 'val']:
print('phase : {}'.format(phase))
if phase == 'train':
fusedNet.train(True)
else:
fusedNet.train(False) # Set model to evaluate mode
tot_loss = 0.0
tot_loc_loss = 0.0
tot_cls_loss = 0.0
num_iter = 0
count = 0
optimizer.zero_grad()
# Iterate over data.
for data in dataloaders[phase]:
num_iter += 1
# get the inputs
f_vid, l_vid, tactile, pos, labels = data
if torch.cuda.is_available():
inputs = Variable(f_vid.cuda())
t = inputs.size(2)
labels = Variable(labels.cuda())
else:
inputs = Variable(f_vid)
t = inputs.size(2)
labels = Variable(labels)
per_frame_logits = fusedNet(inputs.float())
#print('prediction output = ', per_frame_logits.shape)
#print('labels = ',labels.shape)
# compute classification loss (with max-pooling along time B x C x T)
per_frame_logits = per_frame_logits.squeeze(1)
cls_loss = F.binary_cross_entropy_with_logits(
per_frame_logits.double(), labels.double())
tot_cls_loss += cls_loss.item()
cls_loss.backward()
print('{} Loss: {:.4f} and lr: {}'.format(
phase, tot_cls_loss / num_iter, init_lr))
with open('i3d_video.txt', 'a') as file:
file.write("%f\n" % (tot_cls_loss / num_iter))
optimizer.step()
optimizer.zero_grad()
if phase == 'val':
writer.add_scalar('error/' + phase,
(tot_cls_loss / num_iter), num_iter)
else:
writer.add_scalar('error/' + phase,
(tot_cls_loss / num_iter), num_iter)
if (steps % 50 == 0):
torch.save(
fusedNet.module.state_dict(),
save_model + phase + str(steps).zfill(6) + '.pt')
save_checkpoint(fusedNet, optimizer, lr_sched, steps)
#save error at every epoch
writer.add_scalar('errorAtEpoch/' + phase,
(tot_cls_loss / num_iter), steps)
tot_cls_loss = 0.
#if(steps%50 == 0):
# torch.save(fusedNet.module.state_dict(), save_model+phase+str(steps).zfill(6)+'.pt')
# save_checkpoint(fusedNet, optimizer, lr_sched, steps)
steps += 1
lr_sched.step()
def run(init_lr=0.001, max_steps=10, mode='rgb', root='/media/pritesh/Entertainment/Visual-Tactile_Dataset/dataset/',\
train_split='train.txt', test_split='test.txt', batch_size=5, save_model=''):
# setup dataset
train_transforms = transforms.Compose([
videotransforms.RandomCrop(224),
videotransforms.RandomHorizontalFlip(),
])
test_transforms = transforms.Compose([videotransforms.CenterCrop(224)])
dataset = Dataset(train_split, root, mode, train_transforms)
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
shuffle=True,
num_workers=3,
pin_memory=True)
val_dataset = Dataset(test_split, root, mode, test_transforms)
val_dataloader = torch.utils.data.DataLoader(val_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=3,
pin_memory=True)
dataloaders = {'train': dataloader, 'val': val_dataloader}
datasets = {'train': dataset, 'val': val_dataset}
# setup the model
if mode == 'flow':
i3d = InceptionI3d(400, in_channels=2)
i3d.load_state_dict(torch.load('models/flow_imagenet.pt'))
else:
i3d = InceptionI3d(400, in_channels=3)
i3d.load_state_dict(torch.load('models/rgb_imagenet.pt'))
i3d.replace_logits(2)
# #i3d.load_state_dict(torch.load('/ssd/models/000920.pt'))
i3d.cuda()
i3d = nn.DataParallel(i3d)
lr = init_lr
optimizer = optim.SGD(i3d.parameters(),
lr=lr,
momentum=0.9,
weight_decay=0.0000001)
# lr_sched = optim.lr_scheduler.MultiStepLR(optimizer, [300, 1000])
lr_sched = optim.lr_scheduler.MultiStepLR(optimizer, [4, 7])
num_steps_per_update = 4 # accum gradient
steps = 0
# train it
while steps < max_steps: #for epoch in range(num_epochs):
print('Step {}/{}'.format(steps, max_steps))
print('-' * 10)
# Each epoch has a training and validation phase
for phase in ['train', 'val']:
if phase == 'train':
i3d.train(True)
else:
i3d.train(False) # Set model to evaluate mode
tot_loss = 0.0
tot_loc_loss = 0.0
tot_cls_loss = 0.0
num_iter = 0
optimizer.zero_grad()
# Iterate over data.
for data in dataloaders[phase]:
num_iter += 1
# get the inputs
f_vid, l_vid, tactile, pos, labels = data
# wrap them in Variable
# inputs = Variable(f_vid)
# t = inputs.size(2)
# labels = Variable(labels)
inputs = Variable(inputs.cuda())
t = inputs.size(2)
labels = Variable(labels.cuda())
print("go to i3d")
per_frame_logits = i3d(inputs.float())
# upsample to input size
# per_frame_logits = F.upsample(per_frame_logits, t, mode='linear')
per_frame_logits = F.interpolate(per_frame_logits,
t,
mode='linear',
align_corners=False)
# compute localization loss
loc_loss = F.binary_cross_entropy_with_logits(
per_frame_logits, labels)
tot_loc_loss += loc_loss.data[0]
# compute classification loss (with max-pooling along time B x C x T)
cls_loss = F.binary_cross_entropy_with_logits(
torch.max(per_frame_logits, dim=2)[0],
torch.max(labels, dim=2)[0])
tot_cls_loss += cls_loss.data[0]
loss = (0.5 * loc_loss + 0.5 * cls_loss) / num_steps_per_update
tot_loss += loss.data[0]
loss.backward()
if num_iter == num_steps_per_update and phase == 'train':
steps += 1
num_iter = 0
optimizer.step()
optimizer.zero_grad()
lr_sched.step()
if steps % 10 == 0:
print(
'{} Loc Loss: {:.4f} Cls Loss: {:.4f} Tot Loss: {:.4f}'
.format(phase,
tot_loc_loss / (10 * num_steps_per_update),
tot_cls_loss / (10 * num_steps_per_update),
tot_loss / 10))
# save model
torch.save(i3d.module.state_dict(),
save_model + str(steps).zfill(6) + '.pt')
tot_loss = tot_loc_loss = tot_cls_loss = 0.
if phase == 'val':
print('{} Loc Loss: {:.4f} Cls Loss: {:.4f} Tot Loss: {:.4f}'.
format(phase, tot_loc_loss / num_iter,
tot_cls_loss / num_iter,
(tot_loss * num_steps_per_update) / num_iter))