def test(model, device, test_loader, returnAllScores=False):
"""
Evaluate a deep neural network model
Args:
model: pytorch model object
device: cuda or cpu
test_dataloader: test image dataloader
returnAllScores: If true addtionally return all confidence scores and ground truth
Returns:
test loss and average precision. If returnAllScores = True, check Args
"""
#import cbowkai
#from cbowkai import modelll, make_context_vector, idx_to_word, word_to_idx, nomprebab
model.train(False)
running_loss = 0
running_ap = 0
criterion = torch.nn.BCEWithLogitsLoss(reduction='sum')
m = torch.nn.Sigmoid()
if returnAllScores == True:
all_scores = np.empty((0, 40), float)
ground_scores = np.empty((0, 40), float)
with torch.no_grad():
for data, target, senten in tqdm(test_loader):
# print(data.size(), target.size())
target = dealit(target)
data = data.to(device)
bs, ncrops, c, h, w = data.size()
output = model(data.view(-1, c, h, w))
output = output.view(bs, ncrops, -1).mean(1)
loss = criterion(output, target)
running_loss += loss # sum up batch loss
running_ap += get_ap_score(torch.Tensor.cpu(target).detach().numpy(),
torch.Tensor.cpu(m(output)).detach().numpy())
#print(m(output),target)
if returnAllScores == True:
all_scores = np.append(all_scores, torch.Tensor.cpu(m(output)).detach().numpy(), axis=0)
ground_scores = np.append(ground_scores, torch.Tensor.cpu(target).detach().numpy(), axis=0)
del data, target, output
torch.cuda.empty_cache()
num_samples = float(len(test_loader.dataset))
avg_test_loss = running_loss.item() / num_samples
test_map = running_ap / num_samples
print('test_loss: {:.4f}, test_avg_precision:{:.3f}'.format(
avg_test_loss, test_map))
return avg_test_loss,test_map
def train_model(model, device, optimizer, scheduler, train_loader,
valid_loader, save_dir, model_num, epochs, log_file):
"""
Train a deep neural network model
Args:
model: pytorch model object
device: cuda or cpu
optimizer: pytorch optimizer object
scheduler: learning rate scheduler object that wraps the optimizer
train_dataloader: training image dataloader
valid_dataloader: validation image dataloader
save_dir: Location to save model weights, plots and log_file
epochs: number of training epochs
log_file: text file instance to record training and validation history
Returns:
Training history and Validation history (loss and average precision)
"""
import cbowkai
from cbowkai import modelll, make_context_vector, idx_to_word, word_to_idx, nomprebab
tr_loss, tr_map = [], []
val_loss, val_map = [], []
best_val_map = 0.0
# Each epoch has a training and validation phase
for epoch in range(epochs):
print("-------Epoch {}----------".format(epoch + 1))
log_file.write("Epoch {} >>".format(epoch + 1))
for phase in ['train', 'valid']:
running_loss = 0.0
running_ap = 0.0
criterion = torch.nn.BCEWithLogitsLoss(reduction='sum')
m = torch.nn.Sigmoid()
if phase == 'train':
model.train(True) # Set model to training mode
for data, target, senten in tqdm(train_loader):
# print(data)
target = dealit(target)
data = data.to(device)
sentenlist = []
for tpp in senten:
tpi = tpp.split()
context = [tpi[0], tpi[1]]
context_vector = make_context_vector(
context, word_to_idx)
nll_prob = modelll(context_vector)
changeprob = nomprebab(nll_prob)
sentenlist.append(changeprob)
outputtt = torch.Tensor.cuda(torch.Tensor(sentenlist))
# zero the parameter gradients
optimizer.zero_grad()
output = model(data)
output = output * 0.9 + outputtt * 0.1
loss = criterion(output, target)
# Get metrics here
running_loss += loss
running_ap += get_ap_score(
torch.Tensor.cpu(target).detach().numpy(),
torch.Tensor.cpu(m(output)).detach().numpy())
# Backpropagate the system the determine the gradients
loss.backward()
# Update the paramteres of the model
optimizer.step()
# clear variables
del data, target, output
gc.collect()
torch.cuda.empty_cache()
# print("loss = ", running_loss)
num_samples = float(len(train_loader.dataset))
tr_loss_ = running_loss / num_samples
tr_map_ = running_ap / num_samples
print(running_loss, running_ap, num_samples)
print('train_loss: {:.4f}, train_avg_precision:{:.3f}'.format(
tr_loss_, tr_map_))
log_file.write(
'train_loss: {:.4f}, train_avg_precision:{:.3f}, '.format(
tr_loss_, tr_map_))
# Append the values to global arrays
tr_loss.append(tr_loss_), tr_map.append(tr_map_)
else:
model.train(False) # Set model to evaluate mode
# torch.no_grad is for memory savings
with torch.no_grad():
for data, target, senten in tqdm(valid_loader):
target = dealit(target)
data = data.to(device)
sentenlist = []
for tpp in senten:
tpi = tpp.split()
context = [tpi[0], tpi[1]]
context_vector = make_context_vector(
context, word_to_idx)
nll_prob = modelll(context_vector)
changeprob = nomprebab(nll_prob)
sentenlist.append(changeprob)
outputtt = torch.Tensor.cuda(torch.Tensor(sentenlist))
# zero the parameter gradients
output = model(data)
output = output * 0.9 + outputtt * 0.1
loss = criterion(output, target)
running_loss += loss # sum up batch loss
running_ap += get_ap_score(
torch.Tensor.cpu(target).detach().numpy(),
torch.Tensor.cpu(m(output)).detach().numpy())
del data, target, output
gc.collect()
torch.cuda.empty_cache()
num_samples = float(len(valid_loader.dataset))
val_loss_ = running_loss.item() / num_samples
val_map_ = running_ap / num_samples
# Append the values to global arrays
val_loss.append(val_loss_), val_map.append(val_map_)
print('val_loss: {:.4f}, val_avg_precision:{:.3f}'.format(
val_loss_, val_map_))
log_file.write(
'val_loss: {:.4f}, val_avg_precision:{:.3f}\n'.format(
val_loss_, val_map_))
# Save model using val_acc
if val_map_ >= best_val_map:
best_val_map = val_map_
log_file.write("saving best weights...\n")
torch.save(
model.state_dict(),
os.path.join(save_dir,
"model-{}.pth".format(model_num)))
scheduler.step()
return ([tr_loss, tr_map], [val_loss, val_map])
def test(model, device, test_loader, returnAllScores=False):
"""
Evaluate a deep neural network model
Args:
model: pytorch model object
device: cuda or cpu
test_dataloader: test image dataloader
returnAllScores: If true addtionally return all confidence scores and ground truth
Returns:
test loss and average precision. If returnAllScores = True, check Args
"""
import cbowkai
from cbowkai import modelll, make_context_vector, idx_to_word, word_to_idx, nomprebab
model.train(False)
running_loss = 0
running_ap = 0
criterion = torch.nn.BCEWithLogitsLoss(reduction='sum')
m = torch.nn.Sigmoid()
if returnAllScores == True:
all_scores = np.empty((0, 40), float)
ground_scores = np.empty((0, 40), float)
with torch.no_grad():
for data, target, senten in tqdm(test_loader):
# print(data.size(), target.size())
target = dealit(target)
data = data.to(device)
bs, ncrops, c, h, w = data.size()
sentenlist = []
for tpp in senten:
tpi = tpp.split()
context = [tpi[0], tpi[1]]
context_vector = make_context_vector(context, word_to_idx)
nll_prob = modelll(context_vector)
changeprob = nomprebab(nll_prob)
sentenlist.append(changeprob)
outputtt = torch.Tensor.cuda(torch.Tensor(sentenlist))
output = model(data.view(-1, c, h, w))
output = output.view(bs, ncrops, -1).mean(1)
output = output * 0.9 + outputtt * 0.1
loss = criterion(output, target)
running_loss += loss # sum up batch loss
running_ap += get_ap_score(
torch.Tensor.cpu(target).detach().numpy(),
torch.Tensor.cpu(m(output)).detach().numpy())
#print(m(output),target)
if returnAllScores == True:
all_scores = np.append(all_scores,
torch.Tensor.cpu(
m(output)).detach().numpy(),
axis=0)
ground_scores = np.append(
ground_scores,
torch.Tensor.cpu(target).detach().numpy(),
axis=0)
del data, target, output
gc.collect()
torch.cuda.empty_cache()
num_samples = float(len(test_loader.dataset))
avg_test_loss = running_loss.item() / num_samples
test_map = running_ap / num_samples
print('test_loss: {:.4f}, test_avg_precision:{:.3f}'.format(
avg_test_loss, test_map))
f = open('test_result.txt', 'w')
f.write('test loss:' + str(avg_test_loss) + ' ' + 'test_ap:' +
str(test_map))
f.close()
if returnAllScores == False:
return avg_test_loss, running_ap
return avg_test_loss, running_ap, all_scores, ground_scores
def train(model, device, train_loader, optimizer, epoch, lamdaValue, opt):
# set model as training mode
targetNet, sourceImagenet, sourcePlaces = model
targetNet.train()
if opt.isSource:
if opt.multi_source:
sourceImagenet = sourceImagenet.train()
sourcePlaces = sourcePlaces.train()
elif (not opt.multi_source) and opt.sourceKind == 'imagenet':
sourceImagenet = sourceImagenet.train()
elif (not opt.multi_source) and opt.sourceKind == 'places365':
sourcePlaces = sourcePlaces.train()
losses = AverageMeter()
Targetscores = AverageMeter()
targetloss = AverageMeter()
auxiliaryimagenet = AverageMeter()
auxiliaryplaces = AverageMeter()
softmax = nn.Softmax()
sigmoid = torch.nn.Sigmoid()
upsample = nn.Upsample(scale_factor=7/3, mode='bilinear')
N_count = 0
for batch_idx, (images, y) in enumerate(train_loader):
if opt.dataset == 'voc':
y = y.float()
if opt.dataset == 'places365':
y = y.view(-1, )
images, y = images.to(device), y.to(device)
N_count+= images.size(0)
optimizer.zero_grad()
if opt.isSource:
targetOutput, auxiliaryOutput1, auxiliaryOutput2 = targetNet(images)
else:
targetOutput = targetNet(images)
upsample_dataset = ['cifar10', 'cifar100', 'stl10']
if opt.isSource:
with torch.no_grad():
if opt.dataset in upsample_dataset:
images = images.detach()
images = upsample(images)
prob1 = None
prob2 = None
if opt.multi_source:
prob1 = sourceImagenet(images)
prob2 = sourcePlaces(images)
elif (not opt.multi_source) and opt.sourceKind == 'imagenet':
prob1 = sourceImagenet(images)
elif (not opt.multi_source) and opt.sourceKind == 'places365':
prob2 = sourcePlaces(images)
targetloss, auxiliary_imagenet, auxiliary_places, loss = loss_fn(targetOutput,auxiliaryOutput1, auxiliaryOutput2, prob1, prob2, y, opt)
else:
singleLabel = False if opt.dataset == 'voc' else True
loss = model_fit(targetOutput, y, opt.classifier_loss_method, singleLabel=singleLabel, T=opt.T)
losses.update(loss.item(), images.size()[0])
y_pred = torch.max(targetOutput, 1)[1]
if opt.dataset == 'voc':
step_score = get_ap_score(torch.Tensor.cpu(y).detach().numpy(), torch.Tensor.cpu(sigmoid(targetOutput)).detach().numpy())
else:
step_score = accuracy(targetOutput.data, y.data, topk=(1,))[0]
Targetscores.update(step_score,images.size()[0])
loss.backward()
optimizer.step()
if (batch_idx) % 10 == 0:
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}, Accu: {:.2f}%'.format(
epoch, N_count, len(train_loader.dataset), 100. * (batch_idx + 1) / len(train_loader), losses.avg, Targetscores.avg))
return losses, Targetscores
def validation(model, device, optimizer, test_loader, lamdaValue, opt):
targetNet, sourceImagenet, sourcePlaces = model
targetNet.eval()
if opt.isSource:
if opt.multi_source:
sourceImagenet = sourceImagenet.eval()
sourcePlaces = sourcePlaces.eval()
elif (not opt.multi_source) and opt.sourceKind == 'imagenet':
sourceImagenet = sourceImagenet.eval()
elif (not opt.multi_source) and opt.sourceKind == 'places365':
sourcePlaces = sourcePlaces.eval()
accs = AverageMeter()
losses = AverageMeter()
softmax = nn.Softmax()
sigmoid = torch.nn.Sigmoid()
upsample = nn.Upsample(scale_factor=7/3, mode='bilinear')
with torch.no_grad():
for images, y in test_loader:
# distribute data to device
images, y = images.to(device), y.to(device)
if opt.dataset == 'places365':
y = y.view(-1, )
if opt.dataset == 'voc':
y = y.float()
if opt.isSource:
targetOutput, auxiliaryOutput1, auxiliaryOutput2 = targetNet(images)
else:
targetOutput = targetNet(images)
if opt.isSource:
images = upsample(images)
prob1 = None
prob2 = None
if opt.multi_source:
prob1 = sourceImagenet(images)
prob2 = sourcePlaces(images)
elif (not opt.multi_source) and opt.sourceKind == 'imagenet':
prob1 = sourceImagenet(images)
elif (not opt.multi_source) and opt.sourceKind == 'places365':
prob2 = sourcePlaces(images)
targetloss, auxiliary_imagenet, auxiliary_places, loss = loss_fn(targetOutput,auxiliaryOutput1, auxiliaryOutput2, prob1, prob2, y, opt)
else:
singleLabel = False if opt.dataset == 'voc' else True
loss = model_fit(targetOutput, y, opt.classifier_loss_method, singleLabel=singleLabel, T=opt.T)
losses.update(loss.item(), images.size()[0])
if opt.dataset == 'voc':
prec = get_ap_score(torch.Tensor.cpu(y).detach().numpy(), torch.Tensor.cpu(sigmoid(targetOutput)).detach().numpy())
else:
prec = accuracy(targetOutput.data, y.data, topk=(1,))[0]
accs.update(prec.item(), images.size()[0])
print('\nTest set ({:d} samples): Average loss: {:.4f}, Accuracy: {:.2f}%\n'.format(len(test_loader.dataset), losses.avg, accs.avg))
return losses, accs