def validate(val_list, model, criterion):
print('begin test')
test_loader = torch.utils.data.DataLoader(dataset.ListDataset(
val_list,
shuffle=False,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
]),
train=False),
batch_size=args.batch_size)
model.eval()
mae = 0
for i, (img, target) in enumerate(test_loader):
img = img.cuda()
img = Variable(img)
output = model(img)
mae += abs(output.data.sum() -
target.sum().type(torch.FloatTensor).cuda())
mae = mae / len(test_loader)
print(' * MAE {mae:.3f} '.format(mae=mae))
return mae
def train(train_list, model, criterion, optimizer, epoch):
losses = AverageMeter()
batch_time = AverageMeter()
data_time = AverageMeter()
train_loader = torch.utils.data.DataLoader(dataset.ListDataset(
train_list,
shuffle=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
]),
train=True,
seen=model.seen,
batch_size=args.batch_size,
num_workers=args.workers),
batch_size=args.batch_size)
print('epoch %d, processed %d samples, lr %.10f' %
(epoch, epoch * len(train_loader.dataset), args.lr))
model.train()
end = time.time()
for i, (img, target) in enumerate(train_loader):
data_time.update(time.time() - end)
img = img.cuda()
img = Variable(img)
output = model(img)
target = target.type(torch.FloatTensor).unsqueeze(0).cuda()
target = Variable(target)
loss = criterion(output, target)
losses.update(loss.item(), img.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses))
def evaluate(model, opt, iou_thres, conf_thres, nms_thres, img_size,
batch_size):
model.eval()
# Get dataloader
# dataset = ListDataset(path, img_size=img_size, augment=False, multiscale=False)
# dataloader = torch.utils.data.DataLoader(
# dataset, batch_size=batch_size, shuffle=False, num_workers=1, collate_fn=dataset.collate_fn
# )
testDataSet = dataset.ListDataset(opt)
dataloader = DataLoader(testDataSet,
batch_size=opt.batchSize,
shuffle=False,
num_workers=0,
pin_memory=True,
collate_fn=testDataSet.collate_fn)
Tensor = torch.cuda.FloatTensor if torch.cuda.is_available(
) else torch.FloatTensor
labels = []
sample_metrics = [] # List of tuples (TP, confs, pred)
for batch_i, (imgs, targets) in enumerate(
tqdm.tqdm(dataloader, desc="Detecting objects")):
for batch_i_in, (imgs_in_batch,
imgs_in_target) in enumerate(zip(imgs, targets)):
# Extract labels
targets = imgs_in_target
imgs = imgs_in_batch
# print('labels : ', targets)
labels += targets[:, 1].tolist()
# Rescale target
targets[:, 2:] = xywh2xyxy(targets[:, 2:])
targets[:, 2:] *= img_size
# print('targets : ', targets)
imgs = Variable(imgs.type(Tensor),
requires_grad=False).unsqueeze(0)
# print('imgs shape : ', imgs.shape)
with torch.no_grad():
# print('Output')
outputs, _ = model(imgs)
# print('outputs : ', outputs[0].shape)
# print('out shape : ', outputs, type(outputs))
outputs = torch.cat(outputs, dim=1).cpu()
outputs = utils.non_max_suppression(outputs,
conf_thres=conf_thres,
nms_thres=nms_thres)
# print('outputs : ', outputs)
sample_metrics += get_batch_statistics(outputs,
targets,
iou_threshold=iou_thres)
# Concatenate sample statistics
true_positives, pred_scores, pred_labels = [
np.concatenate(x, 0) for x in list(zip(*sample_metrics))
]
precision, recall, AP, f1, ap_class = ap_per_class(true_positives,
pred_scores,
pred_labels, labels)
print('precision, recall, AP, f1, ap_class : ', precision, recall, AP, f1,
ap_class)
return precision, recall, AP, f1, ap_class
np.concatenate(x, 0) for x in list(zip(*sample_metrics))
]
precision, recall, AP, f1, ap_class = ap_per_class(true_positives,
pred_scores,
pred_labels, labels)
print('precision, recall, AP, f1, ap_class : ', precision, recall, AP, f1,
ap_class)
return precision, recall, AP, f1, ap_class
if __name__ == "__main__":
opt = parse_args()
print(opt)
trainDataSet = dataset.ListDataset(opt)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
class_names = load_classes(opt.names)
# net = model.objDetNet(opt)
# net.to(device)
# net.loadPretrainedParams()
# Initiate model
model = model.objDetNet(opt).to(device)
model.loadPretrainedParams()
# if opt.weights_path.endswith(".weights"):
# # Load darknet weights
# model.load_pretrained_params(opt.weights_path)
# else:
# # Load checkpoint weights
def train(epoch):
global processed_batches
t0 = time.time()
# if ngpus > 1:
# cur_model = model.module
# else:
# cur_model = model
cur_model = model
train_loader = torch.utils.data.DataLoader(dataset.ListDataset(
trainlist,
shape=(init_width, init_height),
shuffle=True,
transform=transforms.Compose([
transforms.ToTensor(),
]),
train=True,
seen=cur_model.seen,
batch_size=batch_size,
num_workers=num_workers),
batch_size=batch_size,
shuffle=False,
**kwargs)
lr = adjust_learning_rate(optimizer, processed_batches)
logging('epoch %d, processed %d samples, lr %f' %
(epoch, epoch * len(train_loader.dataset), lr))
model.train()
t1 = time.time()
avg_time = torch.zeros(9)
for batch_idx, (data, target) in enumerate(train_loader):
t2 = time.time()
adjust_learning_rate(optimizer, processed_batches)
processed_batches = processed_batches + 1
#if (batch_idx+1) % dot_interval == 0:
# sys.stdout.write('.')
if use_cuda:
data = data.cuda()
#target= target.cuda()
t3 = time.time()
data, target = Variable(data), Variable(target)
t4 = time.time()
optimizer.zero_grad()
t5 = time.time()
output = model(data)
t6 = time.time()
region_loss.seen = region_loss.seen + data.data.size(0)
loss = region_loss(output, target)
t7 = time.time()
loss.backward()
t8 = time.time()
optimizer.step()
t9 = time.time()
if False and batch_idx > 1:
avg_time[0] = avg_time[0] + (t2 - t1)
avg_time[1] = avg_time[1] + (t3 - t2)
avg_time[2] = avg_time[2] + (t4 - t3)
avg_time[3] = avg_time[3] + (t5 - t4)
avg_time[4] = avg_time[4] + (t6 - t5)
avg_time[5] = avg_time[5] + (t7 - t6)
avg_time[6] = avg_time[6] + (t8 - t7)
avg_time[7] = avg_time[7] + (t9 - t8)
avg_time[8] = avg_time[8] + (t9 - t1)
print('-------------------------------')
print(' load data : %f' % (avg_time[0] / (batch_idx)))
print(' cpu to cuda : %f' % (avg_time[1] / (batch_idx)))
print('cuda to variable : %f' % (avg_time[2] / (batch_idx)))
print(' zero_grad : %f' % (avg_time[3] / (batch_idx)))
print(' forward feature : %f' % (avg_time[4] / (batch_idx)))
print(' forward loss : %f' % (avg_time[5] / (batch_idx)))
print(' backward : %f' % (avg_time[6] / (batch_idx)))
print(' step : %f' % (avg_time[7] / (batch_idx)))
print(' total : %f' % (avg_time[8] / (batch_idx)))
t1 = time.time()
print('')
t1 = time.time()
logging('training with %f samples/s' % (len(train_loader.dataset) /
(t1 - t0)))
if (epoch + 1) % save_interval == 0:
logging('save weights to %s/%06d.weights' % (backupdir, epoch + 1))
cur_model.seen = (epoch + 1) * len(train_loader.dataset)
cur_model.save_weights('%s/%06d.weights' % (backupdir, epoch + 1))
model.load_weights(weightfile)
model.print_network()
region_loss.seen = model.seen
processed_batches = model.seen / batch_size
init_width = model.width
init_height = model.height
init_epoch = model.seen / nsamples
kwargs = {'num_workers': num_workers, 'pin_memory': True} if use_cuda else {}
test_loader = torch.utils.data.DataLoader(dataset.ListDataset(
testlist,
shape=(init_width, init_height),
shuffle=False,
transform=transforms.Compose([
transforms.ToTensor(),
]),
train=False),
batch_size=batch_size,
shuffle=False,
**kwargs)
if use_cuda:
if ngpus > 1:
model = torch.nn.DataParallel(model).cuda()
else:
model = model.cuda()
params_dict = dict(model.named_parameters())
params = []
from create_train_dataset import create_training_image_list, DATA_PATH
"""
This help me understand data loader
"""
if __name__ == "__main__":
train_list = create_training_image_list(DATA_PATH)
train_loader = torch.utils.data.DataLoader(
dataset.ListDataset(train_list,
shuffle=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
]),
train=True,
batch_size=1,
num_workers=1),
batch_size=1
) # if you have different image size, then batch_size must be 1
print(type(train_loader))
for batch_ndx, sample in enumerate(train_loader):
img = sample[0]
density = sample[1]
print("img shape ", img.shape)
print("density shape", density.shape)
