def extract_med(data_loader, model, path_file, mode):
path_hdf5 = path_file + '.hdf5'
path_txt = path_file + '.txt'
hdf5_file = h5py.File(path_hdf5, 'w')
# estimate output shapes
_, output = model(Variable(torch.ones(1, 3, args.size, args.size)))
nb_images = len(data_loader.dataset)
if mode == 'both' or mode == 'att':
shape_att = (nb_images, output.size(1), output.size(2), output.size(3))
print('Warning: shape_att={}'.format(shape_att))
hdf5_att = hdf5_file.create_dataset('att', shape_att,
dtype='f') # , compression='gzip')
model.eval()
batch_time = AvgMeter()
data_time = AvgMeter()
begin = time.time()
end = time.time()
idx = 0
for i, input in enumerate(data_loader):
input_var = Variable(input['visual'], volatile=True)
_, output_att = model(input_var)
nb_regions = output_att.size(2) * output_att.size(3)
batch_size = output_att.size(0)
if mode == 'both' or mode == 'att':
hdf5_att[idx:idx + batch_size] = output_att.data.cpu().numpy()
idx += batch_size
torch.cuda.synchronize()
batch_time.update(time.time() - end)
end = time.time()
if i % 1 == 0:
print('Extract: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'.format(
i,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
))
hdf5_file.close()
# Saving image names in the same order than extraction
with open(path_txt, 'w') as handle:
for name in data_loader.dataset.dataset.imgs:
handle.write(name + '\n')
end = time.time() - begin
print('Finished in {}m and {}s'.format(int(end / 60), int(end % 60)))
def extract(data_loader, model, path_file, mode):
path_hdf5 = path_file + '.hdf5'
path_txt = path_file + '.txt'
hdf5_file = h5py.File(path_hdf5, 'w')
nb_images = len(data_loader.dataset)
if mode == 'both' or mode == 'att':
shape_att = (nb_images, 2048, 14, 14)
hdf5_att = hdf5_file.create_dataset('att', shape_att,
dtype='f')#, compression='gzip')
if mode == 'both' or mode == 'noatt':
shape_noatt = (nb_images, 2048)
hdf5_noatt = hdf5_file.create_dataset('noatt', shape_noatt,
dtype='f')#, compression='gzip')
model.eval()
batch_time = AvgMeter()
data_time = AvgMeter()
begin = time.time()
end = time.time()
idx = 0
for i, input in enumerate(data_loader):
input_var = torch.autograd.Variable(input['visual'], volatile=True)
output_att = model(input_var)
nb_regions = output_att.size(2) * output_att.size(3)
output_noatt = output_att.sum(3).sum(2).div(nb_regions).view(-1, 2048)
batch_size = output_att.size(0)
if mode == 'both' or mode == 'att':
hdf5_att[idx:idx+batch_size] = output_att.data.cpu().numpy()
if mode == 'both' or mode == 'noatt':
hdf5_noatt[idx:idx+batch_size] = output_noatt.data.cpu().numpy()
idx += batch_size
batch_time.update(time.time() - end)
end = time.time()
if i % 1 == 0:
print('Extract: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'.format(
i, len(data_loader),
batch_time=batch_time,
data_time=data_time,))
hdf5_file.close()
# Saving image names in the same order than extraction
with open(path_txt, 'w') as handle:
for name in data_loader.dataset.dataset.imgs:
handle.write(name + '\n')
end = time.time() - begin
print('Finished in {}m and {}s'.format(int(end/60), int(end%60)))
output_att = model(input_var)
nb_regions = output_att.size(2) * output_att.size(3)
output_noatt = output_att.sum(3).sum(2).div(nb_regions).view(-1, 2048)
batch_size = output_att.size(0)
if mode == 'both' or mode == 'att':
#hdf5_att[idx:idx+batch_size] = output_att.data.cpu().numpy()
if mode == 'both' or mode == 'noatt':
#print(np.shape(output_noatt.data.cpu().numpy()))
#print('2',np.shape(hdf5_noatt[idx:idx+batch_size]))
#hdf5_noatt[idx:idx+batch_size] = output_noatt.data.cpu().numpy()
idx += batch_size
torch.cuda.synchronize()
batch_time.update(time.time() - end)
end = time.time()
if i % 1 == 0:
print('Extract: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'.format(
i, len(data_loader),
batch_time=batch_time,
data_time=data_time,))
#hdf5_file.close()
# Saving image names in the same order than extraction
#with open(path_txt, 'w') as handle:
# for name in data_loader.dataset.dataset.imgs:
def extract(data_loader, model, path_file, mode, is_augment_image):
path_hdf5 = path_file + '.hdf5'
path_txt = path_file + '.txt'
if os.path.exists(path_hdf5):
print("remove existing", path_hdf5)
os.remove(path_hdf5)
hdf5_file = h5py.File(path_hdf5, 'w')
# estimate output shapes
output, hidden = model(Variable(torch.ones(1, 3, args.size, args.size)))
nb_images = len(data_loader.dataset)
if mode == 'both' or mode == 'att':
shape_att = (nb_images, output.size(1), output.size(2), output.size(3))
print('Warning: shape_att={}'.format(shape_att))
hdf5_att = hdf5_file.create_dataset('att', shape_att,
dtype='f') # , compression='gzip')
if mode == 'both' or mode == 'noatt':
shape_noatt = (nb_images, output.size(1))
print('Warning: shape_noatt={}'.format(shape_noatt))
hdf5_noatt = hdf5_file.create_dataset(
'noatt', shape_noatt, dtype='f') # , compression='gzip')
model.eval()
batch_time = AvgMeter()
data_time = AvgMeter()
begin = time.time()
end = time.time()
idx = 0
if gen_utils.str2bool(is_augment_image):
print("\n>> extract augmented images\n")
else:
print("\n>> extract original images\n")
with torch.no_grad():
for i, input in enumerate(data_loader):
print_utils.print_tqdm(i, len(data_loader), cutoff=10)
input_var = Variable(input['visual'])
output_att, _ = model(input_var)
nb_regions = output_att.size(2) * output_att.size(3)
output_noatt = output_att.sum(3).sum(2).div(nb_regions).view(
-1, 2048)
batch_size = output_att.size(0)
if mode == 'both' or mode == 'att':
hdf5_att[idx:idx + batch_size] = output_att.data.cpu().numpy()
if mode == 'both' or mode == 'noatt':
hdf5_noatt[idx:idx +
batch_size] = output_noatt.data.cpu().numpy()
idx += batch_size
torch.cuda.synchronize()
batch_time.update(time.time() - end)
end = time.time()
hdf5_file.close()
# Saving image names in the same order than extraction
with open(path_txt, 'w') as handle:
for name in data_loader.dataset.dataset.imgs:
handle.write(name + '\n')
end = time.time() - begin
print('Finished in {}m and {}s'.format(int(end / 60), int(end % 60)))