def get_batch():
opts = tutor_opts.parse_opt()
loader = DataLoader(opts)
get_batch = loader.get_batch(split='train', batch_size=None)
att_feats = get_batch['att_feats']
print(
f' - x=loader.get_batch(), x["att_feats"] : {att_feats.size(), att_feats.type()}'
)
def prefetch():
opts = tutor_opts.parse_opt()
loader = DataLoader(opts)
"""
opts.seq_per_img # 5
opts.batch_size # 16
"""
tmp_fc, tmp_att, tmp_seq, ix, tmp_wrapped = loader._prefetch_process[
'train'].get()
print(f'* tmp_fc :\n{tmp_fc}\n')
print(f'* tmp_att :\n{tmp_att}\n')
print(f'* tmp_seq :\n{tmp_seq}\n')
print(f'* ix :\n{ix}\n')
print(f'* tmp_wrapped :\n{tmp_wrapped}\n')
def dataloader():
class SubsetSampler(data.sampler.Sampler):
r"""Samples elements randomly from a given list of indices, without replacement.
Arguments:
indices (list): a list of indices
"""
def __init__(self, indices):
self.indices = indices
def __iter__(self):
return (self.indices[i] for i in range(len(self.indices)))
def __len__(self):
return len(self.indices)
opts = tutor_opts.parse_opt()
info = json.load(open(opts.input_json))
split_ix = {'train': [], 'val': [], 'test': []}
iterators = {'train': 0, 'val': 0, 'test': 0}
for ix in range(len(info['images'])):
split_ix['train'].append(ix)
dadaloader = data.Dataset()
setattr(dataloader, 'split_ix', split_ix)
setattr(dataloader, 'iterators', iterators)
split_loader = iter(
data.DataLoader(
dataset=dataloader,
batch_size=1,
sampler=SubsetSampler(
dataloader.split_ix['train'][dataloader.iterators['train']:]),
shuffle=False,
pin_memory=True,
num_workers=4, # 4 is usually enough
collate_fn=lambda x: x[0]))
tmp = split_loader.next()
import os, sys
sys.path.append('../')
import tutor_opts
from tutor_loader import DataLoader
opts = tutor_opts.parse_opt()
loader = DataLoader(opts)
def loader_attr():
print('\n* loader dir info')
d = []
for i in dir(loader):
if str(i)[0:2] != "__":
print(f' - {i}')
d.append(str(i))
print('\n* loader vars info')
v = []
for i in vars(loader):
print(f' - {i}')
v.append(str(i))
print('\n* loader method info')
method = set(d) - set(v)
for i in method:
print(f' - {i}')
def loader_simple_information():
def split_ix():
opts = tutor_opts.parse_opt()
info = json.load(open(opts.input_json))
split_ix = {'train': [], 'val': [], 'test': []}
for ix in range(len(info['images'])):
split_ix['train'].append(ix)