ap.add_argument("--shuffle", required=False, type=booltype, default=True)
ap.add_argument("--stratify", required=False, type=booltype, default=True)
ap.add_argument("--uniform", required=False, type=booltype, default=False)
ap.add_argument("--dry-run", required=False, type=booltype, default=False)
args = vars(ap.parse_args())
args = Struct(**args)
num_train = cfg.num_trains[args.dataset]
num_test = cfg.num_tests[args.dataset]
num_classes = cfg.output_sizes[args.dataset]
kwargs = {}
train_loader = get_loader(args.dataset,
num_train,
train=True,
subset=args.repeat)
test_loader = get_loader(args.dataset,
num_test,
train=False,
subset=args.repeat)
for data, target in train_loader:
X_train = data
y_train = target
for data, target in test_loader:
X_test = data
y_test = target
# def repeat_data(data, repeat):
datasets = [
# 'celeba', # regression
# 'cifar', # classification
# 'coco', 'voc', # semantic segmentation
'fmnist',
'mnist',
'svhn' # classification
]
test_subset = ['celeba', 'coco']
for dataset in datasets:
for split in [True, False]:
print('-' * 80)
print('dataset: {}, train:{}'.format(dataset, split))
loader = get_loader(dataset, batch_size=16, train=split, shuffle=True)
print('\tdata_size:', len(loader.dataset))
for data, label in loader:
print('\tbatch_size:', data.shape, label.shape)
break
if dataset not in test_subset:
continue
loader = get_loader(dataset,
batch_size=16,
train=split,
shuffle=True,
subset=0.5)
print('\tdata_size:', len(loader.dataset))
for data, label in loader:
print('\tbatch_size:', data.shape, label.shape)
tb = SummaryWriter(paths.tb_path)
print('+' * 80)
print(paths.model_name)
print('+' * 80)
print(args.__dict__)
print('+' * 80)
# prepare graph and data
_, workers = get_fl_graph(hook, args.num_workers)
print('Loading data: {}'.format(paths.data_path))
X_trains, _, y_trains, _, meta = pkl.load(open(paths.data_path, 'rb'))
test_loader = get_loader(args.dataset,
args.test_batch_size,
train=False,
noise=args.noise)
print('+' * 80)
# ------------------------------------------------------------------------------
# Fire the engines
# ------------------------------------------------------------------------------
model, loss_type = get_model(args, ckpt_path=args.load_model)
if args.batch_size == 0:
args.batch_size = int(meta['batch_size'])
print("Resetting batch size: {}...".format(args.batch_size))
print('+' * 80)
h_epoch = []
import argparse
from data.loader import get_loader
def get_parser():
parser = argparse.ArgumentParser()
parser.add_argument('--data_dir', type=str, required=True)
parser.add_argument('--batch_size', default=1, type=int)
parser.add_argument('--verbose', action="store_true")
parser.add_argument("--num_workers", default=8, type=int)
return parser
args = get_parser().parse_args()
loader = get_loader(args)
# print(iter(loader).next())
import pdb
pdb.set_trace()
for src, tgt in loader:
print(src[0]) #, tgt)
tb = SummaryWriter(paths.tb_path)
print('+' * 80)
print(paths.model_name)
print('+' * 80)
print(args.__dict__)
print('+' * 80)
if args.batch_size == 0:
args.batch_size = args.num_train
print("Resetting batch size: {}...".format(args.batch_size))
train_loader = get_loader(args.dataset,
args.batch_size,
train=True,
subset=args.repeat,
force_resize=cfg.model_im_size[args.clf])
test_loader = get_loader(args.dataset,
args.test_batch_size,
train=False,
shuffle=False,
subset=args.repeat,
force_resize=cfg.model_im_size[args.clf])
print('Train size: ', len(train_loader.dataset))
print('Test size: ', len(test_loader.dataset))
print('+' * 80)
# ------------------------------------------------------------------------------
# Fire the engines