def test_iterator_serialize(self):
dataset = [1, 2, 3, 4, 5, 6]
it = iterators.MultiprocessIterator(dataset, 2, **self.options)
self.assertEqual(it.epoch, 0)
self.assertAlmostEqual(it.epoch_detail, 0 / 6)
batch1 = it.next()
self.assertEqual(len(batch1), 2)
self.assertIsInstance(batch1, list)
self.assertFalse(it.is_new_epoch)
self.assertAlmostEqual(it.epoch_detail, 2 / 6)
batch2 = it.next()
self.assertEqual(len(batch2), 2)
self.assertIsInstance(batch2, list)
self.assertFalse(it.is_new_epoch)
self.assertAlmostEqual(it.epoch_detail, 4 / 6)
target = dict()
it.serialize(DummySerializer(target))
it = iterators.MultiprocessIterator(dataset, 2, **self.options)
it.serialize(DummyDeserializer(target))
self.assertFalse(it.is_new_epoch)
self.assertAlmostEqual(it.epoch_detail, 4 / 6)
batch3 = it.next()
self.assertEqual(len(batch3), 2)
self.assertIsInstance(batch3, list)
self.assertTrue(it.is_new_epoch)
self.assertEqual(sorted(batch1 + batch2 + batch3), dataset)
self.assertAlmostEqual(it.epoch_detail, 6 / 6)
def get_data_iterators(data_dir,
batch_size,
num_workers,
num_classes,
input_image_size=224,
resize_inv_factor=0.875):
assert (resize_inv_factor > 0.0)
resize_value = int(math.ceil(float(input_image_size) / resize_inv_factor))
train_dir_path = os.path.join(data_dir, 'train')
train_dataset = PreprocessedDataset(root=train_dir_path,
scale_size=resize_value,
crop_size=input_image_size)
assert (len(directory_parsing_label_names(train_dir_path)) == num_classes)
val_dir_path = os.path.join(data_dir, 'val')
val_dataset = PreprocessedDataset(root=val_dir_path,
scale_size=resize_value,
crop_size=input_image_size)
assert (len(directory_parsing_label_names(val_dir_path)) == num_classes)
train_iterator = iterators.MultiprocessIterator(dataset=train_dataset,
batch_size=batch_size,
repeat=False,
shuffle=True,
n_processes=num_workers)
val_iterator = iterators.MultiprocessIterator(dataset=val_dataset,
batch_size=batch_size,
repeat=False,
shuffle=False,
n_processes=num_workers)
return train_iterator, val_iterator
def get_data_iterators(data_dir,
batch_size,
num_workers,
num_classes):
train_dir_path = os.path.join(data_dir, 'train')
train_dataset = PreprocessedDataset(root=train_dir_path)
assert(len(directory_parsing_label_names(train_dir_path)) == num_classes)
val_dir_path = os.path.join(data_dir, 'val')
val_dataset = PreprocessedDataset(root=val_dir_path)
assert (len(directory_parsing_label_names(val_dir_path)) == num_classes)
train_iterator = iterators.MultiprocessIterator(
dataset=train_dataset,
batch_size=batch_size,
repeat=False,
shuffle=True,
n_processes=num_workers)
val_iterator = iterators.MultiprocessIterator(
dataset=val_dataset,
batch_size=batch_size,
repeat=False,
shuffle=False,
n_processes=num_workers)
return train_iterator, val_iterator
def test_reproduce_same_permutation(self):
dataset = [1, 2, 3, 4, 5, 6]
numpy.random.seed(self._seed)
it1 = iterators.MultiprocessIterator(dataset, 6)
numpy.random.seed(self._seed)
it2 = iterators.MultiprocessIterator(dataset, 6)
for _ in range(5):
self.assertEqual(it1.next(), it2.next())
def create_iterators(train_dataset, valid_dataset, config):
train = Dataset(**config['dataset']['train'])
valid = Dataset(**config['dataset']['valid'])
train_iter = iterators.MultiprocessIterator(train_dataset, train.batchsize)
valid_iter = iterators.MultiprocessIterator(valid_dataset,
valid.batchsize,
repeat=False,
shuffle=False)
return train_iter, valid_iter
def test_reproduce_same_permutation(self):
dataset = [1, 2, 3, 4, 5, 6]
order_sampler1 = iterators.ShuffleOrderSampler(
numpy.random.RandomState(self._seed))
it1 = iterators.MultiprocessIterator(
dataset, 6, order_sampler=order_sampler1)
order_sampler2 = iterators.ShuffleOrderSampler(
numpy.random.RandomState(self._seed))
it2 = iterators.MultiprocessIterator(
dataset, 6, order_sampler=order_sampler2)
for _ in range(5):
self.assertEqual(it1.next(), it2.next())
def run(batch_size, n_process, prefetch,
model_name, exits_bn, activation_function, number_filter_list,
gpu_id, lossfun, learning_rate, max_epoch, out_dir, epoch):
train, test = get_image()
train = TransformDataset(train, trans)
test = TransformDataset(test, trans)
train_iter = iterators.MultiprocessIterator(train, batch_size, True, True, n_process, prefetch)
test_iter = iterators.MultiprocessIterator(test, batch_size, False, False, n_process, prefetch)
model = model_name(exits_bn, activation_function, number_filter_list)
if gpu_id >= 0:
model.to_gpu(gpu_id)
# Wrap your model by Classifier and include the process of loss calculation within your model.
# Since we do not specify a loss function here, the default 'softmax_cross_entropy' is used.
model = links.Loss_Classifier(model, lossfun)
# selection of your optimizing method
optimizer = optimizers.MomentumSGD(lr=learning_rate, momentum=0.9)
# Give the optimizer a reference to the model
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(0.0005))
# Get an updater that uses the Iterator and Optimizer
updater = training.updaters.StandardUpdater(train_iter, optimizer, device=gpu_id)
# Setup a Trainer
trainer = training.Trainer(updater, (max_epoch, 'epoch'), out='{}'.format(out_dir))
from chainer.training import extensions
trainer.extend(extensions.LogReport()) # generate report
trainer.extend(extensions.snapshot(filename='snapshot_epoch-{.updater.epoch}')) # save updater
trainer.extend(extensions.snapshot_object(model.predictor, filename='model_epoch-{.updater.epoch}')) # save model
trainer.extend(extensions.Evaluator(test_iter, model, device=gpu_id)) # validation
trainer.extend(extensions.PrintReport(['epoch', 'main/loss', 'main/accuracy', 'validation/main/loss', 'validation/main/accuracy', 'elapsed_time'])) # show loss and accuracy
trainer.extend(extensions.ProgressBar()) # show trainning progress
trainer.extend(extensions.PlotReport(['main/loss', 'validation/main/loss'], x_key='epoch', file_name='loss.png')) # loss curve
trainer.extend(extensions.PlotReport(['main/accuracy', 'validation/main/accuracy'], x_key='epoch', file_name='accuracy.png')) # accuracy curve
trainer.extend(extensions.dump_graph('main/loss'))
if epoch > 0:
serializers.load_npz('./{}/snapshot_epoch-{}'.format(out_dir, epoch), trainer)
trainer.updater.get_optimizer('main').lr = learning_rate
trainer.run()
def run_training(
net, train, valid, result_dir, batchsize=64, devices=-1,
training_epoch=300, initial_lr=0.05, lr_decay_rate=0.5,
lr_decay_epoch=30, weight_decay=0.0005):
# Iterator
train_iter = iterators.MultiprocessIterator(train, batchsize)
test_iter = iterators.MultiprocessIterator(valid, batchsize, False, False)
# Model
net = L.Classifier(net)
# Optimizer
optimizer = optimizers.MomentumSGD(lr=initial_lr)
optimizer.setup(net)
if weight_decay > 0:
optimizer.add_hook(chainer.optimizer.WeightDecay(weight_decay))
# Updater
if isinstance(devices, int):
devices['main'] = devices
updater = training.StandardUpdater(
train_iter, optimizer, device=devices)
elif isinstance(devices, dict):
updater = training.ParallelUpdater(
train_iter, optimizer, devices=devices)
# 6. Trainer
trainer = training.Trainer(
updater, (training_epoch, 'epoch'), out=result_dir)
# 7. Trainer extensions
trainer.extend(extensions.LogReport())
trainer.extend(extensions.observe_lr())
trainer.extend(extensions.Evaluator(
test_iter, net, device=devices['main']), name='val')
trainer.extend(extensions.PrintReport(
['epoch', 'main/loss', 'main/accuracy', 'val/main/loss',
'val/main/accuracy', 'elapsed_time', 'lr']))
trainer.extend(extensions.PlotReport(
['main/loss', 'val/main/loss'], x_key='epoch', file_name='loss.png'))
trainer.extend(extensions.PlotReport(
['main/accuracy', 'val/main/accuracy'], x_key='epoch',
file_name='accuracy.png'))
trainer.extend(extensions.ExponentialShift(
'lr', lr_decay_rate), trigger=(lr_decay_epoch, 'epoch'))
trainer.extend(extensions.snapshot_object(net.predictor, 'model_{.updater.epoch}.npz'), trigger=(10, 'epoch'))
trainer.run()
return net
def create_iterators(train_dataset, batchsize, valid_dataset, valid_batchsize,
devices):
if HAVE_NCCL and len(devices) > 1:
train_iter = [
iterators.MultiprocessIterator(i, batchsize)
for i in chainer.datasets.split_dataset_n_random(
train_dataset, len(devices))
]
else:
train_iter = iterators.MultiprocessIterator(train_dataset, batchsize)
valid_iter = iterators.MultiprocessIterator(valid_dataset,
valid_batchsize,
repeat=False,
shuffle=False)
return train_iter, valid_iter
def train(network_object,
batchsize=128,
gpu_id=0,
max_epoch=20,
train_dataset=None,
test_dataset=None,
postfix='',
base_lr=0.01,
lr_decay=None):
#1. Dataset
if train_dataset is None and test_dataset is None:
train, test = cifar.get_cifar10()
else:
train, test = train_dataset, test_dataset
#2. Iterator
train_iter = iterators.MultiprocessIterator(train, batchsize)
test_iter = iterators.MultiprocessIterator(test, batchsize, False, False)
#3. Model
net = L.Classifier(network_object)
#4. Optimizer
optimizer = optimizers.MomentumSGD(lr=base_lr)
optimizer.setup(net)
optimizer.add_hook(chainer.optimizer.WeightDecay(0.0005))
#5. Updater
updater = training.StandardUpdater(train_iter, optimizer, device=gpu_id)
#6. Trainer
trainer = training.Trainer(updater, (max_epoch, 'epoch'), out='{}_cifar10_{}result'.format(network_object.__class__.__name__, postfix))
#7. Trainer extensions
trainer.extend(extensions.LogReport())
trainer.extend(extensions.observe_lr())
trainer.extend(extensions.Evaluator(test_iter, net, device=gpu_id), name='val')
trainer.extend(extensions.PrintReport(['epoch', 'main/loss', 'main/accuracy', 'val/main/loss', 'val/main/accuracy', 'elapsed_time', 'lr']))
trainer.extend(extensions.PlotReport(['main/loss', 'val/main/loss'], x_key='epoch', file_name='loss.png'))
trainer.extend(extensions.PlotReport(['main/accuracy', 'val/main/accuracy'], x_key='epoch', file_name='accuracy.png'))
if lr_decay is not None:
trainer.extend(extensions.ExponentialShift('lr', 0.1), trigger=lr_decay)
trainer.run()
del trainer
return net
def test_iterator_repeat(self):
dataset = [1, 2, 3]
it = iterators.MultiprocessIterator(dataset, 2, **self.options)
for i in range(3):
self.assertEqual(it.epoch, i)
self.assertAlmostEqual(it.epoch_detail, i + 0 / 6)
if i == 0:
self.assertIsNone(it.previous_epoch_detail)
else:
self.assertAlmostEqual(it.previous_epoch_detail, i - 2 / 6)
batch1 = it.next()
self.assertEqual(len(batch1), 2)
self.assertIsInstance(batch1, list)
self.assertFalse(it.is_new_epoch)
self.assertAlmostEqual(it.epoch_detail, i + 2 / 6)
self.assertAlmostEqual(it.previous_epoch_detail, i + 0 / 6)
batch2 = it.next()
self.assertEqual(len(batch2), 2)
self.assertIsInstance(batch2, list)
self.assertFalse(it.is_new_epoch)
self.assertAlmostEqual(it.epoch_detail, i + 4 / 6)
self.assertAlmostEqual(it.previous_epoch_detail, i + 2 / 6)
batch3 = it.next()
self.assertEqual(len(batch3), 2)
self.assertIsInstance(batch3, list)
self.assertTrue(it.is_new_epoch)
self.assertAlmostEqual(it.epoch_detail, i + 6 / 6)
self.assertAlmostEqual(it.previous_epoch_detail, i + 4 / 6)
self.assertEqual(sorted(batch1 + batch2 + batch3),
[1, 1, 2, 2, 3, 3])
def test_iterator_pickle_after_init(self):
dataset = [1, 2, 3, 4, 5, 6]
it = iterators.MultiprocessIterator(dataset, 2, **self.options)
self.assertEqual(it.epoch, 0)
self.assertAlmostEqual(it.epoch_detail, 0 / 6)
self.assertIsNone(it.previous_epoch_detail)
batch1 = it.next()
self.assertEqual(len(batch1), 2)
self.assertIsInstance(batch1, list)
self.assertFalse(it.is_new_epoch)
self.assertAlmostEqual(it.epoch_detail, 2 / 6)
self.assertAlmostEqual(it.previous_epoch_detail, 0 / 6)
batch2 = it.next()
self.assertEqual(len(batch2), 2)
self.assertIsInstance(batch2, list)
self.assertFalse(it.is_new_epoch)
self.assertAlmostEqual(it.epoch_detail, 4 / 6)
self.assertAlmostEqual(it.previous_epoch_detail, 2 / 6)
pickled_it = pickle.dumps(it)
it = pickle.loads(pickled_it)
self.assertFalse(it.is_new_epoch)
self.assertAlmostEqual(it.epoch_detail, 4 / 6)
self.assertAlmostEqual(it.previous_epoch_detail, 2 / 6)
batch3 = it.next()
self.assertEqual(len(batch3), 2)
self.assertIsInstance(batch3, list)
self.assertTrue(it.is_new_epoch)
self.assertEqual(sorted(batch1 + batch2 + batch3), dataset)
self.assertAlmostEqual(it.epoch_detail, 6 / 6)
self.assertAlmostEqual(it.previous_epoch_detail, 4 / 6)
def test_stalled_getitem(self):
nth = self.nth
batch_size = 2
sleep = 0.5
timeout = 0.1
dataset = StallingDataset(nth, sleep)
it = iterators.MultiprocessIterator(dataset,
batch_size=batch_size,
shuffle=False,
dataset_timeout=timeout,
repeat=False)
# TimeoutWarning should be issued.
warning_cls = iterators.MultiprocessIterator.TimeoutWarning
data = []
# No warning until the stalling batch
for i in range(nth // batch_size):
data.append(it.next())
# Warning on the stalling batch
with testing.assert_warns(warning_cls):
data.append(it.next())
# Retrieve data until the end
while True:
try:
data.append(it.next())
except StopIteration:
break
# All data must be retrieved
assert data == [
dataset.data[i * batch_size:(i + 1) * batch_size]
for i in range((len(dataset) + batch_size - 1) // batch_size)
]
def test_iterator_list_type(self):
dataset = [[i, numpy.zeros((10, )) + i] for i in range(6)]
it = iterators.MultiprocessIterator(dataset, 2, **self.options)
for i in range(3):
self.assertEqual(it.epoch, i)
self.assertAlmostEqual(it.epoch_detail, i)
if i == 0:
self.assertIsNone(it.previous_epoch_detail)
else:
self.assertAlmostEqual(it.previous_epoch_detail, i - 2 / 6)
batches = {}
for j in range(3):
batch = it.next()
self.assertEqual(len(batch), 2)
if j != 2:
self.assertFalse(it.is_new_epoch)
else:
self.assertTrue(it.is_new_epoch)
self.assertAlmostEqual(it.epoch_detail,
(3 * i + j + 1) * 2 / 6)
self.assertAlmostEqual(it.previous_epoch_detail,
(3 * i + j) * 2 / 6)
for x in batch:
self.assertIsInstance(x, list)
self.assertIsInstance(x[1], numpy.ndarray)
batches[x[0]] = x[1]
self.assertEqual(len(batches), len(dataset))
for k, v in six.iteritems(batches):
numpy.testing.assert_allclose(dataset[k][1], v)
def test_iterator_dict_type(self):
dataset = [{i: numpy.zeros((10, )) + i} for i in range(6)]
it = iterators.MultiprocessIterator(dataset, 2, **self.options)
for i in range(3):
self.assertEqual(it.epoch, i)
self.assertAlmostEqual(it.epoch_detail, i)
batches = {}
for j in range(3):
batch = it.next()
self.assertEqual(len(batch), 2)
if j != 2:
self.assertFalse(it.is_new_epoch)
else:
self.assertTrue(it.is_new_epoch)
self.assertAlmostEqual(it.epoch_detail,
(3 * i + j + 1) * 2 / 6)
for x in batch:
self.assertIsInstance(x, dict)
k = tuple(x)[0]
v = x[k]
self.assertIsInstance(v, numpy.ndarray)
batches[k] = v
self.assertEqual(len(batches), len(dataset))
for k, v in six.iteritems(batches):
x = dataset[k][tuple(dataset[k])[0]]
numpy.testing.assert_allclose(x, v)
def test_invalid_order_sampler(self):
dataset = [1, 2, 3, 4, 5, 6]
with self.assertRaises(ValueError):
it = iterators.MultiprocessIterator(
dataset, 6, shuffle=None, order_sampler=_InvalidOrderSampler())
it.next()
def test_iterator_shuffle_nondivisible(self):
dataset = list(range(10))
it = iterators.MultiprocessIterator(dataset,
3,
n_processes=self.n_processes)
out = sum([it.next() for _ in range(7)], [])
self.assertNotEqual(out[0:10], out[10:20])
def test_unsupported_reset_middle(self):
dataset = [1, 2, 3, 4, 5]
it = iterators.MultiprocessIterator(dataset,
2,
repeat=False,
**self.options)
it.next()
self.assertRaises(NotImplementedError, it.reset)
def get_data_iterators(batch_size, num_workers):
train_dataset = PreprocessedCIFARDataset(train=True)
train_iterator = iterators.MultiprocessIterator(dataset=train_dataset,
batch_size=batch_size,
repeat=False,
shuffle=True,
n_processes=num_workers)
val_dataset = PreprocessedCIFARDataset(train=False)
val_iterator = iterators.MultiprocessIterator(dataset=val_dataset,
batch_size=batch_size,
repeat=False,
shuffle=False,
n_processes=num_workers)
return train_iterator, val_iterator
def test_iterator_repeat_not_even(self):
dataset = [1, 2, 3, 4, 5]
it = iterators.MultiprocessIterator(dataset,
2,
n_processes=self.n_processes)
batches = sum([it.next() for _ in range(5)], [])
self.assertEqual(sorted(batches), sorted(dataset * 2))
def train_CNN(network_object, batchsize=128, gpu_id=-1, max_epoch=20, train_dataset=None, test_dataset=None, postfix='', base_lr=0.01, lr_decay=None,number = 11):
number = str(number)
# 1. Dataset
if train_dataset is None and test_dataset is None:
train, test = cifar.get_cifar10()
else:
train, test = train_dataset, test_dataset
if gpu_id >= 0:
network_object.to_gpu(gpu_id)
# 2. Iterator
train_iter = iterators.MultiprocessIterator(train, batchsize)
test_iter = iterators.MultiprocessIterator(test, batchsize, False, False)
# 3. Model
net = L.Classifier(network_object)
# 4. Optimizer
optimizer = optimizers.MomentumSGD()
optimizer.setup(net)
optimizer.add_hook(chainer.optimizer.WeightDecay(0.0005))
# 5. Updater
updater = training.StandardUpdater(train_iter, optimizer, device=gpu_id)
# 6. Trainer
trainer = training.Trainer(updater, (max_epoch, 'epoch'), out='{}_crack_{}result'.format(network_object.__class__.__name__, postfix))
# 7. Trainer extensions
trainer.extend(extensions.LogReport(trigger=(1, 'epoch'), log_name="log_"+number))
trainer.extend(extensions.snapshot(filename=number+'snapshot_epoch-{.updater.epoch}'),trigger=(5, 'epoch'))
# trainer.extend(extensions.snapshot(), trigger=(10, 'epoch'))
trainer.extend(extensions.ParameterStatistics(net.predictor.conv1, {'std': np.std}))
trainer.extend(extensions.observe_lr())
trainer.extend(extensions.Evaluator(test_iter, net, device=gpu_id), name='val')
trainer.extend(extensions.PrintReport(['epoch', 'main/loss', 'main/accuracy', 'val/main/loss', 'val/main/accuracy', 'elapsed_time', 'lr']))
trainer.extend(extensions.PlotReport(['main/loss', 'val/main/loss'], x_key='epoch', file_name='loss'+number+'.png'))
trainer.extend(extensions.PlotReport(['main/accuracy', 'val/main/accuracy'], x_key='epoch', file_name='accuracy'+number+'.png'))
trainer.extend(extensions.PlotReport(['l1/W/data/std'], x_key='epoch', file_name='std'+number+'.png'))
if lr_decay is not None:
trainer.extend(extensions.ExponentialShift('lr', 0.1), trigger=lr_decay)
trainer.run()
del trainer
return net
def test_reset_repeat(self):
dataset = [1, 2, 3, 4]
it = iterators.MultiprocessIterator(
dataset, 2, repeat=True, **self.options)
for trial in range(4):
batches = sum([it.next() for _ in range(4)], [])
self.assertEqual(sorted(batches), sorted(2 * dataset))
it.reset()
def test_iterator_not_repeat(self):
dataset = [1, 2, 3, 4, 5]
it = iterators.MultiprocessIterator(
dataset, 2, repeat=False, **self.options)
batches = sum([it.next() for _ in range(3)], [])
self.assertEqual(sorted(batches), dataset)
for _ in range(2):
self.assertRaises(StopIteration, it.next)
def test_iterator_pickle_new(self):
dataset = [1, 2, 3, 4, 5, 6]
it = iterators.MultiprocessIterator(dataset, 2, **self.options)
self.assertEqual(it.epoch, 0)
self.assertAlmostEqual(it.epoch_detail, 0 / 6)
self.assertIsNone(it.previous_epoch_detail)
pickled_it = pickle.dumps(it)
it = pickle.loads(pickled_it)
def test_no_same_indices_order_sampler(self):
dataset = [1, 2, 3, 4, 5, 6]
batchsize = 5
it = iterators.MultiprocessIterator(
dataset, batchsize,
order_sampler=_NoSameIndicesOrderSampler(batchsize))
for _ in range(5):
batch = it.next()
self.assertEqual(len(numpy.unique(batch)), batchsize)
def main():
parser = argparse.ArgumentParser(description='training mnist')
parser.add_argument('--gpu',
'-g',
default=-1,
type=int,
help='GPU ID (negative value indicates CPU)')
parser.add_argument('--batchsize',
'-b',
type=int,
default=8,
help='Number of images in each mini-batch')
parser.add_argument('--load_model',
'-lm',
type=str,
default=None,
help='Path of the model object to load')
args = parser.parse_args()
backbone = 'mobilenet'
model = ModifiedClassifier(
DeepLab(n_class=13, task='semantic', backbone=backbone))
if args.load_model is not None:
serializers.load_npz(args.load_model, model)
else:
print('You need to specify path of the model object')
sys.exit()
if args.gpu >= 0:
chainer.cuda.get_device_from_id(args.gpu).use()
model.to_gpu()
dir_path = './dataset/2D-3D-S'
test_data = Stanford2D3DS(dir_path,
'semantic',
area='5a',
train=False,
n_data=100)
test_iter = iterators.MultiprocessIterator(test_data,
args.batchsize,
repeat=False,
shuffle=False)
label_list = list(test_data.label_dict.keys())[1:]
evaluator = ModifiedEvaluator(test_iter,
model,
label_names=label_list,
device=args.gpu)
observation = evaluator()
for k, v in observation.items():
print(k, v)
def main(dataset_name, snapshot_path):
"""
Args:
dataset_name: 'mpii' or 'lsp'.
init_snapshot_path: path to the snapshot to test
"""
if dataset_name == 'mpii':
TEST_CV_FILEPATH = os.path.join(config.MPII_DATASET_ROOT, 'test_joints.csv')
IMG_PATH_PREFIX = os.path.join(config.MPII_DATASET_ROOT, 'images')
symmetric_joints = "[[12, 13], [11, 14], [10, 15], [2, 3], [1, 4], [0, 5]]"
ignore_label = -100500
elif dataset_name == 'lsp':
TEST_CV_FILEPATH = os.path.join(config.LSP_DATASET_ROOT, 'test_joints.csv')
# IMG_PATH_PREFIX = ''
IMG_PATH_PREFIX = os.path.join(config.LSP_DATASET_ROOT, 'images')
symmetric_joints = "[[8, 9], [7, 10], [6, 11], [2, 3], [1, 4], [0, 5]]"
ignore_label = -1
else :
TEST_CV_FILEPATH = os.path.join(config.MET_DATASET_ROOT, 'test_joints_1.csv')
IMG_PATH_PREFIX = os.path.join(config.MET_DATASET_ROOT, 'images')
symmetric_joints = "[[8, 9], [7, 10], [6, 11], [2, 3], [1, 4], [0, 5]]"
ignore_label = -1
print(TEST_CV_FILEPATH)
print(IMG_PATH_PREFIX)
test_dataset = dataset.PoseDataset(
TEST_CV_FILEPATH,
IMG_PATH_PREFIX, 227,
fliplr=False, rotate=False,
shift=None,
bbox_extension_range=(1.0, 1.0),
coord_normalize=True,
gcn=True,
fname_index=0,
joint_index=1,
symmetric_joints=symmetric_joints,
ignore_label=ignore_label,
should_return_bbox=True,
should_downscale_images=True,
downscale_height=400
)
test_iterator = iterators.MultiprocessIterator(
test_dataset, batch_size=128,
repeat=False, shuffle=False,
n_processes=1, n_prefetch=1)
if dataset_name == 'MET' :
test_net(test_dataset, test_iterator, 'lsp', snapshot_path)
else :
test_net(test_dataset, test_iterator, dataset_name, snapshot_path)
def test_iterator_not_repeat_not_even(self):
dataset = [1, 2, 3, 4, 5]
it = iterators.MultiprocessIterator(
dataset, 2, repeat=False, **self.options)
batch1 = it.next()
batch2 = it.next()
batch3 = it.next()
self.assertRaises(StopIteration, it.next)
self.assertEqual(len(batch3), 1)
self.assertEqual(sorted(batch1 + batch2 + batch3), dataset)
def test_finalize_not_deadlock(self):
dataset = numpy.ones((1000, 1000))
it = iterators.MultiprocessIterator(dataset, 10, n_processes=4)
for _ in range(10):
it.next()
t = threading.Thread(target=lambda: it.finalize())
t.daemon = True
t.start()
t.join(5)
deadlock = t.is_alive()
self.assertFalse(deadlock)
def create_iterator(
settings: Dict,
train_dataset: Union[datasets.LabeledImageDataset, datasets.TupleDataset,
None] = None,
val_dataset: Union[datasets.LabeledImageDataset, datasets.TupleDataset,
None] = None,
test_dataset: Union[datasets.LabeledImageDataset, datasets.TupleDataset,
None] = None
) -> Tuple[Optional[chainer.iterators.MultiprocessIterator]]:
"""Create dataset iterator."""
gpu_num = len(settings["gpu_devices"])
if train_dataset is None:
train_iter = None
else:
if gpu_num == 1:
train_iter = iterators.MultiprocessIterator(
train_dataset,
settings["batch_size"],
n_processes=settings["n_processes"])
else:
assert gpu_num == len(train_dataset),\
" gpu num: {} != dataset num: {}".format(gpu_num, len(train_dataset))
train_iter = [
iterators.MultiprocessIterator(
sub_dataset,
settings["batch_size"],
n_processes=min(2, settings["n_processes"] // gpu_num))
for sub_dataset in train_dataset
]
val_iter = None if val_dataset is None else \
iterators.MultiprocessIterator(
val_dataset, settings["batch_size"], repeat=False, shuffle=False, n_processes=settings["n_processes"])
test_iter = None if test_dataset is None else \
iterators.MultiprocessIterator(
test_dataset, settings["batch_size"], repeat=False, shuffle=False, n_processes=settings["n_processes"])
return train_iter, val_iter, test_iter
