def test_split_dataset_n_random(self):
original = list(range(6))
subsets = datasets.split_dataset_n_random(original, 2)
reconst = sorted(set(subsets[0]).union(subsets[1]))
self.assertEqual(reconst, original)
subsets1 = datasets.split_dataset_n_random(original, 2, seed=3)
reconst = sorted(set(subsets1[0]).union(subsets1[1]))
self.assertEqual(reconst, original)
subsets2 = datasets.split_dataset_n_random(original, 2, seed=3)
self.assertEqual(set(subsets1[0]), set(subsets2[0]))
self.assertEqual(set(subsets1[1]), set(subsets2[1]))
original = list(range(7))
subsets = datasets.split_dataset_n_random(original, 3)
self.assertEqual(len(subsets), 3)
for subset in subsets:
self.assertEqual(len(subset), 2)
reconst = set(subsets[0]).union(subsets[1]).union(subsets[2])
self.assertEqual(len(reconst), 6)
def test_split_dataset_n_random(self):
original = list(range(6))
subsets = datasets.split_dataset_n_random(original, 2)
reconst = sorted(set(subsets[0]).union(subsets[1]))
self.assertEqual(reconst, original)
subsets1 = datasets.split_dataset_n_random(original, 2, seed=3)
reconst = sorted(set(subsets1[0]).union(subsets1[1]))
self.assertEqual(reconst, original)
subsets2 = datasets.split_dataset_n_random(original, 2, seed=3)
self.assertEqual(set(subsets1[0]), set(subsets2[0]))
self.assertEqual(set(subsets1[1]), set(subsets2[1]))
original = list(range(7))
subsets = datasets.split_dataset_n_random(original, 3)
self.assertEqual(len(subsets), 3)
for subset in subsets:
self.assertEqual(len(subset), 2)
reconst = set(subsets[0]).union(subsets[1]).union(subsets[2])
self.assertEqual(len(reconst), 6)
# Setup an optimizer
if args.opt == 'nesterov':
optimizer = chainer.optimizers.NesterovAG()
else:
optimizer = chainer.optimizers.MomentumSGD()
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(1e-4))
if len(args.gpus) < 2:
train_iter = chainer.iterators.SerialIterator(train, args.batchsize)
else:
train_iters = [
MultithreadIterator(i, int(args.batchsize / len(devices)),
n_threads=4)
for i in split_dataset_n_random(train, len(devices))]
test_iter = MultithreadIterator(test, args.batchsize, repeat=False,
shuffle=False, n_threads=4)
# Set up a trainer
if len(args.gpus) < 2:
updater = training.StandardUpdater(train_iter, optimizer,
device=args.gpus[0])
else:
updater = MultiprocessParallelUpdater(train_iters, optimizer,
devices=devices)
trainer = training.Trainer(updater, (args.epoch, 'epoch'),
out=output_dir)
if args.cosine:
def main():
parser = argparse.ArgumentParser()
parser.add_argument('dataset', help="path to train json file")
parser.add_argument('test_dataset', help="path to test dataset json file")
parser.add_argument(
'--dataset-root',
help=
"path to dataset root if dataset file is not already in root folder of dataset"
)
parser.add_argument('--model',
choices=('ssd300', 'ssd512'),
default='ssd512')
parser.add_argument('--batchsize', type=int, default=32)
parser.add_argument('--gpu', type=int, nargs='*', default=[])
parser.add_argument('--out', default='result')
parser.add_argument('--resume')
parser.add_argument('--lr',
type=float,
default=0.001,
help="default learning rate")
parser.add_argument('--port',
type=int,
default=1337,
help="port for bbox sending")
parser.add_argument('--ip',
default='127.0.0.1',
help="destination ip for bbox sending")
parser.add_argument(
'--test-image',
help="path to test image that shall be displayed in bbox vis")
args = parser.parse_args()
if args.dataset_root is None:
args.dataset_root = os.path.dirname(args.dataset)
if args.model == 'ssd300':
model = SSD300(n_fg_class=1, pretrained_model='imagenet')
image_size = (300, 300)
elif args.model == 'ssd512':
model = SSD512(n_fg_class=1, pretrained_model='imagenet')
image_size = (512, 512)
else:
raise NotImplementedError("The model you want to train does not exist")
model.use_preset('evaluate')
train_chain = MultiboxTrainChain(model)
train = TransformDataset(
SheepDataset(args.dataset_root, args.dataset, image_size=image_size),
Transform(model.coder, model.insize, model.mean))
if len(args.gpu) > 1:
gpu_datasets = split_dataset_n_random(train, len(args.gpu))
if not len(gpu_datasets[0]) == len(gpu_datasets[-1]):
adapted_second_split = split_dataset(gpu_datasets[-1],
len(gpu_datasets[0]))[0]
gpu_datasets[-1] = adapted_second_split
else:
gpu_datasets = [train]
train_iter = [
ThreadIterator(gpu_dataset, args.batchsize)
for gpu_dataset in gpu_datasets
]
test = SheepDataset(args.dataset_root,
args.test_dataset,
image_size=image_size)
test_iter = chainer.iterators.MultithreadIterator(test,
args.batchsize,
repeat=False,
shuffle=False,
n_threads=2)
# initial lr is set to 1e-3 by ExponentialShift
optimizer = chainer.optimizers.Adam(alpha=args.lr)
optimizer.setup(train_chain)
for param in train_chain.params():
if param.name == 'b':
param.update_rule.add_hook(GradientScaling(2))
else:
param.update_rule.add_hook(WeightDecay(0.0005))
if len(args.gpu) <= 1:
updater = training.updaters.StandardUpdater(
train_iter[0],
optimizer,
device=args.gpu[0] if len(args.gpu) > 0 else -1,
)
else:
updater = training.updaters.MultiprocessParallelUpdater(
train_iter, optimizer, devices=args.gpu)
updater.setup_workers()
if len(args.gpu) > 0 and args.gpu[0] >= 0:
chainer.backends.cuda.get_device_from_id(args.gpu[0]).use()
model.to_gpu()
trainer = training.Trainer(updater, (200, 'epoch'), args.out)
trainer.extend(DetectionVOCEvaluator(test_iter,
model,
use_07_metric=True,
label_names=voc_bbox_label_names),
trigger=(1000, 'iteration'))
# build logger
# make sure to log all data necessary for prediction
log_interval = 100, 'iteration'
data_to_log = {
'image_size': image_size,
'model_type': args.model,
}
# add all command line arguments
for argument in filter(lambda x: not x.startswith('_'), dir(args)):
data_to_log[argument] = getattr(args, argument)
# create callback that logs all auxiliary data the first time things get logged
def backup_train_config(stats_cpu):
if stats_cpu['iteration'] == log_interval:
stats_cpu.update(data_to_log)
trainer.extend(
extensions.LogReport(trigger=log_interval,
postprocess=backup_train_config))
trainer.extend(extensions.observe_lr(), trigger=log_interval)
trainer.extend(extensions.PrintReport([
'epoch', 'iteration', 'lr', 'main/loss', 'main/loss/loc',
'main/loss/conf', 'validation/main/map'
]),
trigger=log_interval)
trainer.extend(extensions.ProgressBar(update_interval=10))
trainer.extend(extensions.snapshot_object(
model, 'model_iter_{.updater.iteration}'),
trigger=(5000, 'iteration'))
if args.test_image is not None:
plot_image = train._dataset.load_image(args.test_image,
resize_to=image_size)
else:
plot_image, _, _ = train.get_example(0)
plot_image += train._transform.mean
bbox_plotter = BBOXPlotter(
plot_image,
os.path.join(args.out, 'bboxes'),
send_bboxes=True,
upstream_port=args.port,
upstream_ip=args.ip,
)
trainer.extend(bbox_plotter, trigger=(10, 'iteration'))
if args.resume:
serializers.load_npz(args.resume, trainer)
trainer.run()