def setUp(self): bboxes = [generate_random_bbox(5, (256, 324), 24, 120) for _ in range(10)] labels = np.ones((10, 5)) self.dataset = TupleDataset( np.random.uniform(size=(10, 3, 32, 48)), bboxes, labels) self.link = _DetectionStubLink(bboxes, labels) self.iterator = SerialIterator( self.dataset, 5, repeat=False, shuffle=False) self.evaluator = DetectionVOCEvaluator(self.iterator, self.link) self.expect_map = 1
class TestDetectionVOCEvaluator(unittest.TestCase): def setUp(self): bboxes = [ generate_random_bbox(5, (256, 324), 24, 120) for _ in range(10) ] labels = np.ones((10, 5)) self.dataset = TupleDataset(np.random.uniform(size=(10, 3, 32, 48)), bboxes, labels) self.link = _DetectionStubLink(bboxes, labels) self.iterator = SerialIterator(self.dataset, 5, repeat=False, shuffle=False) self.evaluator = DetectionVOCEvaluator(self.iterator, self.link, label_names=('cls0', 'cls1', 'cls2')) self.expected_ap = 1 def test_evaluate(self): reporter = chainer.Reporter() reporter.add_observer('target', self.link) with reporter: mean = self.evaluator.evaluate() # No observation is reported to the current reporter. Instead the # evaluator collect results in order to calculate their mean. self.assertEqual(len(reporter.observation), 0) np.testing.assert_equal(mean['target/map'], self.expected_ap) np.testing.assert_equal(mean['target/ap/cls0'], np.nan) np.testing.assert_equal(mean['target/ap/cls1'], self.expected_ap) np.testing.assert_equal(mean['target/ap/cls2'], np.nan) def test_call(self): mean = self.evaluator() # main is used as default np.testing.assert_equal(mean['main/map'], self.expected_ap) np.testing.assert_equal(mean['main/ap/cls0'], np.nan) np.testing.assert_equal(mean['main/ap/cls1'], self.expected_ap) np.testing.assert_equal(mean['main/ap/cls2'], np.nan) def test_evaluator_name(self): self.evaluator.name = 'eval' mean = self.evaluator() # name is used as a prefix np.testing.assert_equal(mean['eval/main/map'], self.expected_ap) np.testing.assert_equal(mean['eval/main/ap/cls0'], np.nan) np.testing.assert_equal(mean['eval/main/ap/cls1'], self.expected_ap) np.testing.assert_equal(mean['eval/main/ap/cls2'], np.nan) def test_current_report(self): reporter = chainer.Reporter() with reporter: mean = self.evaluator() # The result is reported to the current reporter. self.assertEqual(reporter.observation, mean)
class TestDetectionVOCEvaluator(unittest.TestCase): def setUp(self): bboxes = [generate_random_bbox(5, (256, 324), 24, 120) for _ in range(10)] labels = np.ones((10, 5)) self.dataset = TupleDataset( np.random.uniform(size=(10, 3, 32, 48)), bboxes, labels) self.link = _DetectionStubLink(bboxes, labels) self.iterator = SerialIterator( self.dataset, 5, repeat=False, shuffle=False) self.evaluator = DetectionVOCEvaluator( self.iterator, self.link, label_names=('cls0', 'cls1', 'cls2')) self.expected_ap = 1 def test_evaluate(self): reporter = chainer.Reporter() reporter.add_observer('target', self.link) with reporter: mean = self.evaluator.evaluate() # No observation is reported to the current reporter. Instead the # evaluator collect results in order to calculate their mean. self.assertEqual(len(reporter.observation), 0) np.testing.assert_equal(mean['target/map'], self.expected_ap) np.testing.assert_equal(mean['target/ap/cls0'], np.nan) np.testing.assert_equal(mean['target/ap/cls1'], self.expected_ap) np.testing.assert_equal(mean['target/ap/cls2'], np.nan) def test_call(self): mean = self.evaluator() # main is used as default np.testing.assert_equal(mean['main/map'], self.expected_ap) np.testing.assert_equal(mean['main/ap/cls0'], np.nan) np.testing.assert_equal(mean['main/ap/cls1'], self.expected_ap) np.testing.assert_equal(mean['main/ap/cls2'], np.nan) def test_evaluator_name(self): self.evaluator.name = 'eval' mean = self.evaluator() # name is used as a prefix np.testing.assert_equal(mean['eval/main/map'], self.expected_ap) np.testing.assert_equal(mean['eval/main/ap/cls0'], np.nan) np.testing.assert_equal(mean['eval/main/ap/cls1'], self.expected_ap) np.testing.assert_equal(mean['eval/main/ap/cls2'], np.nan) def test_current_report(self): reporter = chainer.Reporter() with reporter: mean = self.evaluator() # The result is reported to the current reporter. self.assertEqual(reporter.observation, mean)
def setUp(self): bboxes = [generate_random_bbox(5, (256, 324), 24, 120) for _ in range(10)] labels = np.ones((10, 5)) self.dataset = TupleDataset( np.random.uniform(size=(10, 3, 32, 48)), bboxes, labels) self.link = _DetectionStubLink(bboxes, labels) self.iterator = SerialIterator( self.dataset, 5, repeat=False, shuffle=False) self.evaluator = DetectionVOCEvaluator( self.iterator, self.link, label_names=('cls0', 'cls1', 'cls2')) self.expected_ap = 1
def test_consistency(self): reporter = chainer.Reporter() if self.comm.rank == 0: multi_iterator = SerialIterator(self.dataset, self.batchsize, repeat=False, shuffle=False) else: multi_iterator = None multi_link = _DetectionStubLink(self.bboxes, self.labels, self.initial_count) multi_evaluator = DetectionVOCEvaluator(multi_iterator, multi_link, label_names=('cls0', 'cls1', 'cls2'), comm=self.comm) reporter.add_observer('target', multi_link) with reporter: multi_mean = multi_evaluator.evaluate() if self.comm.rank != 0: self.assertEqual(multi_mean, {}) return single_iterator = SerialIterator(self.dataset, self.batchsize, repeat=False, shuffle=False) single_link = _DetectionStubLink(self.bboxes, self.labels) single_evaluator = DetectionVOCEvaluator(single_iterator, single_link, label_names=('cls0', 'cls1', 'cls2')) reporter.add_observer('target', single_link) with reporter: single_mean = single_evaluator.evaluate() self.assertEqual(set(multi_mean.keys()), set(single_mean.keys())) for key in multi_mean.keys(): np.testing.assert_equal(single_mean[key], multi_mean[key])
def main(): parser = argparse.ArgumentParser( description='ChainerCV training example: Faster R-CNN') parser.add_argument('--gpu', '-g', type=int, default=-1) parser.add_argument('--lr', '-l', type=float, default=1e-3) parser.add_argument('--out', '-o', default='result', help='Output directory') parser.add_argument('--seed', '-s', type=int, default=0) parser.add_argument('--step_size', '-ss', type=int, default=50000) parser.add_argument('--iteration', '-i', type=int, default=70000) parser.add_argument('--train_data_dir', '-t', default=WIDER_TRAIN_DIR, help='Training dataset (WIDER_train)') parser.add_argument('--train_annotation', '-ta', default=WIDER_TRAIN_ANNOTATION_MAT, help='Annotation file (.mat) for training dataset') parser.add_argument('--val_data_dir', '-v', default=WIDER_VAL_DIR, help='Validation dataset (WIDER_train)') parser.add_argument('--val_annotation', '-va', default=WIDER_VAL_ANNOTATION_MAT, help='Annotation file (.mat) for validation dataset') args = parser.parse_args() np.random.seed(args.seed) # for logging pocessed files logger = logging.getLogger('logger') logger.setLevel(logging.DEBUG) handler = logging.FileHandler(filename='filelog.log') handler.setLevel(logging.DEBUG) logger.addHandler(handler) blacklist = [] with open(BLACKLIST_FILE, 'r') as f: for line in f: l = line.strip() if l: blacklist.append(line.strip()) # train_data = VOCDetectionDataset(split='trainval', year='2007') # test_data = VOCDetectionDataset(split='test', year='2007', # use_difficult=True, return_difficult=True) train_data = WIDERFACEDataset(args.train_data_dir, args.train_annotation, logger=logger, exclude_file_list=blacklist) test_data = WIDERFACEDataset(args.val_data_dir, args.val_annotation) # faster_rcnn = FasterRCNNVGG16(n_fg_class=len(voc_detection_label_names), # pretrained_model='imagenet') faster_rcnn.use_preset('evaluate') model = FasterRCNNTrainChain(faster_rcnn) if args.gpu >= 0: model.to_gpu(args.gpu) chainer.cuda.get_device(args.gpu).use() optimizer = chainer.optimizers.MomentumSGD(lr=args.lr, momentum=0.9) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(rate=0.0005)) train_data = TransformDataset(train_data, transform) #import pdb; pdb.set_trace() #train_iter = chainer.iterators.MultiprocessIterator( # train_data, batch_size=1, n_processes=None, shared_mem=100000000) train_iter = chainer.iterators.SerialIterator( train_data, batch_size=1) test_iter = chainer.iterators.SerialIterator( test_data, batch_size=1, repeat=False, shuffle=False) updater = chainer.training.updater.StandardUpdater( train_iter, optimizer, device=args.gpu) trainer = training.Trainer( updater, (args.iteration, 'iteration'), out=args.out) trainer.extend( extensions.snapshot_object(model.faster_rcnn, 'snapshot_model.npz'), trigger=(args.iteration, 'iteration')) trainer.extend(extensions.ExponentialShift('lr', 0.1), trigger=(args.step_size, 'iteration')) log_interval = 20, 'iteration' plot_interval = 3000, 'iteration' print_interval = 20, 'iteration' trainer.extend(chainer.training.extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.PrintReport( ['iteration', 'epoch', 'elapsed_time', 'lr', 'main/loss', 'main/roi_loc_loss', 'main/roi_cls_loss', 'main/rpn_loc_loss', 'main/rpn_cls_loss', 'validation/main/map', ]), trigger=print_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) if extensions.PlotReport.available(): trainer.extend( extensions.PlotReport( ['main/loss'], file_name='loss.png', trigger=plot_interval ), trigger=plot_interval ) trainer.extend( DetectionVOCEvaluator( test_iter, model.faster_rcnn, use_07_metric=True, label_names=('face',)), trigger=ManualScheduleTrigger( [args.step_size, args.iteration], 'iteration'), invoke_before_training=False) trainer.extend(extensions.dump_graph('main/loss')) #try: # warnings.filterwarnings('error', category=RuntimeWarning) trainer.run()
def main(): parser = argparse.ArgumentParser( description='ChainerCV training example: Faster R-CNN') parser.add_argument('--dataset', choices=('voc07', 'voc0712'), help='The dataset to use: VOC07, VOC07+12', default='voc07') parser.add_argument('--gpu', '-g', type=int, default=-1) parser.add_argument('--lr', '-l', type=float, default=1e-3) parser.add_argument('--out', '-o', default='result', help='Output directory') parser.add_argument('--seed', '-s', type=int, default=0) parser.add_argument('--step_size', '-ss', type=int, default=50000) parser.add_argument('--iteration', '-i', type=int, default=70000) args = parser.parse_args() np.random.seed(args.seed) if args.dataset == 'voc07': train_data = VOCBboxDataset(split='trainval', year='2007') elif args.dataset == 'voc0712': train_data = ConcatenatedDataset( VOCBboxDataset(year='2007', split='trainval'), VOCBboxDataset(year='2012', split='trainval')) test_data = VOCBboxDataset(split='test', year='2007', use_difficult=True, return_difficult=True) faster_rcnn = FasterRCNNVGG16(n_fg_class=len(voc_bbox_label_names), pretrained_model='imagenet') faster_rcnn.use_preset('evaluate') model = FasterRCNNTrainChain(faster_rcnn) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() optimizer = chainer.optimizers.MomentumSGD(lr=args.lr, momentum=0.9) optimizer.setup(model) optimizer.add_hook(chainer.optimizer_hooks.WeightDecay(rate=0.0005)) train_data = TransformDataset(train_data, Transform(faster_rcnn)) train_iter = chainer.iterators.MultiprocessIterator(train_data, batch_size=1, n_processes=None, shared_mem=100000000) test_iter = chainer.iterators.SerialIterator(test_data, batch_size=1, repeat=False, shuffle=False) updater = chainer.training.updaters.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.iteration, 'iteration'), out=args.out) trainer.extend(extensions.snapshot_object(model.faster_rcnn, 'snapshot_model.npz'), trigger=(args.iteration, 'iteration')) trainer.extend(extensions.ExponentialShift('lr', 0.1), trigger=(args.step_size, 'iteration')) log_interval = 20, 'iteration' plot_interval = 3000, 'iteration' print_interval = 20, 'iteration' trainer.extend(chainer.training.extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.PrintReport([ 'iteration', 'epoch', 'elapsed_time', 'lr', 'main/loss', 'main/roi_loc_loss', 'main/roi_cls_loss', 'main/rpn_loc_loss', 'main/rpn_cls_loss', 'validation/main/map', ]), trigger=print_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) if extensions.PlotReport.available(): trainer.extend(extensions.PlotReport(['main/loss'], file_name='loss.png', trigger=plot_interval), trigger=plot_interval) trainer.extend(DetectionVOCEvaluator(test_iter, model.faster_rcnn, use_07_metric=True, label_names=voc_bbox_label_names), trigger=ManualScheduleTrigger( [args.step_size, args.iteration], 'iteration')) trainer.extend(extensions.dump_graph('main/loss')) trainer.run()
def handler(context): dataset_alias = context.datasets trainval_2007_dataset_id = dataset_alias['trainval2007'] trainval_2012_dataset_id = dataset_alias['trainval2012'] test_2007_dataset_id = dataset_alias['test2007'] trainval_2007_dataset = list( load_dataset_from_api(trainval_2007_dataset_id)) trainval_2012_dataset = list( load_dataset_from_api(trainval_2012_dataset_id)) test_2007_dataset = list(load_dataset_from_api(test_2007_dataset_id)) if network_model == 'ssd300': model = SSD300(n_fg_class=len(voc_bbox_label_names), pretrained_model='imagenet') elif network_model == 'ssd512': model = SSD512(n_fg_class=len(voc_bbox_label_names), pretrained_model='imagenet') model.use_preset('evaluate') train_chain = MultiboxTrainChain(model) if USE_GPU >= 0: chainer.cuda.get_device_from_id(USE_GPU).use() model.to_gpu() trainval_2007 = DetectionDatasetFromAPI(trainval_2007_dataset) trainval_2012 = DetectionDatasetFromAPI(trainval_2012_dataset) test_2007 = DetectionDatasetFromAPI(test_2007_dataset, use_difficult=True, return_difficult=True) train = TransformDataset(ConcatenatedDataset(trainval_2007, trainval_2012), Transform(model.coder, model.insize, model.mean)) train_iter = chainer.iterators.SerialIterator(train, BATCHSIZE) test_iter = chainer.iterators.SerialIterator(test_2007, BATCHSIZE, repeat=False, shuffle=False) # initial lr is set to 1e-3 by ExponentialShift optimizer = chainer.optimizers.MomentumSGD() optimizer.setup(train_chain) for param in train_chain.params(): if param.name == 'b': param.update_rule.add_hook(GradientScaling(2)) else: param.update_rule.add_hook(WeightDecay(0.0005)) updater = training.updaters.StandardUpdater(train_iter, optimizer, device=USE_GPU) trainer = training.Trainer(updater, (nb_iterations, 'iteration'), out=ABEJA_TRAINING_RESULT_DIR) trainer.extend(extensions.ExponentialShift('lr', 0.1, init=1e-3), trigger=triggers.ManualScheduleTrigger([80000, 100000], 'iteration')) trainer.extend(DetectionVOCEvaluator(test_iter, model, use_07_metric=True, label_names=voc_bbox_label_names), trigger=(10000, 'iteration')) log_interval = 100, 'iteration' trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) print_entries = [ 'iteration', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map' ] report_entries = [ 'epoch', 'iteration', 'lr', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map' ] trainer.extend(Statistics(report_entries, nb_iterations, obs_key='iteration'), trigger=log_interval) trainer.extend(Tensorboard(report_entries, out_dir=log_path)) trainer.extend(extensions.PrintReport(print_entries), trigger=log_interval) trainer.extend(extensions.snapshot_object( model, 'model_iter_{.updater.iteration}'), trigger=(nb_iterations, 'iteration')) trainer.run()
def main(): bbox_label_names = ('loop') n_itrs = 70000 n_step = 50000 np.random.seed(0) train_data = DefectDetectionDataset(split='train') test_data = DefectDetectionDataset(split='test') proposal_params = {'min_size': 8} faster_rcnn = FasterRCNNVGG16(n_fg_class=1, pretrained_model='imagenet', ratios=[0.5, 1, 2], anchor_scales=[1, 4, 8, 16], min_size=512, max_size=1024, proposal_creator_params=proposal_params) faster_rcnn.use_preset('evaluate') model = FasterRCNNTrainChain(faster_rcnn) chainer.cuda.get_device_from_id(0).use() model.to_gpu() optimizer = chainer.optimizers.MomentumSGD(lr=1e-3, momentum=0.9) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(rate=0.0005)) train_data = TransformDataset(train_data, Transform(faster_rcnn)) train_iter = chainer.iterators.MultiprocessIterator( train_data, batch_size=1, n_processes=None, shared_mem=100000000) test_iter = chainer.iterators.SerialIterator( test_data, batch_size=1, repeat=False, shuffle=False) updater = chainer.training.updater.StandardUpdater( train_iter, optimizer, device=0) trainer = training.Trainer( updater, (n_itrs, 'iteration'), out='result') trainer.extend( extensions.snapshot_object(model.faster_rcnn, 'snapshot_model_{.updater.iteration}.npz'), trigger=(n_itrs/5, 'iteration')) trainer.extend(extensions.ExponentialShift('lr', 0.1), trigger=(n_step, 'iteration')) log_interval = 50, 'iteration' plot_interval = 100, 'iteration' print_interval = 20, 'iteration' trainer.extend(chainer.training.extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.PrintReport( ['iteration', 'epoch', 'elapsed_time', 'lr', 'main/loss', 'main/roi_loc_loss', 'main/roi_cls_loss', 'main/rpn_loc_loss', 'main/rpn_cls_loss', 'validation/main/map', ]), trigger=print_interval) trainer.extend(extensions.ProgressBar(update_interval=5)) if extensions.PlotReport.available(): trainer.extend( extensions.PlotReport( ['main/loss'], file_name='loss.png', trigger=plot_interval ), trigger=plot_interval ) trainer.extend( DetectionVOCEvaluator( test_iter, model.faster_rcnn, use_07_metric=True, label_names=bbox_label_names), trigger=ManualScheduleTrigger( [100, 500, 1000, 5000, 10000, 20000, 40000, 60000, n_step, n_itrs], 'iteration')) trainer.extend(extensions.dump_graph('main/loss')) trainer.run()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--gpu', type=int, default=-1) parser.add_argument('--batchsize', type=int, default=4) parser.add_argument('--epoch', type=int, default=10) parser.add_argument('--mini', action="store_true") args = parser.parse_args() if hasattr(multiprocessing, 'set_start_method'): multiprocessing.set_start_method('forkserver') p = multiprocessing.Process() p.start() p.join() comm = chainermn.create_communicator('pure_nccl') print(comm.size) device = comm.intra_rank num_class = len(voc_bbox_label_names) data_augmentation_transform = DataAugmentationTransform(512) center_detection_transform = CenterDetectionTransform(512, num_class, 4) train = TransformDataset( ConcatenatedDataset(VOCBboxDataset(year='2007', split='trainval'), VOCBboxDataset(year='2012', split='trainval')), data_augmentation_transform) if comm.rank == 0: train = TransformDataset(train, center_detection_transform) if args.mini: train = datasets.SubDataset(train, 0, 100) else: train = None train = chainermn.scatter_dataset(train, comm, shuffle=True) train_iter = chainer.iterators.MultiprocessIterator(train, args.batchsize // comm.size, n_processes=2) if comm.rank == 0: test = VOCBboxDataset(year='2007', split='test', use_difficult=True, return_difficult=True) if args.mini: test = datasets.SubDataset(test, 0, 20) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) detector = CenterDetector(HourglassNet, 512, num_class) train_chain = CenterDetectorTrain(detector, 1, 0.1, 1, comm=comm) chainer.cuda.get_device_from_id(device).use() train_chain.to_gpu() optimizer = chainermn.create_multi_node_optimizer(Adam(amsgrad=True), comm) optimizer.setup(train_chain) updater = StandardUpdater(train_iter, optimizer, device=device) trainer = Trainer(updater, (args.epoch, 'epoch')) if comm.rank == 0: log_interval = 1, 'epoch' trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport([ 'epoch', 'iteration', 'lr', 'main/loss', 'main/hm_loss', 'main/wh_loss', 'main/offset_loss', 'validation/main/map', ]), trigger=log_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.extend(DetectionVOCEvaluator(test_iter, detector, use_07_metric=True, label_names=voc_bbox_label_names), trigger=(1, 'epoch')) trainer.extend(extensions.snapshot_object( detector, 'detector{.updator.epoch:03}.npz'), trigger=(1, 'epoch')) trainer.run()
def main(): parser = argparse.ArgumentParser() parser.add_argument('dataset', help="path to train json file") parser.add_argument('test_dataset', help="path to test dataset json file") parser.add_argument( '--dataset-root', help= "path to dataset root if dataset file is not already in root folder of dataset" ) parser.add_argument('--model', choices=('ssd300', 'ssd512'), default='ssd512') parser.add_argument('--batchsize', type=int, default=32) parser.add_argument('--gpu', type=int, nargs='*', default=[]) parser.add_argument('--out', default='result') parser.add_argument('--resume') parser.add_argument('--lr', type=float, default=0.001, help="default learning rate") parser.add_argument('--port', type=int, default=1337, help="port for bbox sending") parser.add_argument('--ip', default='127.0.0.1', help="destination ip for bbox sending") parser.add_argument( '--test-image', help="path to test image that shall be displayed in bbox vis") args = parser.parse_args() if args.dataset_root is None: args.dataset_root = os.path.dirname(args.dataset) if args.model == 'ssd300': model = SSD300(n_fg_class=1, pretrained_model='imagenet') image_size = (300, 300) elif args.model == 'ssd512': model = SSD512(n_fg_class=1, pretrained_model='imagenet') image_size = (512, 512) else: raise NotImplementedError("The model you want to train does not exist") model.use_preset('evaluate') train_chain = MultiboxTrainChain(model) train = TransformDataset( SheepDataset(args.dataset_root, args.dataset, image_size=image_size), Transform(model.coder, model.insize, model.mean)) if len(args.gpu) > 1: gpu_datasets = split_dataset_n_random(train, len(args.gpu)) if not len(gpu_datasets[0]) == len(gpu_datasets[-1]): adapted_second_split = split_dataset(gpu_datasets[-1], len(gpu_datasets[0]))[0] gpu_datasets[-1] = adapted_second_split else: gpu_datasets = [train] train_iter = [ ThreadIterator(gpu_dataset, args.batchsize) for gpu_dataset in gpu_datasets ] test = SheepDataset(args.dataset_root, args.test_dataset, image_size=image_size) test_iter = chainer.iterators.MultithreadIterator(test, args.batchsize, repeat=False, shuffle=False, n_threads=2) # initial lr is set to 1e-3 by ExponentialShift optimizer = chainer.optimizers.Adam(alpha=args.lr) optimizer.setup(train_chain) for param in train_chain.params(): if param.name == 'b': param.update_rule.add_hook(GradientScaling(2)) else: param.update_rule.add_hook(WeightDecay(0.0005)) if len(args.gpu) <= 1: updater = training.updaters.StandardUpdater( train_iter[0], optimizer, device=args.gpu[0] if len(args.gpu) > 0 else -1, ) else: updater = training.updaters.MultiprocessParallelUpdater( train_iter, optimizer, devices=args.gpu) updater.setup_workers() if len(args.gpu) > 0 and args.gpu[0] >= 0: chainer.backends.cuda.get_device_from_id(args.gpu[0]).use() model.to_gpu() trainer = training.Trainer(updater, (200, 'epoch'), args.out) trainer.extend(DetectionVOCEvaluator(test_iter, model, use_07_metric=True, label_names=voc_bbox_label_names), trigger=(1000, 'iteration')) # build logger # make sure to log all data necessary for prediction log_interval = 100, 'iteration' data_to_log = { 'image_size': image_size, 'model_type': args.model, } # add all command line arguments for argument in filter(lambda x: not x.startswith('_'), dir(args)): data_to_log[argument] = getattr(args, argument) # create callback that logs all auxiliary data the first time things get logged def backup_train_config(stats_cpu): if stats_cpu['iteration'] == log_interval: stats_cpu.update(data_to_log) trainer.extend( extensions.LogReport(trigger=log_interval, postprocess=backup_train_config)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport([ 'epoch', 'iteration', 'lr', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map' ]), trigger=log_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.extend(extensions.snapshot_object( model, 'model_iter_{.updater.iteration}'), trigger=(5000, 'iteration')) if args.test_image is not None: plot_image = train._dataset.load_image(args.test_image, resize_to=image_size) else: plot_image, _, _ = train.get_example(0) plot_image += train._transform.mean bbox_plotter = BBOXPlotter( plot_image, os.path.join(args.out, 'bboxes'), send_bboxes=True, upstream_port=args.port, upstream_ip=args.ip, ) trainer.extend(bbox_plotter, trigger=(10, 'iteration')) if args.resume: serializers.load_npz(args.resume, trainer) trainer.run()
def handler(context): dataset_alias = context.datasets data = list(load_dataset_from_api(dataset_alias['train'])) np.random.seed(0) data = np.random.permutation(data) nb_data = len(data) nb_train = int(7 * nb_data // 10) train_data_raw = data[:nb_train] test_data_raw = data[nb_train:] premodel = SSD300(n_fg_class=20, pretrained_model='voc0712') model = SSD300(n_fg_class=1) copy_ssd(model, premodel) model.use_preset('evaluate') train_chain = MultiboxTrainChain(model) if USE_GPU >= 0: chainer.cuda.get_device_from_id(USE_GPU).use() model.to_gpu() # initial lr is set to 1e-3 by ExponentialShift optimizer = chainer.optimizers.MomentumSGD() optimizer.setup(train_chain) for param in train_chain.params(): if param.name == 'b': param.update_rule.add_hook(GradientScaling(2)) else: param.update_rule.add_hook(WeightDecay(0.0005)) fix_ssd(train_chain) train_data = DetectionDatasetFromAPI(train_data_raw) test_data = DetectionDatasetFromAPI(test_data_raw, use_difficult=True, return_difficult=True) train_data = TransformDataset( train_data, Transform(model.coder, model.insize, model.mean)) train_iter = chainer.iterators.SerialIterator(train_data, BATCHSIZE) test_iter = chainer.iterators.SerialIterator(test_data, BATCHSIZE, repeat=False, shuffle=False) updater = training.updaters.StandardUpdater(train_iter, optimizer, device=USE_GPU) trainer = training.Trainer(updater, (nb_epochs, 'epoch'), out=ABEJA_TRAINING_RESULT_DIR) trainer.extend(extensions.ExponentialShift('lr', 0.1, init=1e-3), trigger=triggers.ManualScheduleTrigger([1200, 1600], 'epoch')) trainer.extend(DetectionVOCEvaluator(test_iter, model, use_07_metric=True, label_names=['cup']), trigger=(1, 'epoch')) log_interval = 1, 'epoch' trainer.extend(extensions.LogReport(trigger=log_interval)) print_entries = [ 'epoch', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map' ] report_entries = [ 'epoch', 'lr', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map' ] trainer.extend(Statistics(report_entries, nb_epochs), trigger=log_interval) trainer.extend(Tensorboard(report_entries, out_dir=log_path)) trainer.extend(extensions.PrintReport(print_entries), trigger=log_interval) trainer.extend(extensions.snapshot_object(model, 'model_epoch_{.updater.epoch}'), trigger=(nb_epochs, 'epoch')) trainer.run()
else: param.update_rule.add_hook(WeightDecay(0.0005)) # 最適化関数とトレーニングデータセットを入力 # 学習部分ににoptimizerを繋げる. updater = training.updaters.StandardUpdater(train_iter, optimizer, device=gpu) trainer = training.Trainer(updater, (iteration, 'iteration'), out) # 指数関数的に学習率を変更できるように設定 trainer.extend(extensions.ExponentialShift('lr', 0.1, init=1e-3), trigger=triggers.ManualScheduleTrigger(step, 'iteration')) # 学習とともにAPやmAPを評価するように設定 trainer.extend( DetectionVOCEvaluator(test_iter, model, use_07_metric=True, label_names=(voc_bbox_label_names + ('new_label', )))) log_interval = 10, 'iteration' # 評価結果の出力 trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport([ 'epoch', 'iteration', 'lr', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map' ]), trigger=log_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) # 結果の表示
def main(): rospack = rospkg.RosPack() jsk_perception_datasets_path = osp.join( rospack.get_path('jsk_perception'), 'learning_datasets') parser = argparse.ArgumentParser() # Dataset directory parser.add_argument('--train-dataset-dir', type=str, default=osp.join(jsk_perception_datasets_path, 'kitchen_dataset', 'train')) parser.add_argument('--val-dataset-dir', type=str, default=osp.join(jsk_perception_datasets_path, 'kitchen_dataset', 'test')) parser.add_argument('--dataset-type', type=str, default='instance') parser.add_argument( '--model-name', choices=('ssd300', 'ssd512'), default='ssd512') parser.add_argument('--gpu', type=int, default=0) parser.add_argument('--batch-size', type=int, default=8) parser.add_argument('--max-epoch', type=int, default=100) parser.add_argument('--out-dir', type=str, default=None) args = parser.parse_args() if (args.dataset_type == 'instance'): train_dataset = DetectionDataset(args.train_dataset_dir) elif (args.dataset_type == 'bbox'): train_dataset = BboxDetectionDataset(args.train_dataset_dir) else: print('unsuppported dataset type') return fg_label_names = train_dataset.fg_class_names if args.model_name == 'ssd300': model = SSD300( n_fg_class=len(fg_label_names), pretrained_model='imagenet') elif args.model_name == 'ssd512': model = SSD512( n_fg_class=len(fg_label_names), pretrained_model='imagenet') model.use_preset('evaluate') train_chain = MultiboxTrainChain(model) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() train = TransformDataset( train_dataset, Transform(model.coder, model.insize, model.mean)) train_iter = chainer.iterators.MultiprocessIterator(train, args.batch_size) if (args.dataset_type == 'instance'): test_dataset = DetectionDataset(args.val_dataset_dir) elif (args.dataset_type == 'bbox'): test_dataset = BboxDetectionDataset(args.val_dataset_dir) test_iter = chainer.iterators.SerialIterator( test_dataset, args.batch_size, repeat=False, shuffle=False) # initial lr is set to 1e-3 by ExponentialShift optimizer = chainer.optimizers.MomentumSGD() optimizer.setup(train_chain) for param in train_chain.params(): if param.name == 'b': param.update_rule.add_hook(GradientScaling(2)) else: param.update_rule.add_hook(WeightDecay(0.0005)) updater = training.updaters.StandardUpdater( train_iter, optimizer, device=args.gpu) now = datetime.datetime.now() timestamp = now.strftime('%Y%m%d-%H%M%S') if args.out_dir is None: out_dir = osp.join( rospkg.get_ros_home(), 'learning_logs', timestamp) else: out_dir = args.out_dir step_epoch = [args.max_epoch * 2 // 3, args.max_epoch * 5 // 6] trainer = training.Trainer( updater, (args.max_epoch, 'epoch'), out_dir) trainer.extend( extensions.ExponentialShift('lr', 0.1, init=1e-3), trigger=triggers.ManualScheduleTrigger(step_epoch, 'epoch')) trainer.extend( DetectionVOCEvaluator( test_iter, model, use_07_metric=True, label_names=fg_label_names), trigger=triggers.ManualScheduleTrigger( step_epoch + [args.max_epoch], 'epoch')) log_interval = 10, 'iteration' trainer.extend( extensions.LogReport(log_name='log.json', trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport( ['epoch', 'iteration', 'lr', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map']), trigger=log_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.extend( extensions.snapshot_object( model, 'model_snapshot.npz'), trigger=(args.max_epoch, 'epoch')) trainer.run()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--data-dir', type=str, default=os.path.join("RoadDamageDataset", "All")) parser.add_argument('--batchsize', type=int, default=32, help='Learning minibatch size') parser.add_argument('--gpu', type=int, default=-1, help='GPU ID (negative value indicates CPU') parser.add_argument('--base-network', choices=('vgg16', 'resnet101'), default='vgg16', help='Base network') parser.add_argument('--pretrained-model', default=None, help='Pretrained SSD model') parser.add_argument('--pretrained-extractor', default='auto', help='Pretrained CNN model to extract feature maps') parser.add_argument('--out', default='result-detection', help='Directory to output the result') parser.add_argument('--resume', default=None, help='Initialize the trainer from given file') args = parser.parse_args() print("Data directory : {}".format(args.data_dir)) print("Batchsize : {}".format(args.batchsize)) print("GPU ID : {}".format(args.gpu)) print("Base network : {}".format(args.base_network)) print("Pretrained extractor : {}".format(args.pretrained_extractor)) print("Pretrained model : {}".format(args.pretrained_model)) print("Output directory : {}".format(args.out)) print("Resume from : {}".format(args.resume)) if args.base_network == 'vgg16': # pretrained_extractor is currently not available for this class model = chainercv.links.SSD300( n_fg_class=len(roaddamage_label_names), pretrained_model=args.pretrained_model) preprocessing = MeanSubtraction(model.mean) elif args.base_network == 'resnet101': model = ssd_resnet101.SSD224( n_fg_class=len(roaddamage_label_names), pretrained_extractor=args.pretrained_extractor, pretrained_model=args.pretrained_model) preprocessing = ResNetPreparation(model.insize) else: raise ValueError('Invalid base network') model.use_preset('evaluate') train_chain = MultiboxTrainChain(model) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() train = TransformDataset( RoadDamageDataset(args.data_dir, split='train'), Transform(model.coder, model.insize, model.mean) ) train = TransformDataset(train, preprocessing) train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test = RoadDamageDataset(args.data_dir, split='val') test_iter = chainer.iterators.SerialIterator( test, args.batchsize, repeat=False, shuffle=False) # initial lr is set to 3e-4 by ExponentialShift optimizer = chainer.optimizers.MomentumSGD() optimizer.setup(train_chain) for param in train_chain.params(): if param.name == 'b': param.update_rule.add_hook(GradientScaling(2)) else: param.update_rule.add_hook(WeightDecay(0.0005)) updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (120000, 'iteration'), args.out) trainer.extend( extensions.ExponentialShift('lr', 0.1, init=3e-4), trigger=triggers.ManualScheduleTrigger([80000, 100000], 'iteration')) trainer.extend( DetectionVOCEvaluator( test_iter, model, use_07_metric=True, label_names=roaddamage_label_names), trigger=(4000, 'iteration')) log_interval = 10, 'iteration' trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport( ['epoch', 'iteration', 'lr', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map']), trigger=log_interval) # trainer.extend(extensions.ProgressBar()) trainer.extend(extensions.snapshot(), trigger=(4000, 'iteration')) trainer.extend( extensions.snapshot_object(model, 'model_iter_{.updater.iteration}'), trigger=(4000, 'iteration')) if args.resume: serializers.load_npz(args.resume, trainer) print("setup finished") trainer.run() model.to_cpu() serializers.save_npz("model-detector.npz", model)
def main(): # cuDNNのautotuneを有効にする chainer.cuda.set_max_workspace_size(512 * 1024 * 1024) chainer.config.autotune = True gpu_id = 0 batchsize = 6 out_num = 'results' log_interval = 1, 'epoch' epoch_max = 500 initial_lr = 0.0001 lr_decay_rate = 0.1 lr_decay_timing = [200, 300, 400] # モデルの設定 model = SSD300(n_fg_class=len(voc_labels), pretrained_model='imagenet') model.use_preset('evaluate') train_chain = MultiboxTrainChain(model) # GPUの設定 chainer.cuda.get_device_from_id(gpu_id).use() model.to_gpu() # データセットの設定 train_dataset = MyVoTTVOCDataset( 'C:\Python_Programs\chainer_practice\Telescope_corner', 'train') valid_dataset = MyVoTTVOCDataset( 'C:\Python_Programs\chainer_practice\Telescope_corner', 'val') # データ拡張 transformed_train_dataset = TransformDataset( train_dataset, Transform(model.coder, model.insize, model.mean)) # イテレーターの設定 train_iter = chainer.iterators.MultiprocessIterator( transformed_train_dataset, batchsize) valid_iter = chainer.iterators.SerialIterator(valid_dataset, batchsize, repeat=False, shuffle=False) # オプティマイザーの設定 optimizer = chainer.optimizers.MomentumSGD() optimizer.setup(train_chain) for param in train_chain.params(): if param.name == 'b': param.update_rule.add_hook(GradientScaling(2)) else: param.update_rule.add_hook(WeightDecay(0.0005)) # アップデーターの設定 updater = training.updaters.StandardUpdater(train_iter, optimizer, device=gpu_id) # トレーナーの設定 trainer = training.Trainer(updater, (epoch_max, 'epoch'), out_num) trainer.extend(extensions.ExponentialShift('lr', lr_decay_rate, init=initial_lr), trigger=triggers.ManualScheduleTrigger( lr_decay_timing, 'epoch')) trainer.extend(DetectionVOCEvaluator(valid_iter, model, use_07_metric=False, label_names=voc_labels), trigger=(1, 'epoch')) trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport([ 'epoch', 'iteration', 'lr', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map', 'elapsed_time' ]), trigger=log_interval) if extensions.PlotReport.available(): trainer.extend( extensions.PlotReport( ['main/loss', 'main/loss/loc', 'main/loss/conf'], 'epoch', file_name='loss.png')) trainer.extend( extensions.PlotReport(['validation/main/map'], 'epoch', file_name='accuracy.png')) trainer.extend( extensions.snapshot(filename='snapshot_epoch_{.updater.epoch}.npz'), trigger=(10, 'epoch')) # 途中で止めた学習を再開する場合は、trainerにスナップショットをロードして再開する # serializers.load_npz('results/snapshot_epoch_100.npz', trainer) # 学習実行 trainer.run() # 学習データの保存 model.to_cpu() serializers.save_npz('my_ssd_model.npz', model)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--gpus', type=str, default="-1") parser.add_argument('--batchsize', type=int, default=2) parser.add_argument('--epoch', type=int, default=10) parser.add_argument('--mini', action="store_true") args = parser.parse_args() gpus = list(filter(lambda x: x >= 0, map(int, args.gpus.split(",")))) num_class = len(voc_bbox_label_names) data_augmentation_transform = DataAugmentationTransform(512) center_detection_transform = CenterDetectionTransform(512, num_class, 4) train = TransformDataset( ConcatenatedDataset( VOCBboxDataset(year='2007', split='trainval'), VOCBboxDataset(year='2012', split='trainval') ), data_augmentation_transform ) train = TransformDataset(train, center_detection_transform) if args.mini: train = datasets.SubDataset(train, 0, 100) train_iter = chainer.iterators.MultiprocessIterator(train, args.batchsize) test = VOCBboxDataset( year='2007', split='test', use_difficult=True, return_difficult=True) if args.mini: test = datasets.SubDataset(test, 0, 20) test_iter = chainer.iterators.SerialIterator( test, args.batchsize // len(gpus), repeat=False, shuffle=False) detector = CenterDetector(HourglassNet, 512, num_class) train_chain = CenterDetectorTrain(detector, 1, 0.1, 1) gpus.sort() first_gpu = gpus[0] remain_gpu = gpus[1:] train_chain.to_gpu(first_gpu) optimizer = Adam(amsgrad=True) optimizer.setup(train_chain) devices = { "main": first_gpu } for i, gpu in enumerate(remain_gpu): devices[f"{i + 2}"] = gpu updater = training.updaters.ParallelUpdater( train_iter, optimizer, devices=devices, ) log_interval = 1, 'epoch' trainer = Trainer(updater, (args.epoch, 'epoch')) trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport( [ 'epoch', 'iteration', 'lr', 'main/loss', 'main/hm_loss', 'main/wh_loss', 'main/offset_loss', 'validation/main/map', ]), trigger=log_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.extend( DetectionVOCEvaluator( test_iter, detector, use_07_metric=True, label_names=voc_bbox_label_names), trigger=(1, 'epoch')) trainer.extend( extensions.snapshot_object(detector, 'detector{.updater.epoch:03}.npz'), trigger=(1, 'epoch') ) trainer.run()
return loss train_chain = MultiboxTrainChain(model) optimizer = chainer.optimizers.MomentumSGD(1e-3) optimizer.setup(train_chain) for param in train_chain.params(): if param.name == 'b': param.update_rule.add_hook(GradientScaling(2)) else: param.update_rule.add_hook(WeightDecay(0.0005)) evaluator = DetectionVOCEvaluator( test_iter, model, use_07_metric=True, label_names=voc_bbox_label_names, ) class save_model(training.Extension): #ver 1.2 def __init__(self, model, save_name, before_iter=0, saved_dir='saved_model/', save_after=0): self.model = model self.save_name = save_name self.saved_dir = saved_dir self.before_iter = before_iter
def main(): parser = argparse.ArgumentParser() parser.add_argument('--model', choices=('ssd300', 'ssd512'), default='ssd300') parser.add_argument('--batchsize', type=int, default=32) parser.add_argument('--test-batchsize', type=int, default=16) parser.add_argument('--iteration', type=int, default=120000) parser.add_argument('--step', type=int, nargs='*', default=[80000, 100000]) parser.add_argument('--gpu', type=int, default=-1) parser.add_argument('--out', default='result') parser.add_argument('--resume') parser.add_argument('--dtype', type=str, choices=dtypes.keys(), default='float32', help='Select the data type of the model') parser.add_argument('--model-dir', default=None, type=str, help='Where to store models') parser.add_argument('--dataset-dir', default=None, type=str, help='Where to store datasets') parser.add_argument('--dynamic-interval', default=None, type=int, help='Interval for dynamic loss scaling') parser.add_argument('--init-scale', default=1, type=float, help='Initial scale for ada loss') parser.add_argument('--loss-scale-method', default='approx_range', type=str, help='Method for adaptive loss scaling') parser.add_argument('--scale-upper-bound', default=32800, type=float, help='Hard upper bound for each scale factor') parser.add_argument('--accum-upper-bound', default=32800, type=float, help='Accumulated upper bound for all scale factors') parser.add_argument('--update-per-n-iteration', default=100, type=int, help='Update the loss scale value per n iteration') parser.add_argument('--snapshot-per-n-iteration', default=10000, type=int, help='The frequency of taking snapshots') parser.add_argument('--n-uf', default=1e-3, type=float) parser.add_argument('--nosanity-check', default=False, action='store_true') parser.add_argument('--nouse-fp32-update', default=False, action='store_true') parser.add_argument('--profiling', default=False, action='store_true') parser.add_argument('--verbose', action='store_true', default=False, help='Verbose output') args = parser.parse_args() # Setting data types if args.dtype != 'float32': chainer.global_config.use_cudnn = 'never' chainer.global_config.dtype = dtypes[args.dtype] print('==> Setting the data type to {}'.format(args.dtype)) # Initialize model if args.model == 'ssd300': model = SSD300(n_fg_class=len(voc_bbox_label_names), pretrained_model='imagenet') elif args.model == 'ssd512': model = SSD512(n_fg_class=len(voc_bbox_label_names), pretrained_model='imagenet') model.use_preset('evaluate') # Apply adaptive loss scaling recorder = AdaLossRecorder(sample_per_n_iter=100) profiler = Profiler() sanity_checker = SanityChecker(check_per_n_iter=100) if not args.nosanity_check else None # Update the model to support AdaLoss # TODO: refactorize model_ = AdaLossScaled( model, init_scale=args.init_scale, cfg={ 'loss_scale_method': args.loss_scale_method, 'scale_upper_bound': args.scale_upper_bound, 'accum_upper_bound': args.accum_upper_bound, 'update_per_n_iteration': args.update_per_n_iteration, 'recorder': recorder, 'profiler': profiler, 'sanity_checker': sanity_checker, 'n_uf_threshold': args.n_uf, }, transforms=[ AdaLossTransformLinear(), AdaLossTransformConvolution2D(), ], verbose=args.verbose) # Finalize the model train_chain = MultiboxTrainChain(model_) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() cp.random.seed(0) # NOTE: we have to transfer modules explicitly to GPU model.coder.to_gpu() model.extractor.to_gpu() model.multibox.to_gpu() # Prepare dataset if args.model_dir is not None: chainer.dataset.set_dataset_root(args.model_dir) train = TransformDataset( ConcatenatedDataset(VOCBboxDataset(year='2007', split='trainval'), VOCBboxDataset(year='2012', split='trainval')), Transform(model.coder, model.insize, model.mean, dtype=dtypes[args.dtype])) # train_iter = chainer.iterators.MultiprocessIterator( # train, args.batchsize) # , n_processes=8, n_prefetch=2) train_iter = chainer.iterators.MultithreadIterator(train, args.batchsize) # train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test = VOCBboxDataset(year='2007', split='test', use_difficult=True, return_difficult=True) test_iter = chainer.iterators.SerialIterator(test, args.test_batchsize, repeat=False, shuffle=False) # initial lr is set to 1e-3 by ExponentialShift optimizer = chainer.optimizers.MomentumSGD() if args.dtype == 'mixed16': if not args.nouse_fp32_update: print('==> Using FP32 update for dtype=mixed16') optimizer.use_fp32_update() # by default use fp32 update # HACK: support skipping update by existing loss scaling functionality if args.dynamic_interval is not None: optimizer.loss_scaling(interval=args.dynamic_interval, scale=None) else: optimizer.loss_scaling(interval=float('inf'), scale=None) optimizer._loss_scale_max = 1.0 # to prevent actual loss scaling optimizer.setup(train_chain) for param in train_chain.params(): if param.name == 'b': param.update_rule.add_hook(GradientScaling(2)) else: param.update_rule.add_hook(WeightDecay(0.0005)) updater = training.updaters.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.iteration, 'iteration'), args.out) trainer.extend(extensions.ExponentialShift('lr', 0.1, init=1e-3), trigger=triggers.ManualScheduleTrigger( args.step, 'iteration')) trainer.extend(DetectionVOCEvaluator(test_iter, model, use_07_metric=True, label_names=voc_bbox_label_names), trigger=triggers.ManualScheduleTrigger( args.step + [args.iteration], 'iteration')) log_interval = 10, 'iteration' trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.observe_value( 'loss_scale', lambda trainer: trainer.updater.get_optimizer('main')._loss_scale), trigger=log_interval) metrics = [ 'epoch', 'iteration', 'lr', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map' ] if args.dynamic_interval is not None: metrics.insert(2, 'loss_scale') trainer.extend(extensions.PrintReport(metrics), trigger=log_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.extend(extensions.snapshot(), trigger=triggers.ManualScheduleTrigger( args.step + [args.iteration], 'iteration')) trainer.extend(extensions.snapshot_object( model, 'model_iter_{.updater.iteration}'), trigger=(args.iteration, 'iteration')) if args.resume: serializers.load_npz(args.resume, trainer) hook = AdaLossMonitor(sample_per_n_iter=100, verbose=args.verbose, includes=['Grad', 'Deconvolution']) recorder.trainer = trainer hook.trainer = trainer with ExitStack() as stack: stack.enter_context(hook) trainer.run() recorder.export().to_csv(os.path.join(args.out, 'loss_scale.csv')) profiler.export().to_csv(os.path.join(args.out, 'profile.csv')) if sanity_checker: sanity_checker.export().to_csv(os.path.join(args.out, 'sanity_check.csv')) hook.export_history().to_csv(os.path.join(args.out, 'grad_stats.csv'))
def run(input_dir, test_dir, output, batch_size, iterator='SerialIterator', device=-1, pretrained_model='', save_trigger=10000, test_trigger=1000, parser_module='XMLParser', train_module='MultiboxTrainChain', model_module='chainercv.links.SSD300'): pretrained_model = join(PROJECT_DIR, pretrained_model) if pretrained_model and os.path.isfile(pretrained_model): print('Pretrained model {} loaded.'.format(pretrained_model)) else: print('Pretrained model file not found, ' + 'using imagenet as default.') pretrained_model = 'imagenet' parser = _import_module('multibuildingdetector.parsers.{}', parser_module) model = _import_class(model_module)(n_fg_class=len(parser.LABEL_NAMES), pretrained_model=pretrained_model) model.use_preset('evaluate') train_chain = _import_class('multibuildingdetector.trainchains.{}' .format(train_module))(model) if device >= 0: chainer.cuda.get_device_from_id(device).use() model.to_gpu() train, test = load_train_test_set(input_dir, test_dir, parser) augmented_train = TransformDataset( train, ImageAugmentation(model.coder, model.insize, model.mean)) train_iter = getattr(chainer.iterators, iterator)(augmented_train, batch_size) optimizer = chainer.optimizers.Adam() optimizer.setup(train_chain) for param in train_chain.params(): if param.name == 'b': param.update_rule.add_hook(GradientScaling(2)) else: param.update_rule.add_hook(WeightDecay(0.0005)) if device is None: updater = chainer.training.StandardUpdater(train_iter, optimizer) else: updater = chainer.training.StandardUpdater(train_iter, optimizer, device=device) trainer = chainer.training.Trainer(updater, (120000, 'iteration'), output) log_fields = ['main/' + x for x in train_chain.loss_labels] if train_module == 'MultiboxTrainChain': test_iter = chainer.iterators.SerialIterator( test, batch_size, repeat=False, shuffle=False) trainer.extend( DetectionVOCEvaluator( test_iter, model, use_07_metric=True, label_names=parser.LABEL_NAMES), trigger=(test_trigger, 'iteration')) log_fields.append('validation/main/map') else: triplet_test = TransformDataset( test, ImageAugmentation(model.coder, model.insize, model.mean, augment=False)) test_iter = chainer.iterators.SerialIterator( triplet_test, batch_size, repeat=False, shuffle=False) trainer.extend( TripletEvaluator( test_iter, model, label_names=parser.LABEL_NAMES, save_plt=True, save_path=output), trigger=(test_trigger, 'iteration')) log_interval = 10, 'iteration' trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport( ['epoch', 'iteration', 'lr', *log_fields])) trainer.extend(extensions.snapshot_object( model, 'model_iter_{.updater.iteration}'), trigger=(save_trigger, 'iteration')) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.run()
optimizer = chainer.optimizers.MomentumSGD(lr=args.lr) optimizer.setup(train_chain) for param in train_chain.params(): if param.name == 'b': param.update_rule.add_hook(GradientScaling(2)) else: param.update_rule.add_hook(WeightDecay(0.0005)) updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.iteration, "iteration"), args.out) val_interval = args.val_iter, "iteration" trainer.extend(DetectionVOCEvaluator(test_iter, model, use_07_metric=True, label_names=label_names), trigger=val_interval) log_interval = args.log_iter, "iteration" trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport([ 'epoch', 'iteration', 'lr', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map' ]), trigger=log_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.extend(extensions.snapshot(), trigger=val_interval) trainer.extend(extensions.snapshot_object(
chainer.cuda.get_device_from_id(0).use() model.to_gpu() # Data Augmentation train = TransformDataset( ConcatenatedDataset( VOCBboxDataset(year='2007', split='trainval'), VOCBboxDataset(year='2012', split='trainval') ), Transform(model.coder, model.insize, model.mean) ) test = VOCBboxDataset(year='2007', split='test', use_difficult=True, return_difficult=True) train_iter = iterators.SerialIterator(train, args.batchsize) test_iter = iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) # Evaluator evaluator = DetectionVOCEvaluator(test_iter, net.predictor, use_07_metric=True, label_names=voc_bbox_label_names) trainer = create_trainer(train_iter, net, args.gpu_id, args.initial_lr, args.weight_decay, args.freeze_layer, args.small_lr_layers, args.small_initial_lr, args.num_epochs_or_iter, args.epoch_or_iter, args.save_dir) if args.load_path: chainer.serializers.load_npz(args.load_path, trainer) trainer_extend(trainer, net, evaluator, args.small_lr_layers, args.lr_decay_rate, args.lr_decay_epoch, args.epoch_or_iter, args.save_trainer_interval) trainer.run()
param.update_rule.add_hook(WeightDecay(0.0005)) updater = training.updaters.StandardUpdater( train_iter, optimizer, device=gpu_id) trainer = training.Trainer( updater, (training_epoch, 'epoch'), out) trainer.extend( extensions.ExponentialShift('lr', lr_decay_rate, init=initial_lr), trigger=triggers.ManualScheduleTrigger(lr_decay_timing, 'epoch')) trainer.extend( DetectionVOCEvaluator( valid_iter, model, use_07_metric=False, label_names=bball_labels), trigger=(1, 'epoch')) trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport( ['epoch', 'iteration', 'lr', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map', 'elapsed_time']), trigger=log_interval) if extensions.PlotReport.available(): trainer.extend( extensions.PlotReport( ['main/loss', 'main/loss/loc', 'main/loss/conf'], 'epoch', file_name='loss.png')) trainer.extend(
def main(): parser = argparse.ArgumentParser() parser.add_argument('--model', choices=('ssd300', 'ssd512'), default='ssd300') parser.add_argument('--batchsize', type=int, default=32) parser.add_argument('--np', type=int, default=8) parser.add_argument('--test-batchsize', type=int, default=16) parser.add_argument('--iteration', type=int, default=120000) parser.add_argument('--step', type=int, nargs='*', default=[80000, 100000]) parser.add_argument('--lr', type=float, default=1e-3) parser.add_argument('--out', default='result') parser.add_argument('--resume') parser.add_argument('--dtype', type=str, choices=dtypes.keys(), default='float32', help='Select the data type of the model') parser.add_argument('--model-dir', default=None, type=str, help='Where to store models') parser.add_argument('--dataset-dir', default=None, type=str, help='Where to store datasets') parser.add_argument('--dynamic-interval', default=None, type=int, help='Interval for dynamic loss scaling') parser.add_argument('--init-scale', default=1, type=float, help='Initial scale for ada loss') parser.add_argument('--loss-scale-method', default='approx_range', type=str, help='Method for adaptive loss scaling') parser.add_argument('--scale-upper-bound', default=16, type=float, help='Hard upper bound for each scale factor') parser.add_argument('--accum-upper-bound', default=1024, type=float, help='Accumulated upper bound for all scale factors') parser.add_argument('--update-per-n-iteration', default=1, type=int, help='Update the loss scale value per n iteration') parser.add_argument('--snapshot-per-n-iteration', default=10000, type=int, help='The frequency of taking snapshots') parser.add_argument('--n-uf', default=1e-3, type=float) parser.add_argument('--nosanity-check', default=False, action='store_true') parser.add_argument('--nouse-fp32-update', default=False, action='store_true') parser.add_argument('--profiling', default=False, action='store_true') parser.add_argument('--verbose', action='store_true', default=False, help='Verbose output') args = parser.parse_args() # https://docs.chainer.org/en/stable/chainermn/tutorial/tips_faqs.html#using-multiprocessiterator if hasattr(multiprocessing, 'set_start_method'): multiprocessing.set_start_method('forkserver') p = multiprocessing.Process() p.start() p.join() comm = chainermn.create_communicator('pure_nccl') device = comm.intra_rank # Set up workspace # 12 GB GPU RAM for workspace chainer.cuda.set_max_workspace_size(16 * 1024 * 1024 * 1024) chainer.global_config.cv_resize_backend = 'cv2' # Setup the data type # when initializing models as follows, their data types will be casted. # Weethave to forbid the usage of cudnn if args.dtype != 'float32': chainer.global_config.use_cudnn = 'never' chainer.global_config.dtype = dtypes[args.dtype] print('==> Setting the data type to {}'.format(args.dtype)) if args.model_dir is not None: chainer.dataset.set_dataset_root(args.model_dir) if args.model == 'ssd300': model = SSD300(n_fg_class=len(voc_bbox_label_names), pretrained_model='imagenet') elif args.model == 'ssd512': model = SSD512(n_fg_class=len(voc_bbox_label_names), pretrained_model='imagenet') model.use_preset('evaluate') ###################################### # Setup model ####################################### # Apply ada loss transform recorder = AdaLossRecorder(sample_per_n_iter=100) profiler = Profiler() sanity_checker = SanityChecker( check_per_n_iter=100) if not args.nosanity_check else None # Update the model to support AdaLoss # TODO: refactorize model_ = AdaLossScaled( model, init_scale=args.init_scale, cfg={ 'loss_scale_method': args.loss_scale_method, 'scale_upper_bound': args.scale_upper_bound, 'accum_upper_bound': args.accum_upper_bound, 'update_per_n_iteration': args.update_per_n_iteration, 'recorder': recorder, 'profiler': profiler, 'sanity_checker': sanity_checker, 'n_uf_threshold': args.n_uf, # 'power_of_two': False, }, transforms=[ AdaLossTransformLinear(), AdaLossTransformConvolution2D(), ], verbose=args.verbose) if comm.rank == 0: print(model) train_chain = MultiboxTrainChain(model_, comm=comm) chainer.cuda.get_device_from_id(device).use() # to GPU model.coder.to_gpu() model.extractor.to_gpu() model.multibox.to_gpu() shared_mem = 100 * 1000 * 1000 * 4 if args.dataset_dir is not None: chainer.dataset.set_dataset_root(args.dataset_dir) train = TransformDataset( ConcatenatedDataset(VOCBboxDataset(year='2007', split='trainval'), VOCBboxDataset(year='2012', split='trainval')), ('img', 'mb_loc', 'mb_label'), Transform(model.coder, model.insize, model.mean, dtype=dtypes[args.dtype])) if comm.rank == 0: indices = np.arange(len(train)) else: indices = None indices = chainermn.scatter_dataset(indices, comm, shuffle=True) train = train.slice[indices] train_iter = chainer.iterators.MultiprocessIterator(train, args.batchsize // comm.size, n_processes=8, n_prefetch=2, shared_mem=shared_mem) if comm.rank == 0: # NOTE: only performed on the first device test = VOCBboxDataset(year='2007', split='test', use_difficult=True, return_difficult=True) test_iter = chainer.iterators.SerialIterator(test, args.test_batchsize, repeat=False, shuffle=False) # initial lr is set to 1e-3 by ExponentialShift optimizer = chainermn.create_multi_node_optimizer( chainer.optimizers.MomentumSGD(), comm) if args.dtype == 'mixed16': if not args.nouse_fp32_update: print('==> Using FP32 update for dtype=mixed16') optimizer.use_fp32_update() # by default use fp32 update # HACK: support skipping update by existing loss scaling functionality if args.dynamic_interval is not None: optimizer.loss_scaling(interval=args.dynamic_interval, scale=None) else: optimizer.loss_scaling(interval=float('inf'), scale=None) optimizer._loss_scale_max = 1.0 # to prevent actual loss scaling optimizer.setup(train_chain) for param in train_chain.params(): if param.name == 'b': param.update_rule.add_hook(GradientScaling(2)) else: param.update_rule.add_hook(WeightDecay(0.0005)) updater = training.updaters.StandardUpdater(train_iter, optimizer, device=device) # if args.dtype == 'mixed16': # updater.loss_scale = 8 iteration_interval = (args.iteration, 'iteration') trainer = training.Trainer(updater, iteration_interval, args.out) # trainer.extend(extensions.ExponentialShift('lr', 0.1, init=args.lr), # trigger=triggers.ManualScheduleTrigger( # args.step, 'iteration')) if args.batchsize != 32: warmup_attr_ratio = 0.1 # NOTE: this is confusing but it means n_iter warmup_n_epoch = 1000 lr_shift = chainerlp.extensions.ExponentialShift( 'lr', 0.1, init=args.lr * warmup_attr_ratio, warmup_attr_ratio=warmup_attr_ratio, warmup_n_epoch=warmup_n_epoch, schedule=args.step) trainer.extend(lr_shift, trigger=(1, 'iteration')) if comm.rank == 0: if not args.profiling: trainer.extend(DetectionVOCEvaluator( test_iter, model, use_07_metric=True, label_names=voc_bbox_label_names), trigger=triggers.ManualScheduleTrigger( args.step + [args.iteration], 'iteration')) log_interval = 10, 'iteration' trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.observe_value( 'loss_scale', lambda trainer: trainer.updater.get_optimizer('main')._loss_scale), trigger=log_interval) metrics = [ 'epoch', 'iteration', 'lr', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map' ] if args.dynamic_interval is not None: metrics.insert(2, 'loss_scale') trainer.extend(extensions.PrintReport(metrics), trigger=log_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.extend(extensions.snapshot(), trigger=(args.snapshot_per_n_iteration, 'iteration')) trainer.extend(extensions.snapshot_object( model, 'model_iter_{.updater.iteration}'), trigger=(args.iteration, 'iteration')) if args.resume: serializers.load_npz(args.resume, trainer) hook = AdaLossMonitor(sample_per_n_iter=100, verbose=args.verbose, includes=['Grad', 'Deconvolution']) recorder.trainer = trainer hook.trainer = trainer with ExitStack() as stack: if comm.rank == 0: stack.enter_context(hook) trainer.run() # store recorded results if comm.rank == 0: # NOTE: only export in the first rank recorder.export().to_csv(os.path.join(args.out, 'loss_scale.csv')) profiler.export().to_csv(os.path.join(args.out, 'profile.csv')) if sanity_checker: sanity_checker.export().to_csv( os.path.join(args.out, 'sanity_check.csv')) hook.export_history().to_csv(os.path.join(args.out, 'grad_stats.csv'))
def main(): args = parse_args() res = Resource(args, train=True) train, test, train_gt, test_gt = load_train_test( train_dir=const.PREPROCESSED_TRAIN_DIR, gt_dir=const.XML_DIR) res.log_info(f'Train: {len(train)}, test: {len(test)}') model = ARCHS[args.model](n_fg_class=len(const.LABELS), pretrained_model='imagenet') model.use_preset('evaluate') train_chain = MultiboxTrainChain(model) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() train_dataset = TransformDataset( ISIC2018Task1Dataset(train, train_gt), Transform(model.coder, model.insize, model.mean)) train_iter = chainer.iterators.MultithreadIterator( train_dataset, args.batchsize, n_threads=args.loaderjob) test_dataset = TransformDataset( ISIC2018Task1Dataset(test, test_gt), Transform(model.coder, model.insize, model.mean)) test_iter = chainer.iterators.MultithreadIterator(test_dataset, args.batchsize, shuffle=False, repeat=False, n_threads=args.loaderjob) optimizer = chainer.optimizers.Adam() optimizer.setup(train_chain) for param in train_chain.params(): if param.name == 'b': param.update_rule.add_hook(GradientScaling(2)) else: param.update_rule.add_hook(WeightDecay(0.0005)) updater = training.updaters.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), args.out) trainer.extend( DetectionVOCEvaluator(test_iter, model, use_07_metric=False, label_names=const.LABELS)) trainer.extend(extensions.LogReport()) trainer.extend(extensions.observe_lr()) trainer.extend( extensions.PrintReport([ 'epoch', 'iteration', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map' ])) trainer.extend(extensions.ProgressBar(update_interval=10)) snapshot_trigger = triggers.MaxValueTrigger(key='validation/main/map') snapshot_object_trigger = triggers.MaxValueTrigger( key='validation/main/map') trainer.extend(extensions.snapshot(filename='snapshot_best.npz'), trigger=snapshot_trigger) trainer.extend(extensions.snapshot_object(model, 'model_best.npz'), trigger=snapshot_object_trigger) if args.resume: chainer.serializers.load_npz(args.resume, trainer) trainer.run() # save last model chainer.serializers.save_npz(os.path.join(args.out, 'snapshot_last.npz'), trainer) chainer.serializers.save_npz(os.path.join(args.out, 'model_last.npz'), model)
def train(train_data, val_data, label_names, iteration, lr, step_points, batchsize, gpu, out, val_iteration, log_iteration, loaderjob, resume): model = SSD300(n_fg_class=len(label_names), pretrained_model='imagenet') model.use_preset('evaluate') train_chain = MultiboxTrainChain(model) # alpha and k? if gpu >= 0: chainer.cuda.get_device_from_id(gpu).use() model.to_gpu() train_data = TransformDataset( train_data, Transform(model.coder, model.insize, model.mean)) if loaderjob <= 0: train_iter = chainer.iterators.SerialIterator(train_data, batchsize) else: train_iter = chainer.iterators.MultiprocessIterator(train_data, batchsize) # , n_processes=min((loaderjob, batchsize))) val_iter = chainer.iterators.SerialIterator(val_data, batchsize, repeat=False, shuffle=False) # think about repeat # initial lr is set to 1e-4 (default run_train) by ExponentialShift optimizer = chainer.optimizers.MomentumSGD() optimizer.setup(train_chain) for param in train_chain.params(): if param.name == 'b': param.update_rule.add_hook(GradientScaling(2)) else: param.update_rule.add_hook(WeightDecay(0.0005)) updater = training.StandardUpdater(train_iter, optimizer, device=gpu) trainer = training.Trainer(updater, (iteration, 'iteration'), out) trainer.extend( extensions.ExponentialShift('lr', 0.1, init=lr), trigger=triggers.ManualScheduleTrigger(step_points, 'iteration')) val_interval = (val_iteration, 'iteration') trainer.extend( DetectionVOCEvaluator( val_iter, model, use_07_metric=True, label_names=label_names), trigger=val_interval) ### # Logging log_interval = log_iteration, 'iteration' trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport( ['epoch', 'iteration', 'lr', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map']), trigger=log_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.extend(extensions.snapshot(), trigger=val_interval) trainer.extend( extensions.snapshot_object(model, 'model_iter_{.updater.iteration}'), trigger=val_interval) ### if resume: serializers.load_npz(resume, trainer) trainer.run()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--model', choices=('ssd300', 'ssd512'), default='ssd300') parser.add_argument('--batchsize', type=int, default=32) parser.add_argument('--iteration', type=int, default=120000) parser.add_argument('--step', type=int, nargs='*', default=[80000, 100000]) parser.add_argument('--gpu', type=int, default=-1) parser.add_argument('--out', default='result') parser.add_argument('--resume') args = parser.parse_args() if args.model == 'ssd300': model = SSD300(n_fg_class=len(voc_bbox_label_names), pretrained_model='imagenet') elif args.model == 'ssd512': model = SSD512(n_fg_class=len(voc_bbox_label_names), pretrained_model='imagenet') model.use_preset('evaluate') train_chain = MultiboxTrainChain(model) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() train = TransformDataset( ConcatenatedDataset(VOCBboxDataset(year='2007', split='trainval'), VOCBboxDataset(year='2012', split='trainval')), Transform(model.coder, model.insize, model.mean)) train_iter = chainer.iterators.MultiprocessIterator(train, args.batchsize) test = VOCBboxDataset(year='2007', split='test', use_difficult=True, return_difficult=True) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) # initial lr is set to 1e-3 by ExponentialShift optimizer = chainer.optimizers.MomentumSGD() optimizer.setup(train_chain) for param in train_chain.params(): if param.name == 'b': param.update_rule.add_hook(GradientScaling(2)) else: param.update_rule.add_hook(WeightDecay(0.0005)) updater = training.updaters.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.iteration, 'iteration'), args.out) trainer.extend(extensions.ExponentialShift('lr', 0.1, init=1e-3), trigger=triggers.ManualScheduleTrigger( args.step, 'iteration')) trainer.extend(DetectionVOCEvaluator(test_iter, model, use_07_metric=True, label_names=voc_bbox_label_names), trigger=triggers.ManualScheduleTrigger( args.step + [args.iteration], 'iteration')) log_interval = 10, 'iteration' trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport([ 'epoch', 'iteration', 'lr', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map' ]), trigger=log_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.extend(extensions.snapshot(), trigger=triggers.ManualScheduleTrigger( args.step + [args.iteration], 'iteration')) trainer.extend(extensions.snapshot_object( model, 'model_iter_{.updater.iteration}'), trigger=(args.iteration, 'iteration')) if args.resume: serializers.load_npz(args.resume, trainer) trainer.run()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--model', choices=('ssd300', 'ssd512'), default='ssd300') parser.add_argument('--batchsize', type=int, default=32) parser.add_argument('--labelnum', type=int, default=50) parser.add_argument('--gpu', type=int, default=-1) parser.add_argument('--out', default='result') parser.add_argument('--resume') parser.add_argument('--image_label', '-il', help='Path to training image-label list file') parser.add_argument('--bbox', help='Path to training bbox list file') parser.add_argument('--image_label_test', '-ilt', help='Path to training image-label list file') parser.add_argument('--bbox_test', help='Path to training bbox list file') parser.add_argument('--image_root', '-TR', default='.', help='Root directory path of image files') args = parser.parse_args() comm = chainermn.create_communicator('naive') if comm.mpi_comm.rank == 0: print('==========================================') print('Num process (COMM_WORLD): {}'.format(MPI.COMM_WORLD.Get_size())) if args.model == 'ssd300': model = SSD300(n_fg_class=args.labelnum, pretrained_model='imagenet') elif args.model == 'ssd512': model = SSD512(n_fg_class=args.labelnum, pretrained_model='imagenet') model.use_preset('evaluate') train_chain = MultiboxTrainChain(model) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() from test_datasets import DeepFashionBboxDataset if comm.rank == 0: train = DeepFashionBboxDataset(args.bbox, args.image_label, args.image_root) test = DeepFashionBboxDataset(args.bbox_test, args.image_label_test, args.image_root) train = TransformDataset( train, Transform(model.coder, model.insize, model.mean)) else: train, test = None, None train = chainermn.scatter_dataset(train, comm, shuffle=True) test = chainermn.scatter_dataset(test, comm, shuffle=True) train_iter = chainer.iterators.MultiprocessIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) # initial lr is set to 1e-3 by ExponentialShift optimizer = chainer.optimizers.MomentumSGD() optimizer = chainermn.create_multi_node_optimizer(optimizer, comm) optimizer.setup(train_chain) for param in train_chain.params(): if param.name == 'b': param.update_rule.add_hook(GradientScaling(2)) else: param.update_rule.add_hook(WeightDecay(0.0005)) updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (120000, 'iteration'), args.out) checkpoint_interval = (1000, 'iteration') checkpointer = chainermn.create_multi_node_checkpointer( name='imagenet-example', comm=comm) checkpointer.maybe_load(trainer, optimizer) trainer.extend(checkpointer, trigger=checkpoint_interval) trainer.extend(extensions.ExponentialShift('lr', 0.1, init=1e-3), trigger=triggers.ManualScheduleTrigger([80000, 100000], 'iteration')) evaluator = DetectionVOCEvaluator(test_iter, model, use_07_metric=True, label_names=voc_bbox_label_names) evaluator = chainermn.create_multi_node_evaluator(evaluator, comm) trainer.extend(evaluator, trigger=(10000, 'iteration')) if comm.rank == 0: log_interval = 10, 'iteration' trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport([ 'epoch', 'iteration', 'lr', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map' ]), trigger=log_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.extend(extensions.snapshot(), trigger=(10000, 'iteration')) trainer.extend(extensions.snapshot_object( model, 'model_iter_{.updater.iteration}'), trigger=(120000, 'iteration')) if args.resume: serializers.load_npz(args.resume, trainer) trainer.run()
def main(): parser = argparse.ArgumentParser( description='ChainerCV training example: Faster R-CNN') parser.add_argument('--gpu', '-g', type=int, default=-1) parser.add_argument('--lr', '-l', type=float, default=1e-3) parser.add_argument('--out', '-o', default='result', help='Output directory') parser.add_argument('--seed', '-s', type=int, default=0) parser.add_argument('--step_size', '-ss', type=int, default=50000) parser.add_argument('--iteration', '-i', type=int, default=70000) args = parser.parse_args() np.random.seed(args.seed) train_data = VOCDetectionDataset(split='trainval', year='2007') test_data = VOCDetectionDataset(split='test', year='2007', use_difficult=True, return_difficult=True) faster_rcnn = FasterRCNNVGG16(n_fg_class=len(voc_detection_label_names), pretrained_model='imagenet') faster_rcnn.use_preset('evaluate') model = FasterRCNNTrainChain(faster_rcnn) if args.gpu >= 0: model.to_gpu(args.gpu) chainer.cuda.get_device(args.gpu).use() optimizer = chainer.optimizers.MomentumSGD(lr=args.lr, momentum=0.9) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(rate=0.0005)) def transform(in_data): img, bbox, label = in_data _, H, W = img.shape img = faster_rcnn.prepare(img) _, o_H, o_W = img.shape scale = o_H / H bbox = transforms.resize_bbox(bbox, (H, W), (o_H, o_W)) # horizontally flip img, params = transforms.random_flip(img, x_random=True, return_param=True) bbox = transforms.flip_bbox(bbox, (o_H, o_W), x_flip=params['x_flip']) return img, bbox, label, scale train_data = TransformDataset(train_data, transform) train_iter = chainer.iterators.MultiprocessIterator(train_data, batch_size=1, n_processes=None, shared_mem=100000000) test_iter = chainer.iterators.SerialIterator(test_data, batch_size=1, repeat=False, shuffle=False) updater = chainer.training.updater.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.iteration, 'iteration'), out=args.out) trainer.extend(extensions.snapshot_object(model.faster_rcnn, 'snapshot_model.npz'), trigger=(args.iteration, 'iteration')) trainer.extend(extensions.ExponentialShift('lr', 0.1), trigger=(args.step_size, 'iteration')) log_interval = 20, 'iteration' plot_interval = 3000, 'iteration' print_interval = 20, 'iteration' trainer.extend(chainer.training.extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.PrintReport([ 'iteration', 'epoch', 'elapsed_time', 'lr', 'main/loss', 'main/roi_loc_loss', 'main/roi_cls_loss', 'main/rpn_loc_loss', 'main/rpn_cls_loss', 'validation/main/map', ]), trigger=print_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) if extensions.PlotReport.available(): trainer.extend(extensions.PlotReport(['main/loss'], file_name='loss.png', trigger=plot_interval), trigger=plot_interval) trainer.extend( DetectionVOCEvaluator(test_iter, model.faster_rcnn, use_07_metric=True, label_names=voc_detection_label_names), trigger=ManualScheduleTrigger([args.step_size, args.iteration], 'iteration'), invoke_before_training=False) trainer.extend(extensions.dump_graph('main/loss')) trainer.run()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--gpu', type=int, default=-1) parser.add_argument('--batchsize', type=int, default=2) parser.add_argument('--epoch', type=int, default=10) parser.add_argument('--mini', action="store_true") parser.add_argument('--input_size', type=int, default=512) args = parser.parse_args() dtype = np.float32 num_class = len(voc_bbox_label_names) data_augmentation_transform = DataAugmentationTransform(args.input_size) center_detection_transform = CenterDetectionTransform(args.input_size, num_class, 4, dtype=dtype) train = TransformDataset( ConcatenatedDataset(VOCBboxDataset(year='2007', split='trainval'), VOCBboxDataset(year='2012', split='trainval')), data_augmentation_transform) train = TransformDataset(train, center_detection_transform) if args.mini: train = datasets.SubDataset(train, 0, 100) train_iter = chainer.iterators.MultiprocessIterator(train, args.batchsize) test = VOCBboxDataset(year='2007', split='test', use_difficult=True, return_difficult=True) if args.mini: test = datasets.SubDataset(test, 0, 20) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) detector = CenterDetector(HourglassNet, args.input_size, num_class, dtype=dtype) #detector = CenterDetector(SimpleCNN, args.input_size, num_class) train_chain = CenterDetectorTrain(detector, 1, 0.1, 1) #train_chain = CenterDetectorTrain(detector, 1, 0, 0) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() train_chain.to_gpu(args.gpu) optimizer = Adam(alpha=1.25e-4) #optimizer = SGD() optimizer.setup(train_chain) updater = StandardUpdater(train_iter, optimizer, device=args.gpu) log_interval = 1, 'epoch' log_interval_mini = 500, 'iteration' trainer = Trainer(updater, (args.epoch, 'epoch'), out=f"result{args.gpu}") trainer.extend(extensions.LogReport(trigger=log_interval_mini)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport([ 'epoch', 'iteration', 'lr', 'main/loss', 'main/hm_loss', 'main/wh_loss', 'main/offset_loss', 'main/hm_mae', 'main/hm_pos_loss', 'main/hm_neg_loss', 'validation/main/map', ]), trigger=log_interval_mini) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.extend(DetectionVOCEvaluator(test_iter, detector, use_07_metric=True, label_names=voc_bbox_label_names), trigger=(1, 'epoch')) trainer.extend(extensions.snapshot_object( detector, 'detector{.updater.epoch:03}.npz'), trigger=(1, 'epoch')) trainer.run()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--model', choices=('ssd300', 'ssd512'), default='ssd300') parser.add_argument('--batchsize', type=int, default=32) parser.add_argument('--test-batchsize', type=int, default=16) parser.add_argument('--iteration', type=int, default=120000) parser.add_argument('--step', type=int, nargs='*', default=[80000, 100000]) parser.add_argument('--out', default='result') parser.add_argument('--resume') args = parser.parse_args() comm = chainermn.create_communicator() device = comm.intra_rank if args.model == 'ssd300': model = SSD300(n_fg_class=len(voc_bbox_label_names), pretrained_model='imagenet') elif args.model == 'ssd512': model = SSD512(n_fg_class=len(voc_bbox_label_names), pretrained_model='imagenet') model.use_preset('evaluate') train_chain = MultiboxTrainChain(model) chainer.cuda.get_device_from_id(device).use() model.to_gpu() train = TransformDataset( ConcatenatedDataset(VOCBboxDataset(year='2007', split='trainval'), VOCBboxDataset(year='2012', split='trainval')), ('img', 'mb_loc', 'mb_label'), Transform(model.coder, model.insize, model.mean)) if comm.rank == 0: indices = np.arange(len(train)) else: indices = None indices = chainermn.scatter_dataset(indices, comm, shuffle=True) train = train.slice[indices] # http://chainermn.readthedocs.io/en/latest/tutorial/tips_faqs.html#using-multiprocessiterator if hasattr(multiprocessing, 'set_start_method'): multiprocessing.set_start_method('forkserver') train_iter = chainer.iterators.MultiprocessIterator(train, args.batchsize // comm.size, n_processes=2) if comm.rank == 0: test = VOCBboxDataset(year='2007', split='test', use_difficult=True, return_difficult=True) test_iter = chainer.iterators.SerialIterator(test, args.test_batchsize, repeat=False, shuffle=False) # initial lr is set to 1e-3 by ExponentialShift optimizer = chainermn.create_multi_node_optimizer( chainer.optimizers.MomentumSGD(), comm) optimizer.setup(train_chain) for param in train_chain.params(): if param.name == 'b': param.update_rule.add_hook(GradientScaling(2)) else: param.update_rule.add_hook(WeightDecay(0.0005)) updater = training.updaters.StandardUpdater(train_iter, optimizer, device=device) trainer = training.Trainer(updater, (args.iteration, 'iteration'), args.out) trainer.extend(extensions.ExponentialShift('lr', 0.1, init=1e-3), trigger=triggers.ManualScheduleTrigger( args.step, 'iteration')) if comm.rank == 0: trainer.extend(DetectionVOCEvaluator(test_iter, model, use_07_metric=True, label_names=voc_bbox_label_names), trigger=triggers.ManualScheduleTrigger( args.step + [args.iteration], 'iteration')) log_interval = 10, 'iteration' trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport([ 'epoch', 'iteration', 'lr', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map' ]), trigger=log_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.extend(extensions.snapshot(), trigger=triggers.ManualScheduleTrigger( args.step + [args.iteration], 'iteration')) trainer.extend(extensions.snapshot_object( model, 'model_iter_{.updater.iteration}'), trigger=(args.iteration, 'iteration')) if args.resume: serializers.load_npz(args.resume, trainer) trainer.run()
def main(): parser = argparse.ArgumentParser( description='ChainerCV training example: Faster R-CNN') parser.add_argument( '--dataset_path', '-path', type=str, default="/home/takagi.kazunari/projects/datasets/SUNRGBD_2DBB_fixed") parser.add_argument('--gpu', '-g', type=int, default=-1) parser.add_argument('--lr', '-l', type=float, default=1e-3) parser.add_argument('--out', '-o', default='sunrgbd_result', help='Output directory') parser.add_argument('--seed', '-s', type=int, default=0) parser.add_argument('--step_size', '-ss', type=int, default=50000) parser.add_argument('--iteration', '-i', type=int, default=70000) args = parser.parse_args() np.random.seed(args.seed) train_data = SUNRGBDDataset(args.dataset_path, mode="train") test_data = SUNRGBDDataset(args.dataset_path, mode="test") sunrgbd_bbox_label_names = train_data.get_dataset_label() faster_rcnn = FasterRCNNVGG16(n_fg_class=len(sunrgbd_bbox_label_names), pretrained_model='imagenet') faster_rcnn.use_preset('evaluate') model = FasterRCNNTrainChain(faster_rcnn) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() optimizer = chainer.optimizers.MomentumSGD(lr=args.lr, momentum=0.9) optimizer.setup(model) optimizer.add_hook(chainer.optimizer_hooks.WeightDecay(rate=0.0005)) train_data = TransformDataset(train_data, Transform(faster_rcnn)) train_iter = chainer.iterators.MultiprocessIterator(train_data, batch_size=1, n_processes=None, shared_mem=100000000) test_iter = chainer.iterators.SerialIterator(test_data, batch_size=1, repeat=False, shuffle=False) updater = chainer.training.updaters.StandardUpdater(train_iter, optimizer, device=args.gpu) now_time = str(datetime.datetime.today()).replace(" ", "_") save_dir = osp.join(args.out, now_time) trainer = training.Trainer(updater, (args.iteration, 'iteration'), out=save_dir) #save_iteration = [i for i in range(100, args.iteration, args.step_size)] weight_save_interval = 5000, 'iteration' evaluation_interval = 10000, 'iteration' trainer.extend(extensions.snapshot_object( model.faster_rcnn, 'sunrgbd_model_{.updater.iteration}.npz'), trigger=weight_save_interval) trainer.extend(extensions.ExponentialShift('lr', 0.1), trigger=(args.step_size, 'iteration')) log_interval = 20, 'iteration' plot_interval = 10, 'iteration' print_interval = 20, 'iteration' trainer.extend(chainer.training.extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.PrintReport([ 'iteration', 'epoch', 'elapsed_time', 'lr', 'main/loss', 'main/roi_loc_loss', 'main/roi_cls_loss', 'main/rpn_loc_loss', 'main/rpn_cls_loss', 'validation/main/map', ]), trigger=print_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) if extensions.PlotReport.available(): trainer.extend(extensions.PlotReport(['main/loss'], file_name='loss.png', trigger=plot_interval), trigger=plot_interval) #do_evaluation_iteration = [i for i in range(0, args.iteration, 500)] trainer.extend(DetectionVOCEvaluator(test_iter, model.faster_rcnn, use_07_metric=True, label_names=sunrgbd_bbox_label_names), trigger=evaluation_interval) trainer.extend(extensions.dump_graph('main/loss')) trainer.run()