os.path.join(data_dir, "cifar10_val.rec")) download_file('http://data.mxnet.io/data/cifar10/cifar10_val.rec', fnames[1]) download_file('http://data.mxnet.io/data/cifar10/cifar10_train.rec', fnames[0]) return fnames if __name__ == '__main__': # download data (train_fname, val_fname) = download_cifar10() # parse args parser = argparse.ArgumentParser(description="train cifar10", formatter_class=argparse.ArgumentDefaultsHelpFormatter) fit.add_fit_args(parser) data.add_data_args(parser) data.add_data_aug_args(parser) data.set_data_aug_level(parser, 2) parser.set_defaults( # network network = 'resnet', num_layers = 110, # data data_train = train_fname, data_val = val_fname, num_classes = 10, num_examples = 50000, image_shape = '3,28,28', pad_size = 4, # train batch_size = 512, num_epochs = 300,
if __name__ == "__main__": # parse args parser = argparse.ArgumentParser(description="fine-tune a dataset", formatter_class=argparse.ArgumentDefaultsHelpFormatter) train = fit.add_fit_args(parser) data.add_data_args(parser) aug = data.add_data_aug_args(parser) parser.add_argument('--pretrained-model', type=str, help='the pre-trained model') parser.add_argument('--layer-before-fullc', type=str, default='flatten0', help='the name of the layer before the last fullc layer') # use less augmentations for fine-tune data.set_data_aug_level(parser, 1) set_imagenet_aug(parser) # use a small learning rate and less regularizations parser.set_defaults(image_shape='3,224,224', num_epochs=30, lr=.01, lr_step_epochs='20', wd=0, mom=0) args = parser.parse_args() # load pretrained model dir_path = os.path.dirname(os.path.realpath(__file__)) (prefix, epoch) = modelzoo.download_model( args.pretrained_model, os.path.join(dir_path, 'model')) if args.load_epoch is not None: (prefix, epoch) = (args.model_prefix, args.load_epoch) logging.info(prefix) logging.info(epoch)
def start_new_training(): # binding signals SUPPORTED_SIGNALS = ( signal.SIGINT, signal.SIGTERM, ) for signum in SUPPORTED_SIGNALS: try: signal.signal(signum, signal_handler) logger.info("Bind signal '%s' success to %s", signum, signal_handler) except Exception as identifier: logger.warning("Bind signal '%s' failed, err: %s", signum, identifier) try: # parse args parser = argparse.ArgumentParser( description="train imagenet-1k", formatter_class=argparse.ArgumentDefaultsHelpFormatter) fit.add_fit_args(parser) data.add_data_args(parser) data.add_data_aug_args(parser) # use a large aug level data.set_data_aug_level(parser, 3) parser.set_defaults( # network network='resnet', num_layers=50, # data num_classes=10, num_examples=60000, image_shape='3,28,28', min_random_scale=1, # if input image has min size k, suggest to use # 256.0/x, e.g. 0.533 for 480 # train num_epochs=80, lr_step_epochs='30,60', dtype='float32', batch_size=32) args = parser.parse_args() # AVA-SDK new an Instance train_ins = train.TrainInstance() # add CALLBACK batch_end_cb = train_ins.get_monitor_callback( "mxnet", batch_size=args.batch_size, batch_freq=10) args.batch_end_callback = batch_end_cb # load network from importlib import import_module net = import_module('symbols.' + args.network) sym = net.get_symbol(**vars(args)) # train fit.fit(args, sym, data.get_rec_iter) logger.info("training finish") err_msg = "" if train_ins == None: return train_ins.done(err_msg=err_msg) except Exception as err: err_msg = "training failed, err: %s" % (err) logger.info(err_msg) traceback.print_exc(file=sys.stderr) if train_ins == None: return train_ins.done(err_msg=err_msg)