def test_fit(): dataset = task.Dataset(name='ToySST') predictor = task.fit( dataset, net=ag.Categorical('bert_12_768_12'), pretrained_dataset=ag.Categorical('book_corpus_wiki_en_uncased'), epochs=1, num_trials=1, batch_size=4, dev_batch_size=4, max_len=16, ngpus_per_trial=0, seed=2) test_acc = predictor.evaluate(dataset) print('accuracy is %.2f' % test_acc) print('finished')
def task_dog_breed_identification(data_path, dataset): images_path = os.path.join(data_path, dataset, 'images_all') label_path = os.path.join(data_path, dataset, 'labels.csv') test_path = os.path.join(data_path, dataset, 'test') load_dataset = task.Dataset(images_path, label_file=label_path) @ag.obj(learning_rate=ag.space.Real(0.3, 0.5), momentum=ag.space.Real(0.90, 0.95), wd=ag.space.Real(1e-6, 1e-4, log=True), multi_precision=False) class NAG(optim.NAG): pass classifier = task.fit(dataset=load_dataset, net=ag.Categorical('standford_dog_resnext101_64x4d', 'standford_dog_resnet152_v1'), optimizer=NAG(), epochs=20, final_fit_epochs=180, num_trials=40, ngpus_per_trial=8, batch_size=48, verbose=False, ensemble=1) test_dataset = task.Dataset(test_path, train=False, crop_ratio=0.65) inds, probs, probs_all = classifier.predict(test_dataset, set_prob_thresh=0.001) ag.utils.generate_prob_csv(test_dataset, probs_all, custom='./submission.csv')
def test_custom_dataset_fit(): os.system('wget https://autogluon-hackathon.s3.amazonaws.com/demodata.zip') os.system('unzip -o demodata.zip') dataset = task.Dataset(filepath='./demodata/train.csv', usecols=['text', 'target']) predictor = task.fit( dataset, net=ag.Categorical('bert_12_768_12'), pretrained_dataset=ag.Categorical('book_corpus_wiki_en_uncased'), epochs=1, num_trials=1, batch_size=4, dev_batch_size=4, max_len=16, ngpus_per_trial=0, seed=2) test_acc = predictor.evaluate(dataset) print('accuracy is %.2f' % test_acc) print('finished')
def test_tabularHPObagstack(): ############ Benchmark options you can set: ######################## perf_threshold = 1.1 # How much worse can performance on each dataset be vs previous performance without warning seed_val = 10000 # random seed subsample_size = None hyperparameter_tune = True stack_ensemble_levels = 2 num_bagging_folds = 2 verbosity = 2 # how much output to print hyperparameters = None time_limits = None num_trials = None fast_benchmark = True # False # If True, run a faster benchmark (subsample training sets, less epochs, etc), # otherwise we run full benchmark with default AutoGluon settings. # performance_value warnings are disabled when fast_benchmark = True. #### If fast_benchmark = True, can control model training time here. Only used if fast_benchmark=True #### if fast_benchmark: subsample_size = 100 nn_options = { 'num_epochs': 2, 'learning_rate': ag.Real(0.001, 0.01), 'lr_scheduler': ag.Categorical(None, 'cosine', 'step') } gbm_options = { 'num_boost_round': 20, 'learning_rate': ag.Real(0.01, 0.1) } hyperparameters = {'GBM': gbm_options, 'NN': nn_options} time_limits = 150 num_trials = 3 fit_args = { 'num_bagging_folds': num_bagging_folds, 'stack_ensemble_levels': stack_ensemble_levels, 'hyperparameter_tune': hyperparameter_tune, 'verbosity': verbosity, } if hyperparameters is not None: fit_args['hyperparameters'] = hyperparameters if time_limits is not None: fit_args['time_limits'] = time_limits fit_args['num_bagging_sets'] = 2 if num_trials is not None: fit_args['num_trials'] = num_trials ################################################################### run_tabular_benchmarks(fast_benchmark=fast_benchmark, subsample_size=subsample_size, perf_threshold=perf_threshold, seed_val=seed_val, fit_args=fit_args)
help="Meta architecture of the model") args = parser.parse_args() logging.info('args: {}'.format(args)) dataset_train, dataset_test = get_dataset(args) time_limits = 5 * 60 * 60 # 5 days epochs = 12 # use coco pre-trained model for custom datasets transfer = None if ('voc' in args.dataset_name) or ( 'coco' in args.dataset_name) else 'coco' detector = task.fit(dataset_train, num_trials=30, epochs=epochs, net=ag.Categorical('darknet53', 'mobilenet1.0'), meta_arch=args.meta_arch, lr=ag.Categorical(1e-2, 5e-3, 1e-3, 5e-4, 1e-4, 5e-5), transfer=transfer, data_shape=ag.Categorical(320, 416), nthreads_per_trial=16, ngpus_per_trial=1, batch_size=8, lr_decay_epoch=ag.Categorical('80,90', '85,95'), warmup_epochs=ag.Int(1, 10), warmup_iters=ag.Int(250, 1000), wd=ag.Categorical(1e-4, 5e-4, 2.5e-4), syncbn=ag.Bool(), label_smooth=ag.Bool(), time_limits=time_limits, dist_ip_addrs=[])
import autogluon as ag from gluoncv.auto.estimators.faster_rcnn import FasterRCNNEstimator from gluoncv.auto.tasks.object_detection import ObjectDetection # Define search space time_limits = 60 * 60 # 1hr search_args = { 'dataset': 'voc', 'split_ratio': 0.8, 'num_trials': 30, 'epochs': ag.Categorical(30, 40, 50, 60), 'num_workers': 16, 'net': ag.Categorical('resnest101', 'resnest50'), 'meta_arch': 'faster_rcnn', 'search_strategy': 'random', 'search_options': {}, 'lr': ag.Categorical(0.005, 0.002, 2e-4, 5e-4), 'transfer': False, 'data_shape': (640, 800), 'nthreads_per_trial':
'https://autogluon.s3.amazonaws.com/datasets/tiny_motorbike.zip', path=root) filename = ag.unzip(filename_zip, root=root) console.log("Criando TASK TRAIN ") import os data_root = os.path.join(root, filename) dataset_train = task.Dataset(data_root, classes=('motorbike', )) console.info("TRAINING DATA MODEL...") time_limits = 5 * 60 * 60 # 5 hours epochs = 30 detector = task.fit(dataset_train, num_trials=2, epochs=epochs, lr=ag.Categorical(5e-4, 1e-4), ngpus_per_trial=1, time_limits=time_limits) console.success("TRAINING DONE !") console.log("START TEST MODEL ") dataset_test = task.Dataset(data_root, index_file_name='test', classes=('motorbike', )) test_map = detector.evaluate(dataset_test) console.log("mAP on test dataset: {}".format(test_map[1][1])) console.success("SAVE MODEL") savefile = 'model.pkl' detector.save(savefile)
args = parser.parse_args() logging.info('args: {}'.format(args)) time_limits = 7 * 24 * 60 * 60 # 7 days epochs = 20 # use coco pre-trained model for custom datasets transfer = None if ('voc' in args.dataset_name) or ( 'coco' in args.dataset_name) else 'coco' default_args = { 'dataset': 'voc_tiny', 'dataset_root': '~/.mxnet/datasets/', 'gpus': [0, 1, 2, 3], 'meta_arch': 'yolo3', 'backbone': ag.Categorical('mobilenet1.0'), 'lr': ag.Categorical(5e-4, 1e-4), 'loss': gluon.loss.SoftmaxCrossEntropyLoss(), 'split_ratio': 0.8, 'batch_size': 16, 'epochs': 50, 'nthreads_per_trial': 12, 'ngpus_per_trial': 1, 'seed': 233, 'data_shape': 416, 'start_epoch': 0, 'lr_mode': 'step', 'lr_decay': 0.1, 'lr_decay_period': 0, 'lr_decay_epoch': [160, 180], 'warmup_lr': 0.0,
help="whether need to re-download dataset") parser.add_argument('--meta-arch', type=str, default='yolo3', choices=['yolo3', 'faster_rcnn'], help="Meta architecture of the model") args = parser.parse_args() logging.info('args: {}'.format(args)) dataset_train, dataset_test = get_dataset(args) time_limits = 5 * 24 * 60 * 60 # 5 days epochs = 20 # use coco pre-trained model for custom datasets transfer = None if ('voc' in args.dataset_name) or ('coco' in args.dataset_name) else 'coco' if args.meta_arch == 'yolo3': kwargs = {'num_trials': 30, 'epochs': epochs, 'net': ag.Categorical('darknet53', 'mobilenet1.0'), 'meta_arch': args.meta_arch, 'lr': ag.Categorical(1e-2, 5e-3, 1e-3, 5e-4, 1e-4, 5e-5), 'transfer': transfer, 'data_shape': ag.Categorical(320, 416), 'nthreads_per_trial': 16, 'ngpus_per_trial': 8, 'batch_size': 64, 'lr_decay_epoch': ag.Categorical('80,90', '85,95'), 'warmup_epochs': ag.Int(1, 10), 'warmup_iters': ag.Int(250, 1000), 'wd': ag.Categorical(1e-4, 5e-4, 2.5e-4), 'syncbn': ag.Bool(), 'label_smooth': ag.Bool(), 'time_limits': time_limits, 'dist_ip_addrs': []} elif args.meta_arch == 'faster_rcnn': kwargs = {'num_trials': 30, 'epochs': ag.Categorical(30, 40, 50, 60), 'net': ag.Categorical('resnest101', 'resnest50'), 'meta_arch': args.meta_arch, 'lr': ag.Categorical(0.02, 0.01, 0.005, 0.002, 2e-4, 5e-4), 'transfer': transfer, 'data_shape': (640, 800), 'nthreads_per_trial': 16, 'ngpus_per_trial': 8, 'batch_size': 16, 'lr_decay_epoch': ag.Categorical('24,28', '35', '50,55', '40', '45', '55',
import autogluon as ag from gluoncv.auto.estimators import SSDEstimator, FasterRCNNEstimator, YOLOv3Estimator, CenterNetEstimator from gluoncv.auto.tasks import ObjectDetection if __name__ == '__main__': args = { 'dataset': 'voc_tiny', 'meta_arch': 'ssd', 'lr': ag.Categorical(1e-3, 5e-4), 'epochs': 5, 'num_trials': 2 } if 'meta_arch' in args: if args['meta_arch'] == 'ssd': estimator = SSDEstimator elif args['meta_arch'] == 'faster_rcnn': estimator = FasterRCNNEstimator elif args['meta_arch'] == 'yolo3': estimator = YOLOv3Estimator elif args['meta_arch'] == 'center_net': estimator = CenterNetEstimator else: raise NotImplementedError('%s is not implemented.', args['meta_arch']) else: estimator = None task = ObjectDetection(args, estimator)