def test_fit():
dataset = task.Dataset(name='ToySST')
predictor = task.fit(
dataset,
net=ag.Categorical('bert_12_768_12'),
pretrained_dataset=ag.Categorical('book_corpus_wiki_en_uncased'),
epochs=1,
num_trials=1,
batch_size=4,
dev_batch_size=4,
max_len=16,
ngpus_per_trial=0,
seed=2)
test_acc = predictor.evaluate(dataset)
print('accuracy is %.2f' % test_acc)
print('finished')
def task_dog_breed_identification(data_path, dataset):
images_path = os.path.join(data_path, dataset, 'images_all')
label_path = os.path.join(data_path, dataset, 'labels.csv')
test_path = os.path.join(data_path, dataset, 'test')
load_dataset = task.Dataset(images_path, label_file=label_path)
@ag.obj(learning_rate=ag.space.Real(0.3, 0.5),
momentum=ag.space.Real(0.90, 0.95),
wd=ag.space.Real(1e-6, 1e-4, log=True),
multi_precision=False)
class NAG(optim.NAG):
pass
classifier = task.fit(dataset=load_dataset,
net=ag.Categorical('standford_dog_resnext101_64x4d',
'standford_dog_resnet152_v1'),
optimizer=NAG(),
epochs=20,
final_fit_epochs=180,
num_trials=40,
ngpus_per_trial=8,
batch_size=48,
verbose=False,
ensemble=1)
test_dataset = task.Dataset(test_path, train=False, crop_ratio=0.65)
inds, probs, probs_all = classifier.predict(test_dataset,
set_prob_thresh=0.001)
ag.utils.generate_prob_csv(test_dataset,
probs_all,
custom='./submission.csv')
def test_custom_dataset_fit():
os.system('wget https://autogluon-hackathon.s3.amazonaws.com/demodata.zip')
os.system('unzip -o demodata.zip')
dataset = task.Dataset(filepath='./demodata/train.csv',
usecols=['text', 'target'])
predictor = task.fit(
dataset,
net=ag.Categorical('bert_12_768_12'),
pretrained_dataset=ag.Categorical('book_corpus_wiki_en_uncased'),
epochs=1,
num_trials=1,
batch_size=4,
dev_batch_size=4,
max_len=16,
ngpus_per_trial=0,
seed=2)
test_acc = predictor.evaluate(dataset)
print('accuracy is %.2f' % test_acc)
print('finished')
def test_tabularHPObagstack():
############ Benchmark options you can set: ########################
perf_threshold = 1.1 # How much worse can performance on each dataset be vs previous performance without warning
seed_val = 10000 # random seed
subsample_size = None
hyperparameter_tune = True
stack_ensemble_levels = 2
num_bagging_folds = 2
verbosity = 2 # how much output to print
hyperparameters = None
time_limits = None
num_trials = None
fast_benchmark = True # False
# If True, run a faster benchmark (subsample training sets, less epochs, etc),
# otherwise we run full benchmark with default AutoGluon settings.
# performance_value warnings are disabled when fast_benchmark = True.
#### If fast_benchmark = True, can control model training time here. Only used if fast_benchmark=True ####
if fast_benchmark:
subsample_size = 100
nn_options = {
'num_epochs': 2,
'learning_rate': ag.Real(0.001, 0.01),
'lr_scheduler': ag.Categorical(None, 'cosine', 'step')
}
gbm_options = {
'num_boost_round': 20,
'learning_rate': ag.Real(0.01, 0.1)
}
hyperparameters = {'GBM': gbm_options, 'NN': nn_options}
time_limits = 150
num_trials = 3
fit_args = {
'num_bagging_folds': num_bagging_folds,
'stack_ensemble_levels': stack_ensemble_levels,
'hyperparameter_tune': hyperparameter_tune,
'verbosity': verbosity,
}
if hyperparameters is not None:
fit_args['hyperparameters'] = hyperparameters
if time_limits is not None:
fit_args['time_limits'] = time_limits
fit_args['num_bagging_sets'] = 2
if num_trials is not None:
fit_args['num_trials'] = num_trials
###################################################################
run_tabular_benchmarks(fast_benchmark=fast_benchmark,
subsample_size=subsample_size,
perf_threshold=perf_threshold,
seed_val=seed_val,
fit_args=fit_args)
help="Meta architecture of the model")
args = parser.parse_args()
logging.info('args: {}'.format(args))
dataset_train, dataset_test = get_dataset(args)
time_limits = 5 * 60 * 60 # 5 days
epochs = 12
# use coco pre-trained model for custom datasets
transfer = None if ('voc' in args.dataset_name) or (
'coco' in args.dataset_name) else 'coco'
detector = task.fit(dataset_train,
num_trials=30,
epochs=epochs,
net=ag.Categorical('darknet53', 'mobilenet1.0'),
meta_arch=args.meta_arch,
lr=ag.Categorical(1e-2, 5e-3, 1e-3, 5e-4, 1e-4, 5e-5),
transfer=transfer,
data_shape=ag.Categorical(320, 416),
nthreads_per_trial=16,
ngpus_per_trial=1,
batch_size=8,
lr_decay_epoch=ag.Categorical('80,90', '85,95'),
warmup_epochs=ag.Int(1, 10),
warmup_iters=ag.Int(250, 1000),
wd=ag.Categorical(1e-4, 5e-4, 2.5e-4),
syncbn=ag.Bool(),
label_smooth=ag.Bool(),
time_limits=time_limits,
dist_ip_addrs=[])
import autogluon as ag
from gluoncv.auto.estimators.faster_rcnn import FasterRCNNEstimator
from gluoncv.auto.tasks.object_detection import ObjectDetection
# Define search space
time_limits = 60 * 60 # 1hr
search_args = {
'dataset':
'voc',
'split_ratio':
0.8,
'num_trials':
30,
'epochs':
ag.Categorical(30, 40, 50, 60),
'num_workers':
16,
'net':
ag.Categorical('resnest101', 'resnest50'),
'meta_arch':
'faster_rcnn',
'search_strategy':
'random',
'search_options': {},
'lr':
ag.Categorical(0.005, 0.002, 2e-4, 5e-4),
'transfer':
False,
'data_shape': (640, 800),
'nthreads_per_trial':
'https://autogluon.s3.amazonaws.com/datasets/tiny_motorbike.zip',
path=root)
filename = ag.unzip(filename_zip, root=root)
console.log("Criando TASK TRAIN ")
import os
data_root = os.path.join(root, filename)
dataset_train = task.Dataset(data_root, classes=('motorbike', ))
console.info("TRAINING DATA MODEL...")
time_limits = 5 * 60 * 60 # 5 hours
epochs = 30
detector = task.fit(dataset_train,
num_trials=2,
epochs=epochs,
lr=ag.Categorical(5e-4, 1e-4),
ngpus_per_trial=1,
time_limits=time_limits)
console.success("TRAINING DONE !")
console.log("START TEST MODEL ")
dataset_test = task.Dataset(data_root,
index_file_name='test',
classes=('motorbike', ))
test_map = detector.evaluate(dataset_test)
console.log("mAP on test dataset: {}".format(test_map[1][1]))
console.success("SAVE MODEL")
savefile = 'model.pkl'
detector.save(savefile)
args = parser.parse_args()
logging.info('args: {}'.format(args))
time_limits = 7 * 24 * 60 * 60 # 7 days
epochs = 20
# use coco pre-trained model for custom datasets
transfer = None if ('voc' in args.dataset_name) or (
'coco' in args.dataset_name) else 'coco'
default_args = {
'dataset': 'voc_tiny',
'dataset_root': '~/.mxnet/datasets/',
'gpus': [0, 1, 2, 3],
'meta_arch': 'yolo3',
'backbone': ag.Categorical('mobilenet1.0'),
'lr': ag.Categorical(5e-4, 1e-4),
'loss': gluon.loss.SoftmaxCrossEntropyLoss(),
'split_ratio': 0.8,
'batch_size': 16,
'epochs': 50,
'nthreads_per_trial': 12,
'ngpus_per_trial': 1,
'seed': 233,
'data_shape': 416,
'start_epoch': 0,
'lr_mode': 'step',
'lr_decay': 0.1,
'lr_decay_period': 0,
'lr_decay_epoch': [160, 180],
'warmup_lr': 0.0,
help="whether need to re-download dataset")
parser.add_argument('--meta-arch', type=str, default='yolo3', choices=['yolo3', 'faster_rcnn'],
help="Meta architecture of the model")
args = parser.parse_args()
logging.info('args: {}'.format(args))
dataset_train, dataset_test = get_dataset(args)
time_limits = 5 * 24 * 60 * 60 # 5 days
epochs = 20
# use coco pre-trained model for custom datasets
transfer = None if ('voc' in args.dataset_name) or ('coco' in args.dataset_name) else 'coco'
if args.meta_arch == 'yolo3':
kwargs = {'num_trials': 30, 'epochs': epochs,
'net': ag.Categorical('darknet53', 'mobilenet1.0'), 'meta_arch': args.meta_arch,
'lr': ag.Categorical(1e-2, 5e-3, 1e-3, 5e-4, 1e-4, 5e-5), 'transfer': transfer,
'data_shape': ag.Categorical(320, 416), 'nthreads_per_trial': 16,
'ngpus_per_trial': 8, 'batch_size': 64,
'lr_decay_epoch': ag.Categorical('80,90', '85,95'),
'warmup_epochs': ag.Int(1, 10), 'warmup_iters': ag.Int(250, 1000),
'wd': ag.Categorical(1e-4, 5e-4, 2.5e-4), 'syncbn': ag.Bool(),
'label_smooth': ag.Bool(), 'time_limits': time_limits, 'dist_ip_addrs': []}
elif args.meta_arch == 'faster_rcnn':
kwargs = {'num_trials': 30, 'epochs': ag.Categorical(30, 40, 50, 60),
'net': ag.Categorical('resnest101', 'resnest50'),
'meta_arch': args.meta_arch,
'lr': ag.Categorical(0.02, 0.01, 0.005, 0.002, 2e-4, 5e-4), 'transfer': transfer,
'data_shape': (640, 800), 'nthreads_per_trial': 16,
'ngpus_per_trial': 8, 'batch_size': 16,
'lr_decay_epoch': ag.Categorical('24,28', '35', '50,55', '40', '45', '55',
import autogluon as ag
from gluoncv.auto.estimators import SSDEstimator, FasterRCNNEstimator, YOLOv3Estimator, CenterNetEstimator
from gluoncv.auto.tasks import ObjectDetection
if __name__ == '__main__':
args = {
'dataset': 'voc_tiny',
'meta_arch': 'ssd',
'lr': ag.Categorical(1e-3, 5e-4),
'epochs': 5,
'num_trials': 2
}
if 'meta_arch' in args:
if args['meta_arch'] == 'ssd':
estimator = SSDEstimator
elif args['meta_arch'] == 'faster_rcnn':
estimator = FasterRCNNEstimator
elif args['meta_arch'] == 'yolo3':
estimator = YOLOv3Estimator
elif args['meta_arch'] == 'center_net':
estimator = CenterNetEstimator
else:
raise NotImplementedError('%s is not implemented.',
args['meta_arch'])
else:
estimator = None
task = ObjectDetection(args, estimator)
