def download_model(model_name, logger=None): dir_path = os.path.dirname(os.path.realpath(__file__)) model_path = os.path.join(dir_path, 'model') if logger is not None: logger.info('Downloading model %s... into path %s' % (model_name, model_path)) return modelzoo.download_model(args.model, os.path.join(dir_path, 'model'))
def score(model, data_val, metrics, gpus, batch_size, rgb_mean=None, mean_img=None, image_shape='3,224,224', data_nthreads=4, label_name='softmax_label', max_num_examples=None): # create data iterator data_shape = tuple([int(i) for i in image_shape.split(',')]) if mean_img is not None: mean_args = {'mean_img':mean_img} elif rgb_mean is not None: rgb_mean = [float(i) for i in rgb_mean.split(',')] mean_args = {'mean_r':rgb_mean[0], 'mean_g':rgb_mean[1], 'mean_b':rgb_mean[2]} data = mx.io.ImageRecordIter( path_imgrec = data_val, label_width = 1, preprocess_threads = data_nthreads, batch_size = batch_size, data_shape = data_shape, label_name = label_name, rand_crop = False, rand_mirror = False, **mean_args) if isinstance(model, str): # download model dir_path = os.path.dirname(os.path.realpath(__file__)) (prefix, epoch) = modelzoo.download_model( model, os.path.join(dir_path, 'model')) sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) elif isinstance(model, tuple) or isinstance(model, list): assert len(model) == 3 (sym, arg_params, aux_params) = model else: raise TypeError('model type [%s] is not supported' % str(type(model))) # create module if gpus == '': devs = mx.cpu() else: devs = [mx.gpu(int(i)) for i in gpus.split(',')] mod = mx.mod.Module(symbol=sym, context=devs, label_names=[label_name,]) mod.bind(for_training=False, data_shapes=data.provide_data, label_shapes=data.provide_label) mod.set_params(arg_params, aux_params) if not isinstance(metrics, list): metrics = [metrics,] tic = time.time() num = 0 for batch in data: mod.forward(batch, is_train=False) for m in metrics: mod.update_metric(m, batch.label) num += batch_size if max_num_examples is not None and num > max_num_examples: break return (num / (time.time() - tic), )
def score(model, data_val, metrics, gpus, batch_size, rgb_mean=None, mean_img=None, image_shape='3,224,224', data_nthreads=4, label_name='softmax_label'): # create data iterator data_shape = tuple([int(i) for i in image_shape.split(',')]) if mean_img is not None: mean_args = {'mean_img':mean_img} elif rgb_mean is not None: rgb_mean = [float(i) for i in rgb_mean.split(',')] mean_args = {'mean_r':rgb_mean[0], 'mean_g':rgb_mean[1], 'mean_b':rgb_mean[2]} data = mx.io.ImageRecordIter( path_imgrec = data_val, label_width = 1, preprocess_threads = data_nthreads, batch_size = batch_size, data_shape = data_shape, label_name = label_name, rand_crop = False, rand_mirror = False, **mean_args) if isinstance(model, str): # download model dir_path = os.path.dirname(os.path.realpath(__file__)) (prefix, epoch) = modelzoo.download_model( model, os.path.join(dir_path, 'model')) sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) elif isinstance(model, tuple) or isinstance(model, list): assert len(model) == 3 (sym, arg_params, aux_params) = model else: raise TypeError('model type [%s] is not supported' % str(type(model))) # create module if gpus == '': devs = mx.cpu() else: devs = [mx.gpu(int(i)) for i in gpus.split(',')] mod = mx.mod.Module(symbol=sym, context=devs, label_names=[label_name,]) mod.bind(for_training=False, data_shapes=data.provide_data, label_shapes=data.provide_label) mod.set_params(arg_params, aux_params) if not isinstance(metrics, list): metrics = [metrics,] tic = time.time() num = 0 for batch in data: mod.forward(batch, is_train=False) for m in metrics: mod.update_metric(m, batch.label) num += batch_size return (num / (time.time() - tic), )
def score(model, data_val, metrics, gpus, batch_size, rgb_mean, image_shape='3,224,224', data_nthreads=4): # create data iterator rgb_mean = [float(i) for i in rgb_mean.split(',')] data_shape = tuple([int(i) for i in image_shape.split(',')]) data = mx.io.ImageRecordIter(path_imgrec=data_val, label_width=1, mean_r=rgb_mean[0], mean_g=rgb_mean[1], mean_b=rgb_mean[2], preprocess_threads=data_nthreads, batch_size=batch_size, data_shape=data_shape, rand_crop=False, rand_mirror=False) # download model dir_path = os.path.dirname(os.path.realpath(__file__)) (prefix, epoch) = modelzoo.download_model(model, os.path.join(dir_path, 'model')) # create module sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) if gpus == '': devs = mx.cpu() else: devs = [mx.gpu(int(i)) for i in gpus.split(',')] mod = mx.mod.Module(symbol=sym, context=devs) mod.bind(for_training=False, data_shapes=data.provide_data, label_shapes=data.provide_label) mod.set_params(arg_params, aux_params) if not isinstance(metrics, list): metrics = [ metrics, ] tic = time.time() num = 0 for batch in data: mod.forward(batch, is_train=False) for m in metrics: mod.update_metric(m, batch.label) num += batch_size return (num / (time.time() - tic), )
def score(model, data_val, metrics, gpus, batch_size, rgb_mean, image_shape='3,224,224', data_nthreads=4): # create data iterator rgb_mean = [float(i) for i in rgb_mean.split(',')] data_shape = tuple([int(i) for i in image_shape.split(',')]) data = mx.io.ImageRecordIter( path_imgrec = data_val, label_width = 1, mean_r = rgb_mean[0], mean_g = rgb_mean[1], mean_b = rgb_mean[2], preprocess_threads = data_nthreads, batch_size = batch_size, data_shape = data_shape, rand_crop = False, rand_mirror = False) # download model dir_path = os.path.dirname(os.path.realpath(__file__)) (prefix, epoch) = modelzoo.download_model( model, os.path.join(dir_path, 'model')) # create module sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) if gpus == '': devs = mx.cpu() else: devs = [mx.gpu(int(i)) for i in gpus.split(',')] mod = mx.mod.Module(symbol=sym, context=devs) mod.bind(for_training=False, data_shapes=data.provide_data, label_shapes=data.provide_label) mod.set_params(arg_params, aux_params) if not isinstance(metrics, list): metrics = [metrics,] tic = time.time() num = 0 for batch in data: mod.forward(batch, is_train=False) for m in metrics: mod.update_metric(m, batch.label) num += batch_size return (num / (time.time() - tic), )
parser.add_argument('--pretrained-model', type=str, help='the pre-trained model') parser.add_argument('--layer-before-fullc', type=str, default='flatten0', help='the name of the layer before the last fullc layer') # use less augmentations for fine-tune data.set_data_aug_level(parser, 1) set_imagenet_aug(parser) # use a small learning rate and less regularizations parser.set_defaults(image_shape='3,224,224', num_epochs=30, lr=.01, lr_step_epochs='20', wd=0, mom=0) args = parser.parse_args() # load pretrained model dir_path = os.path.dirname(os.path.realpath(__file__)) (prefix, epoch) = modelzoo.download_model( args.pretrained_model, os.path.join(dir_path, 'model')) if args.load_epoch is not None: (prefix, epoch) = (args.model_prefix, args.load_epoch) logging.info(prefix) logging.info(epoch) sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) # remove the last fullc layer (new_sym, new_args) = get_fine_tune_model( sym, arg_params, args.num_classes, args.layer_before_fullc) print(args.batch_size,args.image_shape[0],args.image_shape[1],args.image_shape[2]) image_shape = [args.batch_size] for part in str.split(args.image_shape,','): image_shape.append(int(part)) arg_shape,out_shape,aux_shape = new_sym.infer_shape(data=(image_shape[0],image_shape[1],image_shape[2],image_shape[3]))
# use less augmentations for fine-tune. by default here it uses no augmentations # use a small learning rate and less regularizations parser.set_defaults(image_shape='3,224,224', num_epochs=30, lr=.01, lr_step_epochs='20', wd=0, mom=0) args = parser.parse_args() # load pretrained model and params dir_path = os.path.dirname(os.path.realpath(__file__)) (prefix, epoch) = modelzoo.download_model( args.pretrained_model, os.path.join(dir_path, 'model')) if prefix is None: (prefix, epoch) = (args.pretrained_model, args.load_epoch) sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) if args.dtype != 'float32': # load symbol of trained network, so we can cast it to support other dtype # fine tuning a network in a datatype which was not used for training originally, # requires access to the code used to generate the symbol used to train that model. # we then need to modify the symbol to add a layer at the beginning # to cast data to that dtype. We also need to cast output of layers before softmax # to float32 so that softmax can still be in float32. # if the network chosen from symols/ folder doesn't have cast for the new datatype, # it will still train in fp32 if args.network not in ['inception-v3',\ 'inception-v4', 'resnet-v1', 'resnet', 'resnext', 'vgg']: