def download_model(model_name, logger=None):
dir_path = os.path.dirname(os.path.realpath(__file__))
model_path = os.path.join(dir_path, 'model')
if logger is not None:
logger.info('Downloading model %s... into path %s' %
(model_name, model_path))
return modelzoo.download_model(args.model, os.path.join(dir_path, 'model'))
def score(model, data_val, metrics, gpus, batch_size, rgb_mean=None, mean_img=None,
image_shape='3,224,224', data_nthreads=4, label_name='softmax_label', max_num_examples=None):
# create data iterator
data_shape = tuple([int(i) for i in image_shape.split(',')])
if mean_img is not None:
mean_args = {'mean_img':mean_img}
elif rgb_mean is not None:
rgb_mean = [float(i) for i in rgb_mean.split(',')]
mean_args = {'mean_r':rgb_mean[0], 'mean_g':rgb_mean[1],
'mean_b':rgb_mean[2]}
data = mx.io.ImageRecordIter(
path_imgrec = data_val,
label_width = 1,
preprocess_threads = data_nthreads,
batch_size = batch_size,
data_shape = data_shape,
label_name = label_name,
rand_crop = False,
rand_mirror = False,
**mean_args)
if isinstance(model, str):
# download model
dir_path = os.path.dirname(os.path.realpath(__file__))
(prefix, epoch) = modelzoo.download_model(
model, os.path.join(dir_path, 'model'))
sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch)
elif isinstance(model, tuple) or isinstance(model, list):
assert len(model) == 3
(sym, arg_params, aux_params) = model
else:
raise TypeError('model type [%s] is not supported' % str(type(model)))
# create module
if gpus == '':
devs = mx.cpu()
else:
devs = [mx.gpu(int(i)) for i in gpus.split(',')]
mod = mx.mod.Module(symbol=sym, context=devs, label_names=[label_name,])
mod.bind(for_training=False,
data_shapes=data.provide_data,
label_shapes=data.provide_label)
mod.set_params(arg_params, aux_params)
if not isinstance(metrics, list):
metrics = [metrics,]
tic = time.time()
num = 0
for batch in data:
mod.forward(batch, is_train=False)
for m in metrics:
mod.update_metric(m, batch.label)
num += batch_size
if max_num_examples is not None and num > max_num_examples:
break
return (num / (time.time() - tic), )
def score(model, data_val, metrics, gpus, batch_size, rgb_mean=None, mean_img=None,
image_shape='3,224,224', data_nthreads=4, label_name='softmax_label'):
# create data iterator
data_shape = tuple([int(i) for i in image_shape.split(',')])
if mean_img is not None:
mean_args = {'mean_img':mean_img}
elif rgb_mean is not None:
rgb_mean = [float(i) for i in rgb_mean.split(',')]
mean_args = {'mean_r':rgb_mean[0], 'mean_g':rgb_mean[1],
'mean_b':rgb_mean[2]}
data = mx.io.ImageRecordIter(
path_imgrec = data_val,
label_width = 1,
preprocess_threads = data_nthreads,
batch_size = batch_size,
data_shape = data_shape,
label_name = label_name,
rand_crop = False,
rand_mirror = False,
**mean_args)
if isinstance(model, str):
# download model
dir_path = os.path.dirname(os.path.realpath(__file__))
(prefix, epoch) = modelzoo.download_model(
model, os.path.join(dir_path, 'model'))
sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch)
elif isinstance(model, tuple) or isinstance(model, list):
assert len(model) == 3
(sym, arg_params, aux_params) = model
else:
raise TypeError('model type [%s] is not supported' % str(type(model)))
# create module
if gpus == '':
devs = mx.cpu()
else:
devs = [mx.gpu(int(i)) for i in gpus.split(',')]
mod = mx.mod.Module(symbol=sym, context=devs, label_names=[label_name,])
mod.bind(for_training=False,
data_shapes=data.provide_data,
label_shapes=data.provide_label)
mod.set_params(arg_params, aux_params)
if not isinstance(metrics, list):
metrics = [metrics,]
tic = time.time()
num = 0
for batch in data:
mod.forward(batch, is_train=False)
for m in metrics:
mod.update_metric(m, batch.label)
num += batch_size
return (num / (time.time() - tic), )
def score(model,
data_val,
metrics,
gpus,
batch_size,
rgb_mean,
image_shape='3,224,224',
data_nthreads=4):
# create data iterator
rgb_mean = [float(i) for i in rgb_mean.split(',')]
data_shape = tuple([int(i) for i in image_shape.split(',')])
data = mx.io.ImageRecordIter(path_imgrec=data_val,
label_width=1,
mean_r=rgb_mean[0],
mean_g=rgb_mean[1],
mean_b=rgb_mean[2],
preprocess_threads=data_nthreads,
batch_size=batch_size,
data_shape=data_shape,
rand_crop=False,
rand_mirror=False)
# download model
dir_path = os.path.dirname(os.path.realpath(__file__))
(prefix, epoch) = modelzoo.download_model(model,
os.path.join(dir_path, 'model'))
# create module
sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch)
if gpus == '':
devs = mx.cpu()
else:
devs = [mx.gpu(int(i)) for i in gpus.split(',')]
mod = mx.mod.Module(symbol=sym, context=devs)
mod.bind(for_training=False,
data_shapes=data.provide_data,
label_shapes=data.provide_label)
mod.set_params(arg_params, aux_params)
if not isinstance(metrics, list):
metrics = [
metrics,
]
tic = time.time()
num = 0
for batch in data:
mod.forward(batch, is_train=False)
for m in metrics:
mod.update_metric(m, batch.label)
num += batch_size
return (num / (time.time() - tic), )
def score(model, data_val, metrics, gpus, batch_size, rgb_mean,
image_shape='3,224,224', data_nthreads=4):
# create data iterator
rgb_mean = [float(i) for i in rgb_mean.split(',')]
data_shape = tuple([int(i) for i in image_shape.split(',')])
data = mx.io.ImageRecordIter(
path_imgrec = data_val,
label_width = 1,
mean_r = rgb_mean[0],
mean_g = rgb_mean[1],
mean_b = rgb_mean[2],
preprocess_threads = data_nthreads,
batch_size = batch_size,
data_shape = data_shape,
rand_crop = False,
rand_mirror = False)
# download model
dir_path = os.path.dirname(os.path.realpath(__file__))
(prefix, epoch) = modelzoo.download_model(
model, os.path.join(dir_path, 'model'))
# create module
sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch)
if gpus == '':
devs = mx.cpu()
else:
devs = [mx.gpu(int(i)) for i in gpus.split(',')]
mod = mx.mod.Module(symbol=sym, context=devs)
mod.bind(for_training=False,
data_shapes=data.provide_data,
label_shapes=data.provide_label)
mod.set_params(arg_params, aux_params)
if not isinstance(metrics, list):
metrics = [metrics,]
tic = time.time()
num = 0
for batch in data:
mod.forward(batch, is_train=False)
for m in metrics:
mod.update_metric(m, batch.label)
num += batch_size
return (num / (time.time() - tic), )
parser.add_argument('--pretrained-model', type=str,
help='the pre-trained model')
parser.add_argument('--layer-before-fullc', type=str, default='flatten0',
help='the name of the layer before the last fullc layer')
# use less augmentations for fine-tune
data.set_data_aug_level(parser, 1)
set_imagenet_aug(parser)
# use a small learning rate and less regularizations
parser.set_defaults(image_shape='3,224,224', num_epochs=30,
lr=.01, lr_step_epochs='20', wd=0, mom=0)
args = parser.parse_args()
# load pretrained model
dir_path = os.path.dirname(os.path.realpath(__file__))
(prefix, epoch) = modelzoo.download_model(
args.pretrained_model, os.path.join(dir_path, 'model'))
if args.load_epoch is not None:
(prefix, epoch) = (args.model_prefix, args.load_epoch)
logging.info(prefix)
logging.info(epoch)
sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch)
# remove the last fullc layer
(new_sym, new_args) = get_fine_tune_model(
sym, arg_params, args.num_classes, args.layer_before_fullc)
print(args.batch_size,args.image_shape[0],args.image_shape[1],args.image_shape[2])
image_shape = [args.batch_size]
for part in str.split(args.image_shape,','):
image_shape.append(int(part))
arg_shape,out_shape,aux_shape = new_sym.infer_shape(data=(image_shape[0],image_shape[1],image_shape[2],image_shape[3]))
def download_model(model_name, logger=None):
dir_path = os.path.dirname(os.path.realpath(__file__))
model_path = os.path.join(dir_path, 'model')
if logger is not None:
logger.info('Downloading model %s... into path %s' % (model_name, model_path))
return modelzoo.download_model(args.model, os.path.join(dir_path, 'model'))
# use less augmentations for fine-tune. by default here it uses no augmentations
# use a small learning rate and less regularizations
parser.set_defaults(image_shape='3,224,224',
num_epochs=30,
lr=.01,
lr_step_epochs='20',
wd=0,
mom=0)
args = parser.parse_args()
# load pretrained model and params
dir_path = os.path.dirname(os.path.realpath(__file__))
(prefix, epoch) = modelzoo.download_model(
args.pretrained_model, os.path.join(dir_path, 'model'))
if prefix is None:
(prefix, epoch) = (args.pretrained_model, args.load_epoch)
sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch)
if args.dtype != 'float32':
# load symbol of trained network, so we can cast it to support other dtype
# fine tuning a network in a datatype which was not used for training originally,
# requires access to the code used to generate the symbol used to train that model.
# we then need to modify the symbol to add a layer at the beginning
# to cast data to that dtype. We also need to cast output of layers before softmax
# to float32 so that softmax can still be in float32.
# if the network chosen from symols/ folder doesn't have cast for the new datatype,
# it will still train in fp32
if args.network not in ['inception-v3',\
'inception-v4', 'resnet-v1', 'resnet', 'resnext', 'vgg']:
