def split_model(symbol, params, inputs_ext, keys, logger=logging):
infer_shapes = spass.sym_infer_shape(symbol, params, inputs_ext)
bases = [s for s in sutils.topo_sort(symbol) if s.attr('name') in keys]
base = mx.sym.Group(bases)
base_params = {k: params[k] for k in base.list_inputs() if k in params}
base_inputs_ext = inputs_ext
graph = {}
inputs = {k: v for k, v in inputs_ext.items()}
for sym in sutils.topo_sort(symbol):
name, op_name = sym.attr('name'), sym.attr('op_name')
childs, attr = sutils.sym_iter(sym.get_children()), sym.list_attr()
node = sym
if childs is not None:
childs = [graph[c.attr('name')] for c in childs]
node = sutils.get_mxnet_op(op_name)(*childs, **attr, name=name)
if name in keys:
node = mx.sym.var(name)
inputs[name] = {'shape': infer_shapes[name]}
graph[name] = node
nodes = [graph[sym.attr('name')] for sym in symbol]
top = nodes[0] if len(nodes) == 1 else mx.sym.Group(nodes)
top_params = {k: params[k] for k in top.list_inputs() if k in params}
top_inputs_ext = {k: v for k, v in inputs.items() if k not in inputs_ext}
return base, base_params, base_inputs_ext, top, top_params, top_inputs_ext
def get_conv_names(modelname):
conv_name_dct_old = {}
weight_thresh = {}
myfile = path.expanduser('~/tvm-cvm/cvm/models/' + modelname + '_mid.ini')
for k, v in [v.split(':') for v in load_file(myfile)]:
wname = k.strip()
thresh = float(v.replace(',', '').strip())
weight_thresh[wname] = thresh
myfile = path.expanduser('~/tvm-cvm/cvm/models/' + modelname +
'_normal.ini')
weight_normal = [v.strip() for v in load_file(myfile)]
sym_file = path.expanduser('~/tvm-cvm/data/' + modelname + '.prepare.json')
params_file = path.expanduser('~/tvm-cvm/data/' + modelname +
'.prepare.params')
sym = mx.sym.load(sym_file)
params = nd.load(params_file)
weight_2_conv = []
for sym in sutils.topo_sort(sym, params):
if sym.attr('op_name') == 'Convolution':
name = sym.attr('name')
wname = sutils.sym_iter(sym.get_children())[1].attr('name')
if wname in weight_thresh or wname in weight_normal:
continue
weight_2_conv.append((wname, name))
return weight_2_conv
def _reshape(inputs, attrs, params):
X, shape = inputs
graph = {}
for op in sutils.topo_sort(shape):
name, op_name = op.attr('name'), op.attr('op_name')
childs, attr = sutils.sym_iter(op.get_children()), op.list_attr()
if childs is not None:
childs = [graph[c.attr('name')] for c in childs]
if sutils.is_var(op, params):
pass
elif childs is None:
params[name] = sutils.get_nd_op(op_name)(**attr)
op = mx.sym.var(name, shape=params[name].shape)
else:
childs = [graph[c.attr('name')] for c in childs]
assert all([sutils.is_params(c, params) for c in childs])
in_params = [params[c.attr('name')] for c in childs]
if op_name == "expand_dims" and in_params[0].shape == ():
params[name] = nd.array([in_params[0].asnumpy()],
dtype=in_params[0].dtype)
elif op_name == "Reshape" and sutils.get_attr(attr, 'shape') == []:
assert in_params[0].shape == (1, )
params[name] = nd.array(in_params[0].asnumpy()[0],
dtype=in_params[0].dtype)
else:
params[name] = sutils.get_nd_op(op_name)(*in_params, **attr)
op = mx.sym.var(name, shape=params[name].shape)
graph[name] = op
assert sutils.is_params(graph[shape.attr('name')], params)
shape = params[shape.attr('name')].asnumpy().tolist()
shape[0] = -1 # since dim zero is batch, set -1 for flexiblity.
return mx.sym.reshape(X, shape)
def _collect_params(self, symbol):
params = {}
for op in sutils.topo_sort(symbol):
if sutils.is_var(op, params):
_, shp, _ = op.infer_shape()
params[op.attr('name')] = mx.nd.uniform(-1, 1, shp[0])
return params
def get_mxnet_outs(symbol, params, input_shape, ctx, gpu_flag, logger, iter_num):
batch_size, _, input_size, _ = input_shape
data, _ = load_data(input_size=input_size, batch_size=batch_size, layout='NCHW')
data = nd.array(data)
_, deps = topo_sort(symbol, with_deps=True)
out_cache, ans = {}, {}
times = {}
def _impl(op, params, graph, **kwargs):
deps = kwargs['deps']
name, op_name = op.attr('name'), op.attr('op_name')
childs, attr = sym_iter(op.get_children()), op.list_attr()
if op_name == 'null':
start_time = None
out = data if is_inputs(op, params) else params[name]
elif childs is None:
start_time= time.time()
out = get_nd_op(op_name)(**attr)
if gpu_flag:
nd.waitall()
end_time = time.time()
else:
cinfos = [(c.attr('name'), get_entry_id(c)) for c in childs]
nd_inputs = [out_cache[n[0]][n[1]] for n in cinfos]
start_time = time.time()
out = get_nd_op(op_name)(*nd_inputs, **attr)
if gpu_flag:
nd.waitall()
end_time = time.time()
for n, _ in cinfos:
assert n in deps
deps[n].remove(name)
if len(deps[n]) == 0:
del out_cache[n]
if start_time is not None:
if op_name not in times:
times[op_name] = {}
times[op_name][name] = end_time - start_time
out = [out] if len(op) == 1 else out
out_cache[name] = [o.as_in_context(ctx) for o in out]
topo_visit_transformer(symbol, params, _impl, deps=deps, data=data)
out_cache.clear()
stime = {}
for opn, dct in times.items():
stime[opn] = {}
tl = list(dct.values())
# stime[opn]['num'] = len(tl)
# stime[opn]['max'] = max(tl)
# stime[opn]['min'] = min(tl)
# stime[opn]['mean'] = np.mean(tl)
# stime[opn]['std'] = np.std(tl)
stime[opn]['total'] = sum(tl)
logger.info('successfully run: #%s'%iter_num)
return stime
def split_model(model, keys):
symbol, params = model.symbol, model.params
nodes = [s for s in topo_sort(symbol) if s.attr('name') in keys]
base = nodes[0] if len(nodes) == 1 else mx.sym.Group(nodes)
base_params = {k: params[k] for k in base.list_inputs() if k in params}
graph = {}
infer_shapes = tpass.infer_shape(symbol, params)
for sym in topo_sort(symbol):
name, op_name = sym.attr('name'), sym.attr('op_name')
childs, attr = sym_iter(sym.get_children()), sym.list_attr()
node = sym
if childs is not None:
childs = [sutils.get_node(c, graph) for c in childs]
node = get_mxnet_op(op_name)(*childs, **attr, name=name)
if name in keys:
node = mx.sym.var(name, \
shape=infer_shapes[name][sutils.get_entry_id(sym)])
graph[name] = node
nodes = [sutils.get_node(c, graph) for c in symbol]
top = nodes[0] if len(nodes) == 1 else mx.sym.Group(nodes)
top_params = {k: params[k] for k in top.list_inputs() if k in params}
return Model(base, base_params), Model(top, top_params)
def cmp_model(modelname,
batch_size=160,
revise_outs=False,
tf_check_point=None,
mx_check_point=None,
tf_start_point=None):
input_size = model_input_size[modelname]
# mxnet model
symbol_file = path.join(mx_dir, "tf_" + modelname + ".json")
params_file = path.join(mx_dir, "tf_" + modelname + ".params")
ftf.tf_dump_model(modelname, revise_outs=revise_outs)
symbol = mx.sym.load(symbol_file)
params = mx.nd.load(params_file)
if mx_check_point is None:
mx_check_point = sutils.topo_sort(symbol)[-1].attr('name')
mx_data, mx_label = load_data_1(batch_size=batch_size,
input_size=input_size,
layout='NCHW')
mx_outs = get_mxnet_outs(symbol, params, mx_data, mx_check_point)
print('mxnet check with shape: ', mx_outs.shape)
# evaluate
if revise_outs:
get_metric(mx_outs, mx_label)
return
# tensorflow model
model = ftf.modelfile[modelname]
tfparser = TFParser(model)
tfgraph = tfparser.parse()
if tf_start_point is None:
for tfnode in ftf.topo_sort(tfgraph):
name, op_name = tfnode.name, tfnode.op
if op_name in ftf.default_tf_start_types:
tf_start_point = name
if tf_check_point is None:
tf_check_point = ftf.topo_sort(tfgraph)[-1].name
tf_data, tf_label = load_data_1(batch_size=batch_size,
input_size=input_size,
layout='NHWC')
tf_outs = get_tensorflow_outs(model, tf_data, tf_check_point,
tf_start_point)
print("tensorflow check with shape: ", tf_outs.shape)
# compare the result between tf and mx
calculate_res(mx_outs, tf_outs)
def __init__(self, model, input_prec=8):
self.old_names = model.output_names()
self.current_model = model
self._data = None
self.th_dict = {}
self._op_default_input_precs()
self.precs = {s.attr('name'):{} \
for s in topo_sort(self.current_model)}
if 'data' not in self.precs:
raise RuntimeError("please invoke `init` function first")
self.precs['data'][OUT_KEY] = input_prec
self.scales = {}
self.softmax_lambd = 10
self.shift_bits = 5
def summary(sym, err_op=None):
_s = ""
for op in sutils.topo_sort(sym):
name, op_name = op.attr('name'), op.attr('op_name')
childs, attr = sutils.sym_iter(op.get_children()), op.list_attr()
prefix = "Op " if op_name == "null" else "Var"
if (err_op is not None) and (name == err_op.attr('name')):
prefix = "> " + prefix
_s += "%5s:%-10s, Name=%-15s, Attr=%-40s" \
% (prefix, op_name, name, attr)
if childs is not None:
cinfos = ["%s(%d)" % (c.attr('name'), sutils.get_entry_id(c)) \
for c in childs]
_s += ", Inputs=%s" % ", ".join(cinfos)
_s += "\n"
return _s
def dump(model, symbol, params):
logger = logging.getLogger('model dump')
prefix = "./data/tf_%s" % (model)
sym_file, params_file = utils.extend_fname(prefix)
with open(sym_file, "w") as f:
f.write(symbol.tojson())
snames = [s.attr('name') for s in sutils.topo_sort(symbol)]
items = dict(params.items())
for k, v in items.items():
if v.shape == ():
print("%40s \t%s %s" % (k, type(v), v.shape), k in snames)
assert k not in snames
del params[k]
nd.save(params_file, params)
logger.info("Model successfully dumped to '%s'", sym_file)
def merge_model(base, base_params, base_inputs_ext, top, top_params, maps):
graph = {maps[c.attr('name')]: c for c in base}
for sym in sutils.topo_sort(top):
name, op_name = sym.attr('name'), sym.attr('op_name')
childs, attr = sutils.sym_iter(sym.get_children()), sym.list_attr()
node = sym
if childs is not None:
childs = [graph[c.attr('name')] for c in childs]
node = sutils.get_mxnet_op(op_name)(*childs, **attr, name=name)
if name in graph:
node = graph[name]
graph[name] = node
symbols = [graph[s.attr('name')] for s in top]
symbol = symbols[0] if len(symbols) == 1 else mx.sym.Group(symbols)
params = base_params
params.update(top_params)
params = {k: params[k] for k in symbol.list_inputs() if k in params}
return symbol, params
def get_mxnet_outs(symbol, params, data, check_point, ctx=mx.cpu()):
data = nd.array(data)
_, deps = sutils.topo_sort(symbol, with_deps=True)
out_cache, ans = {}, {}
def _impl(op, params, graph, **kwargs):
deps = kwargs['deps']
name, op_name = op.attr('name'), op.attr('op_name')
childs, attr = sutils.sym_iter(op.get_children()), op.list_attr()
if op_name == 'null':
out = data if sutils.is_inputs(op, params) else params[name]
elif childs is None:
out = sutils.get_nd_op(op_name)(**attr)
else:
cinfos = [(c.attr('name'), sutils.get_entry_id(c)) for c in childs]
nd_inputs = [out_cache[n[0]][n[1]] for n in cinfos]
out = sutils.get_nd_op(op_name)(*nd_inputs, **attr)
for n, _ in cinfos:
assert n in deps
if name not in deps[n]:
# for op like: op = broadcast_mul(X, X)
# `cinfos` will have duplicate entries
# avoid removing more than once
continue
deps[n].remove(name)
if len(deps[n]) == 0:
del out_cache[n]
if name == check_point:
ans[check_point] = out
out = [out] if len(op) == 1 else out
out_cache[name] = [o.as_in_context(ctx) for o in out]
print(
"itermediate result calculated from start_point: `data` to check_point: `%s`"
% check_point)
sutils.topo_visit_transformer(symbol, params, _impl, deps=deps, data=data)
out_cache.clear()
# res = ans[check_point].asnumpy()
res = ans[check_point]
return res
def merge_model(base_model, top_model, base_name_maps=None, callback=None):
base_name_maps = {} if base_name_maps is None else base_name_maps
graph = {base_name_maps.get(c.attr('name'), c.attr('name')):c \
for c in base_model.symbol}
for sym in topo_sort(top_model.symbol):
name, op_name = sym.attr('name'), sym.attr('op_name')
childs, attr = sym_iter(sym.get_children()), sym.list_attr()
node = sym
if childs is not None:
childs = [sutils.get_node(c, graph) for c in childs]
node = get_mxnet_op(op_name)(*childs, **attr, name=name)
if name in graph:
node = graph[name]
if callback is not None:
node = callback(node, top_model.params, graph)
graph[name] = node
nodes = [sutils.get_node(s, graph) for s in top_model.symbol]
symbol = nodes[0] if len(nodes) == 1 else mx.sym.Group(nodes)
params = base_model.params
params.update(top_model.params)
return Model(symbol, params)
def run_mx(modelname,
batch_size=160,
quantized=False,
mx_check_point=None,
float_dtype="float32",
evaluate=False):
input_size = model_input_size[modelname]
suffix = ".mrt.quantize" if quantized else ""
symbol_file = path.join(mx_dir, modelname + suffix + ".json")
params_file = path.join(mx_dir, modelname + suffix + ".params")
symbol = mx.sym.load(symbol_file)
params = mx.nd.load(params_file)
params = tpass.convert_params_dtype(params, dest_dtype="float32")
if mx_check_point is None:
mx_check_point = sutils.topo_sort(symbol)[-1].attr('name')
mx_data, mx_label = load_data_3(modelname,
batch_size=batch_size,
input_size=input_size,
layout='NCHW',
quantized=quantized)
if quantized:
ext_file = path.join(mx_dir, modelname + suffix + ".ext")
_, _, _, scales = sim.load_ext(ext_file)
else:
op_names = tpass.collect_op_names(symbol, params)
print(op_names)
scales = None
mx_outs = get_mxnet_outs(symbol, params, mx_data, mx_check_point)
if scales and mx_check_point in scales:
scale_factor = scales[mx_check_point]
print('multiply scale factor: ', scale_factor)
mx_outs /= scale_factor
print('mxnet check with shape: ', mx_outs.shape)
if evaluate:
get_metric(mx_outs, mx_label)
return mx_outs
def test_sym_pass(batch_size=10, iter_num=10):
logger = logging.getLogger("log.test.sym.pass")
version = ""
sym_fname, param_fname = load_fname(version)
sym, params = mx.sym.load(sym_fname), nd.load(param_fname)
params = {k.split(':')[1]: v for k, v in params.items()}
calib_ctx = mx.gpu(2)
ctx = [mx.gpu(int(i)) for i in "1,2,3,4,5,6,7".split(',') if i.strip()]
inputs_ext = {
'data': {
'shape': (batch_size, 3, 224, 224),
}
}
inputs = [mx.sym.var(name) for name in inputs_ext]
logger.info("load dataset, symbol and parameters")
order = sutils.topo_sort(sym)
for op_head in order:
if op_head.attr('name') == 'classifier':
break
sym = op_head
net = mx.gluon.nn.SymbolBlock(sym, inputs)
load_parameters(net, params, ctx=ctx)
data_iter = ds.load_imagenet_rec(batch_size)
def data_iter_func():
data = data_iter.next()
return data.data[0], data.label[0]
for i in range(10):
if i == 3:
break
data, _ = data_iter_func()
data_iter.reset()
acc_top1 = mx.metric.Accuracy()
acc_top5 = mx.metric.TopKAccuracy(5)
acc_top1.reset()
acc_top5.reset()
def resnet(data, label):
data = gluon.utils.split_and_load(data,
ctx_list=ctx,
batch_axis=0,
even_split=False)
res = [net.forward(d) for d in data]
res = nd.concatenate(res)
acc_top1.update(label, res)
_, top1 = acc_top1.get()
acc_top5.update(label, res)
_, top5 = acc_top5.get()
return "top1={:6.2%} top5={:6.2%}".format(top1, top5)
sym, params = spass.sym_quant_prepare(sym, params, inputs_ext)
qsym, qparams, precs, _ = calib.sym_simulate(sym, params, inputs_ext, data,
calib_ctx)
qsym, qparams = calib.sym_realize(qsym, qparams, inputs_ext, precs, "cvm")
dump_sym, dump_params, dump_ext = load_fname(version, "sym.quantize", True)
sim.save_ext(dump_ext, inputs_ext)
nd.save(dump_params, qparams)
open(dump_sym, "w").write(qsym.tojson())
dump_sym, dump_params, dump_ext = load_fname(version, "sym.quantize", True)
(inputs_ext, ) = sim.load_ext(dump_ext)
net3 = utils.load_model(dump_sym, dump_params, inputs, ctx=ctx)
qacc_top1 = mx.metric.Accuracy()
qacc_top5 = mx.metric.TopKAccuracy(5)
qacc_top1.reset()
qacc_top5.reset()
def cvm_quantize(data, label):
data = sim.load_real_data(data, 'data', inputs_ext)
data = gluon.utils.split_and_load(data,
ctx_list=ctx,
batch_axis=0,
even_split=False)
res = [net3.forward(d) for d in data]
res = nd.concatenate(res)
qacc_top1.update(label, res)
_, top1 = qacc_top1.get()
qacc_top5.update(label, res)
_, top5 = qacc_top5.get()
return "top1={:6.2%} top5={:6.2%}".format(top1, top5)
utils.multi_validate(resnet,
data_iter_func,
cvm_quantize,
iter_num=iter_num,
logger=logger)
model_prefix+".mrt.calibrate", with_ext=True)))
mrt = MRT.load(model_name_calib, datadir=model_dir)
if keys != "":
_checkpoint_exist(sec, sym_top_file, prm_top_file)
top = Model.load(sym_top_file, prm_top_file)
logger.info("`%s` stage checkd" % sec)
# quantization
sec = 'QUANTIZATION'
model_name_quant = model_name + '.mrt.quantize'
if start_point < 4:
restore_names = _get_val(
cfg, sec, 'Restore_name', dtype=ARRAY(str_t), dval=[])
name_to_op = {}
from sym_utils import topo_sort
for sym in topo_sort(mrt.current_model.symbol):
name, op_name = sym.attr('name'), sym.attr('op_name')
if op_name not in name_to_op:
name_to_op[op_name] = []
name_to_op[op_name].append(name)
new_names = []
for name in restore_names:
if name.startswith("_OP_") and name[4:] in name_to_op:
for new_name in name_to_op[name[4:]]:
new_names.append(new_name)
else:
new_names.append(name)
restore_names = set(new_names)
if '_ALL_EXCEPT_' in restore_names:
from tfm_base import _pass_manager
from tfm_ops import disabled_restore_ops
def input_names(self):
""" List model input names. """
return [s.attr("name") for s in topo_sort(self.symbol) \
if sutils.is_inputs(s, self.params)]
def compile_to_cvm(model,
model_name,
datadir="/data/std_out",
input_shape=None,
target="cuda"):
""" Compile Mxnet model into CVM Accept-JSON&BIN-Format
"""
logger = logging.getLogger("mrt.compile")
symbol, params = model.symbol, model.params
datadir = path.join(datadir, model_name)
os.makedirs(datadir, exist_ok=True)
# transform from mxnet symbol to cvm
logger.info("Transform Mxnet symbol into CVM")
nnvm_sym, _ = to_nnvm(symbol, params)
dtype, nnvm_params = "int32", {}
tvm_ctx = tvm.context(target, 0)
for sym in topo_sort(symbol):
if sutils.is_params(sym, params):
key, value = sym.attr('name'), params[sym.attr('name')]
flat = value.asnumpy()
assert np.abs(flat).max() <= sutils.INT32_MAX, \
"key: {}\nvalue: {}".format(key, value)
assert (flat.astype(dtype).astype("float64") == flat).all(), \
"key: {}\nvalue: {}".format(key, value)
nnvm_params[key] = tvm.nd.array(flat.astype(dtype), tvm_ctx)
# compile to JSON&Bytes format
# graph = nnvm.graph.create(nnvm_sym)
# open("/tmp/tmp.nnvm.json", "w").write(graph.json())
logger.info("Compile into CVM graph")
if input_shape is None:
for sym in topo_sort(symbol):
if sutils.is_inputs(sym, params):
_, oshp, _ = sym.infer_shape()
assert len(oshp) == 1
input_shape = oshp[0]
input_shapes = {'data': input_shape}
with nnvm.compiler.build_config(opt_level=0):
deploy_graph, _, nnvm_params = nnvm.compiler.build(nnvm_sym,
target=target,
shape=input_shapes,
params=nnvm_params,
dtype=dtype)
# tvm parameters reduce
logger.info("Parameters precision reduce")
for sym in topo_sort(nnvm_sym):
if sutils.is_params(sym, nnvm_params):
name, attr = sym.attr('name'), sym.list_attr()
precision = sutils.get_attr(attr, "precision")
dtype = "int32" if precision > 8 else "int8"
nnvm_params[name] = tvm.nd.array(
params[name].asnumpy().astype(dtype), tvm_ctx)
# dump
logger.info("CVM Json&Params dump")
with open(path.join(datadir, "symbol"), "w") as fout:
fout.write(deploy_graph.json())
param_bytes = nnvm.compiler.save_param_dict(nnvm_params)
with open(path.join(datadir, "params"), "wb") as fout:
fout.write(param_bytes)
return deploy_graph, nnvm_params