def split_model(symbol, params, inputs_ext, keys, logger=logging): infer_shapes = spass.sym_infer_shape(symbol, params, inputs_ext) bases = [s for s in sutils.topo_sort(symbol) if s.attr('name') in keys] base = mx.sym.Group(bases) base_params = {k: params[k] for k in base.list_inputs() if k in params} base_inputs_ext = inputs_ext graph = {} inputs = {k: v for k, v in inputs_ext.items()} for sym in sutils.topo_sort(symbol): name, op_name = sym.attr('name'), sym.attr('op_name') childs, attr = sutils.sym_iter(sym.get_children()), sym.list_attr() node = sym if childs is not None: childs = [graph[c.attr('name')] for c in childs] node = sutils.get_mxnet_op(op_name)(*childs, **attr, name=name) if name in keys: node = mx.sym.var(name) inputs[name] = {'shape': infer_shapes[name]} graph[name] = node nodes = [graph[sym.attr('name')] for sym in symbol] top = nodes[0] if len(nodes) == 1 else mx.sym.Group(nodes) top_params = {k: params[k] for k in top.list_inputs() if k in params} top_inputs_ext = {k: v for k, v in inputs.items() if k not in inputs_ext} return base, base_params, base_inputs_ext, top, top_params, top_inputs_ext
def get_conv_names(modelname): conv_name_dct_old = {} weight_thresh = {} myfile = path.expanduser('~/tvm-cvm/cvm/models/' + modelname + '_mid.ini') for k, v in [v.split(':') for v in load_file(myfile)]: wname = k.strip() thresh = float(v.replace(',', '').strip()) weight_thresh[wname] = thresh myfile = path.expanduser('~/tvm-cvm/cvm/models/' + modelname + '_normal.ini') weight_normal = [v.strip() for v in load_file(myfile)] sym_file = path.expanduser('~/tvm-cvm/data/' + modelname + '.prepare.json') params_file = path.expanduser('~/tvm-cvm/data/' + modelname + '.prepare.params') sym = mx.sym.load(sym_file) params = nd.load(params_file) weight_2_conv = [] for sym in sutils.topo_sort(sym, params): if sym.attr('op_name') == 'Convolution': name = sym.attr('name') wname = sutils.sym_iter(sym.get_children())[1].attr('name') if wname in weight_thresh or wname in weight_normal: continue weight_2_conv.append((wname, name)) return weight_2_conv
def _reshape(inputs, attrs, params): X, shape = inputs graph = {} for op in sutils.topo_sort(shape): name, op_name = op.attr('name'), op.attr('op_name') childs, attr = sutils.sym_iter(op.get_children()), op.list_attr() if childs is not None: childs = [graph[c.attr('name')] for c in childs] if sutils.is_var(op, params): pass elif childs is None: params[name] = sutils.get_nd_op(op_name)(**attr) op = mx.sym.var(name, shape=params[name].shape) else: childs = [graph[c.attr('name')] for c in childs] assert all([sutils.is_params(c, params) for c in childs]) in_params = [params[c.attr('name')] for c in childs] if op_name == "expand_dims" and in_params[0].shape == (): params[name] = nd.array([in_params[0].asnumpy()], dtype=in_params[0].dtype) elif op_name == "Reshape" and sutils.get_attr(attr, 'shape') == []: assert in_params[0].shape == (1, ) params[name] = nd.array(in_params[0].asnumpy()[0], dtype=in_params[0].dtype) else: params[name] = sutils.get_nd_op(op_name)(*in_params, **attr) op = mx.sym.var(name, shape=params[name].shape) graph[name] = op assert sutils.is_params(graph[shape.attr('name')], params) shape = params[shape.attr('name')].asnumpy().tolist() shape[0] = -1 # since dim zero is batch, set -1 for flexiblity. return mx.sym.reshape(X, shape)
def _collect_params(self, symbol): params = {} for op in sutils.topo_sort(symbol): if sutils.is_var(op, params): _, shp, _ = op.infer_shape() params[op.attr('name')] = mx.nd.uniform(-1, 1, shp[0]) return params
def get_mxnet_outs(symbol, params, input_shape, ctx, gpu_flag, logger, iter_num): batch_size, _, input_size, _ = input_shape data, _ = load_data(input_size=input_size, batch_size=batch_size, layout='NCHW') data = nd.array(data) _, deps = topo_sort(symbol, with_deps=True) out_cache, ans = {}, {} times = {} def _impl(op, params, graph, **kwargs): deps = kwargs['deps'] name, op_name = op.attr('name'), op.attr('op_name') childs, attr = sym_iter(op.get_children()), op.list_attr() if op_name == 'null': start_time = None out = data if is_inputs(op, params) else params[name] elif childs is None: start_time= time.time() out = get_nd_op(op_name)(**attr) if gpu_flag: nd.waitall() end_time = time.time() else: cinfos = [(c.attr('name'), get_entry_id(c)) for c in childs] nd_inputs = [out_cache[n[0]][n[1]] for n in cinfos] start_time = time.time() out = get_nd_op(op_name)(*nd_inputs, **attr) if gpu_flag: nd.waitall() end_time = time.time() for n, _ in cinfos: assert n in deps deps[n].remove(name) if len(deps[n]) == 0: del out_cache[n] if start_time is not None: if op_name not in times: times[op_name] = {} times[op_name][name] = end_time - start_time out = [out] if len(op) == 1 else out out_cache[name] = [o.as_in_context(ctx) for o in out] topo_visit_transformer(symbol, params, _impl, deps=deps, data=data) out_cache.clear() stime = {} for opn, dct in times.items(): stime[opn] = {} tl = list(dct.values()) # stime[opn]['num'] = len(tl) # stime[opn]['max'] = max(tl) # stime[opn]['min'] = min(tl) # stime[opn]['mean'] = np.mean(tl) # stime[opn]['std'] = np.std(tl) stime[opn]['total'] = sum(tl) logger.info('successfully run: #%s'%iter_num) return stime
def split_model(model, keys): symbol, params = model.symbol, model.params nodes = [s for s in topo_sort(symbol) if s.attr('name') in keys] base = nodes[0] if len(nodes) == 1 else mx.sym.Group(nodes) base_params = {k: params[k] for k in base.list_inputs() if k in params} graph = {} infer_shapes = tpass.infer_shape(symbol, params) for sym in topo_sort(symbol): name, op_name = sym.attr('name'), sym.attr('op_name') childs, attr = sym_iter(sym.get_children()), sym.list_attr() node = sym if childs is not None: childs = [sutils.get_node(c, graph) for c in childs] node = get_mxnet_op(op_name)(*childs, **attr, name=name) if name in keys: node = mx.sym.var(name, \ shape=infer_shapes[name][sutils.get_entry_id(sym)]) graph[name] = node nodes = [sutils.get_node(c, graph) for c in symbol] top = nodes[0] if len(nodes) == 1 else mx.sym.Group(nodes) top_params = {k: params[k] for k in top.list_inputs() if k in params} return Model(base, base_params), Model(top, top_params)
def cmp_model(modelname, batch_size=160, revise_outs=False, tf_check_point=None, mx_check_point=None, tf_start_point=None): input_size = model_input_size[modelname] # mxnet model symbol_file = path.join(mx_dir, "tf_" + modelname + ".json") params_file = path.join(mx_dir, "tf_" + modelname + ".params") ftf.tf_dump_model(modelname, revise_outs=revise_outs) symbol = mx.sym.load(symbol_file) params = mx.nd.load(params_file) if mx_check_point is None: mx_check_point = sutils.topo_sort(symbol)[-1].attr('name') mx_data, mx_label = load_data_1(batch_size=batch_size, input_size=input_size, layout='NCHW') mx_outs = get_mxnet_outs(symbol, params, mx_data, mx_check_point) print('mxnet check with shape: ', mx_outs.shape) # evaluate if revise_outs: get_metric(mx_outs, mx_label) return # tensorflow model model = ftf.modelfile[modelname] tfparser = TFParser(model) tfgraph = tfparser.parse() if tf_start_point is None: for tfnode in ftf.topo_sort(tfgraph): name, op_name = tfnode.name, tfnode.op if op_name in ftf.default_tf_start_types: tf_start_point = name if tf_check_point is None: tf_check_point = ftf.topo_sort(tfgraph)[-1].name tf_data, tf_label = load_data_1(batch_size=batch_size, input_size=input_size, layout='NHWC') tf_outs = get_tensorflow_outs(model, tf_data, tf_check_point, tf_start_point) print("tensorflow check with shape: ", tf_outs.shape) # compare the result between tf and mx calculate_res(mx_outs, tf_outs)
def __init__(self, model, input_prec=8): self.old_names = model.output_names() self.current_model = model self._data = None self.th_dict = {} self._op_default_input_precs() self.precs = {s.attr('name'):{} \ for s in topo_sort(self.current_model)} if 'data' not in self.precs: raise RuntimeError("please invoke `init` function first") self.precs['data'][OUT_KEY] = input_prec self.scales = {} self.softmax_lambd = 10 self.shift_bits = 5
def summary(sym, err_op=None): _s = "" for op in sutils.topo_sort(sym): name, op_name = op.attr('name'), op.attr('op_name') childs, attr = sutils.sym_iter(op.get_children()), op.list_attr() prefix = "Op " if op_name == "null" else "Var" if (err_op is not None) and (name == err_op.attr('name')): prefix = "> " + prefix _s += "%5s:%-10s, Name=%-15s, Attr=%-40s" \ % (prefix, op_name, name, attr) if childs is not None: cinfos = ["%s(%d)" % (c.attr('name'), sutils.get_entry_id(c)) \ for c in childs] _s += ", Inputs=%s" % ", ".join(cinfos) _s += "\n" return _s
def dump(model, symbol, params): logger = logging.getLogger('model dump') prefix = "./data/tf_%s" % (model) sym_file, params_file = utils.extend_fname(prefix) with open(sym_file, "w") as f: f.write(symbol.tojson()) snames = [s.attr('name') for s in sutils.topo_sort(symbol)] items = dict(params.items()) for k, v in items.items(): if v.shape == (): print("%40s \t%s %s" % (k, type(v), v.shape), k in snames) assert k not in snames del params[k] nd.save(params_file, params) logger.info("Model successfully dumped to '%s'", sym_file)
def merge_model(base, base_params, base_inputs_ext, top, top_params, maps): graph = {maps[c.attr('name')]: c for c in base} for sym in sutils.topo_sort(top): name, op_name = sym.attr('name'), sym.attr('op_name') childs, attr = sutils.sym_iter(sym.get_children()), sym.list_attr() node = sym if childs is not None: childs = [graph[c.attr('name')] for c in childs] node = sutils.get_mxnet_op(op_name)(*childs, **attr, name=name) if name in graph: node = graph[name] graph[name] = node symbols = [graph[s.attr('name')] for s in top] symbol = symbols[0] if len(symbols) == 1 else mx.sym.Group(symbols) params = base_params params.update(top_params) params = {k: params[k] for k in symbol.list_inputs() if k in params} return symbol, params
def get_mxnet_outs(symbol, params, data, check_point, ctx=mx.cpu()): data = nd.array(data) _, deps = sutils.topo_sort(symbol, with_deps=True) out_cache, ans = {}, {} def _impl(op, params, graph, **kwargs): deps = kwargs['deps'] name, op_name = op.attr('name'), op.attr('op_name') childs, attr = sutils.sym_iter(op.get_children()), op.list_attr() if op_name == 'null': out = data if sutils.is_inputs(op, params) else params[name] elif childs is None: out = sutils.get_nd_op(op_name)(**attr) else: cinfos = [(c.attr('name'), sutils.get_entry_id(c)) for c in childs] nd_inputs = [out_cache[n[0]][n[1]] for n in cinfos] out = sutils.get_nd_op(op_name)(*nd_inputs, **attr) for n, _ in cinfos: assert n in deps if name not in deps[n]: # for op like: op = broadcast_mul(X, X) # `cinfos` will have duplicate entries # avoid removing more than once continue deps[n].remove(name) if len(deps[n]) == 0: del out_cache[n] if name == check_point: ans[check_point] = out out = [out] if len(op) == 1 else out out_cache[name] = [o.as_in_context(ctx) for o in out] print( "itermediate result calculated from start_point: `data` to check_point: `%s`" % check_point) sutils.topo_visit_transformer(symbol, params, _impl, deps=deps, data=data) out_cache.clear() # res = ans[check_point].asnumpy() res = ans[check_point] return res
def merge_model(base_model, top_model, base_name_maps=None, callback=None): base_name_maps = {} if base_name_maps is None else base_name_maps graph = {base_name_maps.get(c.attr('name'), c.attr('name')):c \ for c in base_model.symbol} for sym in topo_sort(top_model.symbol): name, op_name = sym.attr('name'), sym.attr('op_name') childs, attr = sym_iter(sym.get_children()), sym.list_attr() node = sym if childs is not None: childs = [sutils.get_node(c, graph) for c in childs] node = get_mxnet_op(op_name)(*childs, **attr, name=name) if name in graph: node = graph[name] if callback is not None: node = callback(node, top_model.params, graph) graph[name] = node nodes = [sutils.get_node(s, graph) for s in top_model.symbol] symbol = nodes[0] if len(nodes) == 1 else mx.sym.Group(nodes) params = base_model.params params.update(top_model.params) return Model(symbol, params)
def run_mx(modelname, batch_size=160, quantized=False, mx_check_point=None, float_dtype="float32", evaluate=False): input_size = model_input_size[modelname] suffix = ".mrt.quantize" if quantized else "" symbol_file = path.join(mx_dir, modelname + suffix + ".json") params_file = path.join(mx_dir, modelname + suffix + ".params") symbol = mx.sym.load(symbol_file) params = mx.nd.load(params_file) params = tpass.convert_params_dtype(params, dest_dtype="float32") if mx_check_point is None: mx_check_point = sutils.topo_sort(symbol)[-1].attr('name') mx_data, mx_label = load_data_3(modelname, batch_size=batch_size, input_size=input_size, layout='NCHW', quantized=quantized) if quantized: ext_file = path.join(mx_dir, modelname + suffix + ".ext") _, _, _, scales = sim.load_ext(ext_file) else: op_names = tpass.collect_op_names(symbol, params) print(op_names) scales = None mx_outs = get_mxnet_outs(symbol, params, mx_data, mx_check_point) if scales and mx_check_point in scales: scale_factor = scales[mx_check_point] print('multiply scale factor: ', scale_factor) mx_outs /= scale_factor print('mxnet check with shape: ', mx_outs.shape) if evaluate: get_metric(mx_outs, mx_label) return mx_outs
def test_sym_pass(batch_size=10, iter_num=10): logger = logging.getLogger("log.test.sym.pass") version = "" sym_fname, param_fname = load_fname(version) sym, params = mx.sym.load(sym_fname), nd.load(param_fname) params = {k.split(':')[1]: v for k, v in params.items()} calib_ctx = mx.gpu(2) ctx = [mx.gpu(int(i)) for i in "1,2,3,4,5,6,7".split(',') if i.strip()] inputs_ext = { 'data': { 'shape': (batch_size, 3, 224, 224), } } inputs = [mx.sym.var(name) for name in inputs_ext] logger.info("load dataset, symbol and parameters") order = sutils.topo_sort(sym) for op_head in order: if op_head.attr('name') == 'classifier': break sym = op_head net = mx.gluon.nn.SymbolBlock(sym, inputs) load_parameters(net, params, ctx=ctx) data_iter = ds.load_imagenet_rec(batch_size) def data_iter_func(): data = data_iter.next() return data.data[0], data.label[0] for i in range(10): if i == 3: break data, _ = data_iter_func() data_iter.reset() acc_top1 = mx.metric.Accuracy() acc_top5 = mx.metric.TopKAccuracy(5) acc_top1.reset() acc_top5.reset() def resnet(data, label): data = gluon.utils.split_and_load(data, ctx_list=ctx, batch_axis=0, even_split=False) res = [net.forward(d) for d in data] res = nd.concatenate(res) acc_top1.update(label, res) _, top1 = acc_top1.get() acc_top5.update(label, res) _, top5 = acc_top5.get() return "top1={:6.2%} top5={:6.2%}".format(top1, top5) sym, params = spass.sym_quant_prepare(sym, params, inputs_ext) qsym, qparams, precs, _ = calib.sym_simulate(sym, params, inputs_ext, data, calib_ctx) qsym, qparams = calib.sym_realize(qsym, qparams, inputs_ext, precs, "cvm") dump_sym, dump_params, dump_ext = load_fname(version, "sym.quantize", True) sim.save_ext(dump_ext, inputs_ext) nd.save(dump_params, qparams) open(dump_sym, "w").write(qsym.tojson()) dump_sym, dump_params, dump_ext = load_fname(version, "sym.quantize", True) (inputs_ext, ) = sim.load_ext(dump_ext) net3 = utils.load_model(dump_sym, dump_params, inputs, ctx=ctx) qacc_top1 = mx.metric.Accuracy() qacc_top5 = mx.metric.TopKAccuracy(5) qacc_top1.reset() qacc_top5.reset() def cvm_quantize(data, label): data = sim.load_real_data(data, 'data', inputs_ext) data = gluon.utils.split_and_load(data, ctx_list=ctx, batch_axis=0, even_split=False) res = [net3.forward(d) for d in data] res = nd.concatenate(res) qacc_top1.update(label, res) _, top1 = qacc_top1.get() qacc_top5.update(label, res) _, top5 = qacc_top5.get() return "top1={:6.2%} top5={:6.2%}".format(top1, top5) utils.multi_validate(resnet, data_iter_func, cvm_quantize, iter_num=iter_num, logger=logger)
model_prefix+".mrt.calibrate", with_ext=True))) mrt = MRT.load(model_name_calib, datadir=model_dir) if keys != "": _checkpoint_exist(sec, sym_top_file, prm_top_file) top = Model.load(sym_top_file, prm_top_file) logger.info("`%s` stage checkd" % sec) # quantization sec = 'QUANTIZATION' model_name_quant = model_name + '.mrt.quantize' if start_point < 4: restore_names = _get_val( cfg, sec, 'Restore_name', dtype=ARRAY(str_t), dval=[]) name_to_op = {} from sym_utils import topo_sort for sym in topo_sort(mrt.current_model.symbol): name, op_name = sym.attr('name'), sym.attr('op_name') if op_name not in name_to_op: name_to_op[op_name] = [] name_to_op[op_name].append(name) new_names = [] for name in restore_names: if name.startswith("_OP_") and name[4:] in name_to_op: for new_name in name_to_op[name[4:]]: new_names.append(new_name) else: new_names.append(name) restore_names = set(new_names) if '_ALL_EXCEPT_' in restore_names: from tfm_base import _pass_manager from tfm_ops import disabled_restore_ops
def input_names(self): """ List model input names. """ return [s.attr("name") for s in topo_sort(self.symbol) \ if sutils.is_inputs(s, self.params)]
def compile_to_cvm(model, model_name, datadir="/data/std_out", input_shape=None, target="cuda"): """ Compile Mxnet model into CVM Accept-JSON&BIN-Format """ logger = logging.getLogger("mrt.compile") symbol, params = model.symbol, model.params datadir = path.join(datadir, model_name) os.makedirs(datadir, exist_ok=True) # transform from mxnet symbol to cvm logger.info("Transform Mxnet symbol into CVM") nnvm_sym, _ = to_nnvm(symbol, params) dtype, nnvm_params = "int32", {} tvm_ctx = tvm.context(target, 0) for sym in topo_sort(symbol): if sutils.is_params(sym, params): key, value = sym.attr('name'), params[sym.attr('name')] flat = value.asnumpy() assert np.abs(flat).max() <= sutils.INT32_MAX, \ "key: {}\nvalue: {}".format(key, value) assert (flat.astype(dtype).astype("float64") == flat).all(), \ "key: {}\nvalue: {}".format(key, value) nnvm_params[key] = tvm.nd.array(flat.astype(dtype), tvm_ctx) # compile to JSON&Bytes format # graph = nnvm.graph.create(nnvm_sym) # open("/tmp/tmp.nnvm.json", "w").write(graph.json()) logger.info("Compile into CVM graph") if input_shape is None: for sym in topo_sort(symbol): if sutils.is_inputs(sym, params): _, oshp, _ = sym.infer_shape() assert len(oshp) == 1 input_shape = oshp[0] input_shapes = {'data': input_shape} with nnvm.compiler.build_config(opt_level=0): deploy_graph, _, nnvm_params = nnvm.compiler.build(nnvm_sym, target=target, shape=input_shapes, params=nnvm_params, dtype=dtype) # tvm parameters reduce logger.info("Parameters precision reduce") for sym in topo_sort(nnvm_sym): if sutils.is_params(sym, nnvm_params): name, attr = sym.attr('name'), sym.list_attr() precision = sutils.get_attr(attr, "precision") dtype = "int32" if precision > 8 else "int8" nnvm_params[name] = tvm.nd.array( params[name].asnumpy().astype(dtype), tvm_ctx) # dump logger.info("CVM Json&Params dump") with open(path.join(datadir, "symbol"), "w") as fout: fout.write(deploy_graph.json()) param_bytes = nnvm.compiler.save_param_dict(nnvm_params) with open(path.join(datadir, "params"), "wb") as fout: fout.write(param_bytes) return deploy_graph, nnvm_params