def optimize_for_inference(args, outputs):
args_map = {
"enable_io16xc32": "f16_io_f32_comp",
"enable_ioc16": "f16_io_comp",
"enable_hwcd4": "use_nhwcd4",
"enable_nchw4": "use_nchw4",
"enable_nchw88": "use_nchw88",
"enable_nchw44": "use_nchw44",
"enable_nchw44_dot": "use_nchw44_dot",
"enable_nchw32": "use_nchw32",
"enable_chwn4": "use_chwn4",
"enable_fuse_conv_bias_nonlinearity": "fuse_conv_bias_nonlinearity",
"enable_fuse_conv_bias_with_z": "fuse_conv_bias_with_z",
}
kwargs = {}
for k, v in args_map.items():
if getattr(args, k):
assert (
args.optimize_for_inference
), "optimize_for_inference should be set when {} is given".format(k)
kwargs[v] = True
outputs = [G.VarNode(output) for output in outputs]
if args.optimize_for_inference:
outputs = [i._node for i in G.optimize_for_inference(outputs, **kwargs)]
return outputs
def test_replace_vars():
g = mgb_graph.Graph()
g.options.async_exec_level = 0b100
device = "xpux"
dtype = np.float32
a = mgb_graph.InputNode(device=device, dtype=dtype, graph=g)
const = g.make_const(1.234)
a_plus_a = F.add(a.outputs[0], a.outputs[0])
a_plus_a_mul_const = F.mul(a_plus_a, const)
rst = F.add(a_plus_a_mul_const, a.outputs[0])
(new, ) = cgtools.replace_vars([rst._node], {const._node: a_plus_a._node})
out = mgb_graph.OutputNode(mgb_graph.VarNode(new))
func = g.compile(out.outputs[0])
func.execute()
x = make_dev_tensor(5.0, device=device)
a.set_value(x)
res = out.get_value().numpy()
np.testing.assert_equal(res, np.array([105.0]))
def test_replace_oprs():
g = mgb_graph.Graph()
g.options.async_exec_level = 0b100
device = "xpux"
dtype = np.float32
a = mgb_graph.InputNode(device=device, dtype=dtype, graph=g)
const = g.make_const(1.25, device=device)
add_op = Elemwise(Elemwise.Mode.ADD)
mul_op = Elemwise(Elemwise.Mode.MUL)
a_plus_a = apply_normal_varnode(add_op, a.outputs[0], a.outputs[0])[0]
old_opr = a_plus_a.op
a_plus_a_mul_const = apply_normal_varnode(mul_op, a_plus_a, const)[0]
a_mul_a = apply_normal_varnode(mul_op, a.outputs[0], a.outputs[0])[0]
new_opr = a_mul_a.op
(new, ) = cgtools.replace_oprs([a_plus_a_mul_const._node],
{old_opr._node: new_opr._node})
out = mgb_graph.OutputNode(mgb_graph.VarNode(new))
func = g.compile(out.outputs[0])
func.execute()
x = make_dev_tensor(5.0, device=device)
a.set_value(x)
res = out.get_value().numpy()
np.testing.assert_equal(res, np.array([5.0 * 5.0 * 1.25]))
def make_feeds(args):
cg_rt, _, outputs = G.load_graph(args.input)
inputs = cgtools.get_dep_vars(outputs, "Host2DeviceCopy")
inputs = {i.name: i for i in inputs}
if not args.no_assert:
replace_varmap = {}
inp_map = {}
# replace var use InputNode
for name, var in inputs.items():
inp = G.InputNode(device="xpux",
dtype=var.dtype,
shape=var.shape,
graph=cg_rt)
replace_varmap[var] = inp.outputs[0]
inp_map[name] = inp
new = cgtools.replace_vars(outputs, replace_varmap)
if isinstance(new, rt.VarNode):
new = list(new)
output_nodes = [G.OutputNode(var) for var in new]
func = cg_rt.compile([node.outputs[0] for node in output_nodes])
def make_dev_tensor(value, dtype=None, device=None):
return as_raw_tensor(value, dtype=dtype,
device=device)._dev_tensor()
def calculate(*args, **kwargs):
output_val = []
# set inputs value
for name, var in inputs.items():
val = kwargs.pop(name, None)
assert val is not None, "miss input name{}".format(name)
dev_tensor = make_dev_tensor(val,
dtype=var.dtype,
device="xpux")
inp_map[name].set_value(dev_tensor)
func.execute()
for res in output_nodes:
output_val.append(res.get_value().numpy())
return output_val
def expect_name(var):
return "{}:expect".format(var.name)
testcases = []
np.set_printoptions(precision=2, threshold=4, suppress=True)
data_list = []
for item in args.data:
if item.startswith("@"):
with open(item[1:], "r") as f:
data_list.extend(
[line.rstrip() for line in f if line.rstrip() != ""])
else:
data_list.append(item)
for inp_spec in data_list:
cur_testcase = gen_one_testcase(args, inputs, inp_spec)
assert len(cur_testcase) == len(
inputs), "required inputs: {}; given data: {}".format(
inputs.keys(), cur_testcase.keys())
if not args.no_assert:
outputs_get = calculate(**cur_testcase)
for var, val in zip(outputs, outputs_get):
cur_testcase[expect_name(var)] = val
logger.info(
"generate test groundtruth: var={} shape={} range=({}, {})"
" mean={} var={}".format(var, val.shape, val.min(),
val.max(), np.mean(val),
np.var(val)))
testcases.append(cur_testcase)
logger.info("add testcase: \n {}".format("\n ".join(
"{}: shape={} dtype={} range=({:.2f},{:.2f}) "
"mean={:.2f} sd={:.2f}".format(k, v.shape, v.dtype, v.min(),
v.max(), np.mean(v), np.std(v))
for k, v in sorted(cur_testcase.items()))))
if not args.no_assert:
def expect_shp(var):
ret = var.shape
if ret:
return ret
return testcases[0][expect_name(var)].shape
def assert_equal(expect, real, **kwargs):
op = builtin.AssertEqual(**kwargs)
(res, ) = apply(op, expect, real)
return res
verbose = not args.silent
outputs_new = []
for i in outputs:
device = rt.CompNode("xpux")
dtype = i.dtype
name = expect_name(i)
shape = expect_shp(i)
# make expect output as one input of model.
expect_get = rt.make_h2d(cg_rt, device, dtype, shape, name)
# insert assert opr to check expect and real.
outputs_new.append(
assert_equal(
G.VarNode(expect_get),
G.VarNode(i),
verbose=verbose,
maxerr=args.maxerr,
))
inputs[expect_name(i)] = expect_get
outputs = outputs_new
return cg_rt, {"outputs": outputs, "testcases": testcases}
def assert_equal(expect, real, **kwargs):
op = builtin.AssertEqual(**kwargs)
(res, ) = G.apply_normal_varnode(op, expect, real)
return G.VarNode(res)
