def test_verify_code_gen2(self):
_, res = verify_code(source9, exc=False)
self.assertIn('CodeNodeVisitor', str(res))
tree = res.print_tree()
self.assertIn('comprehension', tree)
self.assertIn('\n', tree)
rows = res.Rows
node = rows[0]['node']
text = res.print_node(node)
self.assertIn('body=', text)
def test_verify_code2(self):
_, res = verify_code(source2)
self.assertIn('CodeNodeVisitor', str(res))
tree = res.print_tree()
self.assertIn('BinOp:', tree)
self.assertIn('\n', tree)
rows = res.Rows
node = rows[0]['node']
text = res.print_node(node)
self.assertIn('body=', text)
def test_verify_code_index(self):
_, res = verify_code(source8)
self.assertIn('CodeNodeVisitor', str(res))
tree = res.print_tree()
self.assertIn('Subscript', tree)
self.assertIn('\n', tree)
rows = res.Rows
node = rows[0]['node']
text = res.print_node(node)
self.assertIn('body=', text)
def test_verify_code_ops_in(self):
for op in ['in', 'not in']:
with self.subTest(op=op):
_, res = verify_code(source3.format(op))
self.assertIn('CodeNodeVisitor', str(res))
tree = res.print_tree()
self.assertIn('Compare', tree)
self.assertIn('\n', tree)
rows = res.Rows
node = rows[0]['node']
text = res.print_node(node)
self.assertIn('body=', text)
def test_verify_code_not(self):
for op in ['not', '-', '+']:
with self.subTest(op=op):
_, res = verify_code(source7.format(op))
self.assertIn('CodeNodeVisitor', str(res))
tree = res.print_tree()
self.assertIn('UnaryOp', tree)
self.assertIn('\n', tree)
rows = res.Rows
node = rows[0]['node']
text = res.print_node(node)
self.assertIn('body=', text)
def test_verify_code_ops(self):
for op in ['**', 'and', '*', '/', '-', '+', 'or', '&', '|']:
with self.subTest(op=op):
_, res = verify_code(source3.format(op))
self.assertIn('CodeNodeVisitor', str(res))
tree = res.print_tree()
if 'BinOp' not in tree and 'BoolOp' not in tree:
raise AssertionError("Unable to find %r in\n%r" %
(op, str(tree)))
self.assertIn('\n', tree)
rows = res.Rows
node = rows[0]['node']
text = res.print_node(node)
self.assertIn('body=', text)
def _create_asv_benchmark_file( # pylint: disable=R0914
location,
model,
scenario,
optimisations,
new_conv_options,
extra,
dofit,
problem,
runtime,
X_train,
X_test,
y_train,
y_test,
Xort_test,
init_types,
conv_options,
method_name,
n_features,
dims,
opsets,
output_index,
predict_kwargs,
prefix_import,
exc,
execute=False,
location_pyspy=None,
patterns=None):
"""
Creates a benchmark file based in the information received
through the argument. It uses template @see cl TemplateBenchmark.
"""
if patterns is None:
raise ValueError("Patterns list is empty.") # pragma: no cover
def format_conv_options(d_options, class_name):
if d_options is None:
return None
res = {}
for k, v in d_options.items():
if isinstance(k, type):
if "." + class_name + "'" in str(k):
res[class_name] = v
continue
raise ValueError( # pragma: no cover
"Class '{}', unable to format options {}".format(
class_name, d_options))
res[k] = v
return res
def _nick_name_options(model, opts):
# Shorten common onnx options, see _CommonAsvSklBenchmark._to_onnx.
if opts is None:
return opts
short_opts = shorten_onnx_options(model, opts)
if short_opts is not None:
return short_opts
res = {}
for k, v in opts.items():
if hasattr(k, '__name__'):
res["####" + k.__name__ + "####"] = v
else:
res[k] = v
return res
def _make_simple_name(name):
simple_name = name.replace("bench_", "").replace("_bench", "")
simple_name = simple_name.replace("bench.", "").replace(".bench", "")
simple_name = simple_name.replace(".", "-")
repl = {'_': '', 'solverliblinear': 'liblinear'}
for k, v in repl.items():
simple_name = simple_name.replace(k, v)
return simple_name
runtimes_abb = {
'scikit-learn': 'skl',
'onnxruntime1': 'ort',
'onnxruntime2': 'ort2',
'python': 'pyrt',
'python_compiled': 'pyrtc',
}
runtime = [runtimes_abb[k] for k in runtime]
# Looping over configuration.
names = []
for optimisation in optimisations:
merged_options = [
_merge_options(nconv_options, conv_options)
for nconv_options in new_conv_options
]
nck_opts = [_nick_name_options(model, opts) for opts in merged_options]
try:
name = _asv_class_name(model,
scenario,
optimisation,
extra,
dofit,
conv_options,
problem,
shorten=True)
except ValueError as e:
if exc:
raise e # pragma: no cover
warnings.warn(str(e))
continue
filename = name.replace(".", "_") + ".py"
try:
class_content = _select_pattern_problem(problem, patterns)
except ValueError as e:
if exc:
raise e
warnings.warn(str(e))
continue
full_class_name = _asv_class_name(model,
scenario,
optimisation,
extra,
dofit,
conv_options,
problem,
shorten=False)
class_name = name.replace("bench.", "").replace(".", "_") + "_bench"
# n_features, N, runtimes
rep = {
"['skl', 'pyrtc', 'ort'], # values for runtime":
str(runtime),
"[1, 10, 100, 1000, 10000, 100000], # values for N":
str(dims),
"[4, 20], # values for nf":
str(n_features),
"[get_opset_number_from_onnx()], # values for opset":
str(opsets),
"['float', 'double'], # values for dtype":
"['float']" if '-64' not in problem else "['float', 'double']",
"[None], # values for optim":
"%r" % nck_opts,
}
for k, v in rep.items():
if k not in class_content:
raise ValueError(
"Unable to find '{}'\n{}.".format( # pragma: no cover
k, class_content))
class_content = class_content.replace(k, v + ',')
class_content = class_content.split(
"def _create_model(self):")[0].strip("\n ")
if "####" in class_content:
class_content = class_content.replace("'####",
"").replace("####'", "")
if "####" in class_content:
raise RuntimeError( # pragma: no cover
"Substring '####' should not be part of the script for '{}'\n{}"
.format(model.__name__, class_content))
# Model setup
class_content, atts = add_model_import_init(class_content, model,
optimisation, extra,
merged_options)
class_content = class_content.replace("class TemplateBenchmark",
"class {}".format(class_name))
# dtype, dofit
atts.append("chk_method_name = %r" % method_name)
atts.append("par_scenario = %r" % scenario)
atts.append("par_problem = %r" % problem)
atts.append("par_optimisation = %r" % optimisation)
if not dofit:
atts.append("par_dofit = False")
if merged_options is not None and len(merged_options) > 0:
atts.append("par_convopts = %r" %
format_conv_options(conv_options, model.__name__))
atts.append("par_full_test_name = %r" % full_class_name)
simple_name = _make_simple_name(name)
atts.append("benchmark_name = %r" % simple_name)
atts.append("pretty_name = %r" % simple_name)
if atts:
class_content = class_content.replace("# additional parameters",
"\n ".join(atts))
if prefix_import != '.':
class_content = class_content.replace(
" from .", "from .{}".format(prefix_import))
# Check compilation
try:
compile(class_content, filename, 'exec')
except SyntaxError as e: # pragma: no cover
raise SyntaxError("Unable to compile model '{}'\n{}".format(
model.__name__, class_content)) from e
# Verifies missing imports.
to_import, _ = verify_code(class_content, exc=False)
try:
miss = find_missing_sklearn_imports(to_import)
except ValueError as e: # pragma: no cover
raise ValueError("Unable to check import in script\n{}".format(
class_content)) from e
class_content = class_content.replace("# __IMPORTS__",
"\n".join(miss))
verify_code(class_content, exc=True)
class_content = class_content.replace("par_extra = {",
"par_extra = {\n")
class_content = remove_extra_spaces_and_pep8(class_content,
aggressive=True)
# Check compilation again
try:
obj = compile(class_content, filename, 'exec')
except SyntaxError as e: # pragma: no cover
raise SyntaxError("Unable to compile model '{}'\n{}".format(
model.__name__, _display_code_lines(class_content))) from e
# executes to check import
if execute:
try:
exec(obj, globals(), locals()) # pylint: disable=W0122
except Exception as e: # pragma: no cover
raise RuntimeError(
"Unable to process class '{}' ('{}') a script due to '{}'\n{}"
.format(model.__name__, filename, str(e),
_display_code_lines(class_content))) from e
# Saves
fullname = os.path.join(location, filename)
names.append(fullname)
with open(fullname, "w", encoding='utf-8') as f:
f.write(class_content)
if location_pyspy is not None:
# adding configuration for pyspy
class_name = re.compile('class ([A-Za-z_0-9]+)[(]').findall(
class_content)[0]
fullname_pyspy = os.path.splitext(
os.path.join(location_pyspy, filename))[0]
pyfold = os.path.splitext(os.path.split(fullname)[-1])[0]
dtypes = ['float', 'double'] if '-64' in problem else ['float']
for dim in dims:
for nf in n_features:
for opset in opsets:
for dtype in dtypes:
for opt in nck_opts:
tmpl = pyspy_template.replace(
'__PATH__', location)
tmpl = tmpl.replace('__CLASSNAME__',
class_name)
tmpl = tmpl.replace('__PYFOLD__', pyfold)
opt = "" if opt == {} else opt
first = True
for rt in runtime:
if first:
tmpl += textwrap.dedent("""
def profile0_{rt}(iter, cl, N, nf, opset, dtype, optim):
return setup_profile0(iter, cl, '{rt}', N, nf, opset, dtype, optim)
iter = profile0_{rt}(iter, cl, {dim}, {nf}, {opset}, '{dtype}', {opt})
print(datetime.now(), "iter", iter)
""").format(rt=rt,
dim=dim,
nf=nf,
opset=opset,
dtype=dtype,
opt="%r" % opt)
first = False
tmpl += textwrap.dedent("""
def profile_{rt}(iter, cl, N, nf, opset, dtype, optim):
return setup_profile(iter, cl, '{rt}', N, nf, opset, dtype, optim)
profile_{rt}(iter, cl, {dim}, {nf}, {opset}, '{dtype}', {opt})
print(datetime.now(), "iter", iter)
""").format(rt=rt,
dim=dim,
nf=nf,
opset=opset,
dtype=dtype,
opt="%r" % opt)
thename = "{n}_{dim}_{nf}_{opset}_{dtype}_{opt}.py".format(
n=fullname_pyspy,
dim=dim,
nf=nf,
opset=opset,
dtype=dtype,
opt=opt)
with open(thename, 'w', encoding='utf-8') as f:
f.write(tmpl)
names.append(thename)
ext = '.bat' if sys.platform.startswith(
'win') else '.sh'
script = os.path.splitext(thename)[0] + ext
short = os.path.splitext(
os.path.split(thename)[-1])[0]
with open(script, 'w', encoding='utf-8') as f:
f.write(
'py-spy record --native --function --rate=10 -o {n}_fct.svg -- {py} {n}.py\n'
.format(py=sys.executable, n=short))
f.write(
'py-spy record --native --rate=10 -o {n}_line.svg -- {py} {n}.py\n'
.format(py=sys.executable, n=short))
return names
def test_verify_code(self):
self.assertRaise(lambda: verify_code(source), ImperfectPythonCode)