def buildDataset(cls, all_results: Dataset, testie: Testie) -> List[tuple]:
dtype = testie.variables.dtype()
y = OrderedDict()
dataset = []
for i, (run, results_types) in enumerate(all_results.items()):
vars = list(run.variables.values())
if not results_types is None and len(results_types) > 0:
dataset.append(vars)
for result_type, results in results_types.items():
r = np.mean(results)
y.setdefault(result_type, []).append(r)
dtype['values'] = [None] * len(dtype['formats'])
for i, f in enumerate(dtype['formats']):
if f is str:
dtype['formats'][i] = int
values = OrderedSet()
for row in dataset:
values.add(row[i])
row[i] = values.index(row[i])
dtype['values'][i] = list(values)
X = np.array(dataset, ndmin=2)
lset = []
for result_type, v in y.items():
lset.append((result_type, X, np.array(v),dtype))
return lset
def writeversion(self,
testie,
all_results: Dataset,
allow_overwrite: bool = False,
kind=False,
reload=True):
if not reload and all_results:
prev = self.load_results(testie=testie, kind=kind, cache=True)
if prev and len(all_results) < len(prev):
print(
"ERROR ! Have less results than before. Forcing update write !"
)
reload = True
return
if reload:
results = self.load_results(testie=testie, kind=kind, cache=False)
if results:
results.update(all_results)
all_results = results
if kind:
for kind, kresult in all_results.items():
filename = self.__resultFilename(testie) + '-' + kind
self._writeversion(filename, kresult, allow_overwrite)
else:
filename = self.__resultFilename(testie)
self._writeversion(filename, all_results, allow_overwrite)
def writeversion(self, script, all_results: Dataset):
filename = self.__resultFilename(script)
try:
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
except OSError:
print("Error : could not create %s" % os.path.dirname(filename))
f = open(filename, 'w+')
f.seek(0)
for run, results in all_results.items():
v = []
for key, val in sorted(run.variables.items()):
if type(val) is tuple:
val = val[1]
v.append(key + ":" + str(val).replace(':','\:'))
type_results = []
for t,r in results.items():
str_results = []
if r is None:
pass
else:
for val in r:
str_results.append(str(val))
type_results.append(t+':'+(','.join(str_results)))
f.write(','.join(v) + "={" + '},{'.join(type_results) + "}\n")
f.close()
def buildDataset(cls, all_results: Dataset, testie: Testie):
dtype = testie.variables.dtype()
y = []
dataset = []
for i, (run, results) in enumerate(all_results.items()):
vars = list(run.variables.values())
if not results is None:
dataset.append(vars)
y.append(np.mean(results))
dtype['formats'] = dtype['formats']
dtype['names'] = dtype['names']
for i, f in enumerate(dtype['formats']):
if f is str:
dtype['formats'][i] = int
values = OrderedSet()
for row in dataset:
values.add(row[i])
row[i] = values.index(row[i])
X = np.array(dataset, ndmin=2)
return X, np.array(y, dtype=[('result', float)])
def compare(self,
testie: Testie,
variable_list,
all_results: Dataset,
build,
old_all_results,
last_build,
allow_supplementary=True,
init_done=False) -> Tuple[int, int]:
"""
Compare two sets of results for the given list of variables and returns the amount of failing test
:param init_done: True if initialization for current testie is already done (init sections for the testie and its import)
:param testie: One testie to get the config from
:param variable_list:
:param all_results:
:param build:
:param old_all_results:
:param last_build:
:param allow_supplementary:
:return: the amount of failed tests (0 means all passed)
"""
if not old_all_results:
return 0, 0
tests_passed = 0
tests_total = 0
supp_done = False
tot_runs = testie.config["n_runs"] + testie.config[
"n_supplementary_runs"]
for v in variable_list:
tests_total += 1
run = Run(v)
results_types = all_results.get(run)
# TODO : some config could implement acceptable range no matter the old value
if results_types is None or len(results_types) == 0:
continue
need_supp = False
for result_type, result in results_types.items():
if run in old_all_results and not old_all_results[run] is None:
old_result = old_all_results[run].get(result_type, None)
if old_result is None:
continue
ok, diff = self.accept_diff(testie, result, old_result)
if not ok and len(
result) < tot_runs and allow_supplementary:
need_supp = True
break
elif last_build:
if not testie.options.quiet_regression:
print("No old values for %s for version %s." %
(run, last_build.version))
if old_all_results:
old_all_results[run] = {}
if need_supp:
if not testie.options.quiet_regression:
print(
"Difference of %.2f%% is outside acceptable margin for %s. Running supplementary tests..."
% (diff * 100, run.format_variables()))
if not init_done:
testie.do_init_all(build=build,
options=testie.options,
do_test=testie.options.do_test)
init_done = True
if hasattr(testie, 'late_variables'):
v = testie.late_variables.execute(v, testie)
new_results_types, output, err = testie.execute(
build,
run,
v,
n_runs=testie.config["n_supplementary_runs"],
allowed_types={SectionScript.TYPE_SCRIPT})
for result_type, results in new_results_types.items():
results_types[result_type] += results
if not testie.options.quiet_regression:
print("Result after supplementary tests done :",
results_types)
if new_results_types is not None:
supp_done = True
all_results[run] = results_types
for result_type, result in results_types.items():
old_result = old_all_results[run].get(
result_type, None)
if old_result is None:
continue
ok, diff = self.accept_diff(testie, result, old_result)
if ok is False:
break
else:
ok = True
if len(results_types) > 0:
if not ok:
print(
"ERROR: Test %s is outside acceptable margin between %s and %s : difference of %.2f%% !"
% (testie.filename, build.version, last_build.version,
diff * 100))
else:
tests_passed += 1
if not testie.options.quiet_regression:
print("Acceptable difference of %.2f%% for %s" %
((diff * 100), run.format_variables()))
if supp_done:
build.writeversion(testie, all_results)
return tests_passed, tests_total
def run(build: Build,
all_results: Dataset,
testie: Testie,
max_depth=3,
filename=None):
print("Building dataset...")
X, y = Statistics.buildDataset(all_results, testie)
print("Learning dataset built with %d samples and %d features..." %
(X.shape[0], X.shape[1]))
clf = tree.DecisionTreeRegressor(max_depth=max_depth)
clf = clf.fit(X, y)
if max_depth is None or max_depth > 8:
print("No tree graph when maxdepth is > 8")
else:
dot_data = tree.export_graphviz(
clf,
out_file=None,
filled=True,
rounded=True,
special_characters=True,
feature_names=testie.variables.dtype()['names'])
graph = pydotplus.graph_from_dot_data(dot_data)
if filename:
f = filename
else:
f = build.result_path(testie.filename, 'pdf', suffix='_clf')
graph.write(f, format=os.path.splitext(f)[1][1:])
print("Decision tree visualization written to %s" % f)
print("")
print("Feature importances :")
# noinspection PyUnresolvedReferences
for key, f in zip(testie.variables.dtype()['names'],
clf.feature_importances_):
print(" %s : %0.2f" % (key, f))
vars_values = {}
for run, results in all_results.items():
for k, v in run.variables.items():
vars_values.setdefault(k, set()).add(v)
print('')
print("Better :")
best = X[y['result'].argmax()]
print(" ", end='')
f = next(iter(all_results.items()))
for i, (k, v) in enumerate(f[0].variables.items()):
print("%s = %s, " % (k, best[i]), end='')
print(' : %.02f' % y['result'].max())
print('')
print("Means and std/mean per variables :")
for k, vals in vars_values.items():
if len(vals) is 1:
continue
print("%s :" % k)
for v in sorted(vals):
tot = 0
std = 0
n = 0
for run, results in all_results.items():
if run.variables[k] == v:
if not results is None:
tot += np.mean(results)
std += np.std(results)
n += 1
if n == 0:
print(" %s : None" % v)
else:
print(" %s : (%.02f,%.02f), " % (v, tot / n, std / n /
(tot / n)))
print("")
def execute_all(
self,
build,
options,
prev_results: Dataset = None,
do_test=True,
allowed_types=SectionScript.ALL_TYPES_SET) -> Tuple[Dataset, bool]:
"""Execute script for all variables combinations. All tools reliy on this function for execution of the testie
:param allowed_types:Tyeps of scripts allowed to run. Set with either init, scripts or both
:param do_test: Actually run the tests
:param options: NPF options object
:param build: A build object
:param prev_results: Previous set of result for the same build to update or retrieve
:return: Dataset(Dict of variables as key and arrays of results as value)
"""
init_done = False
if not SectionScript.TYPE_SCRIPT in allowed_types:
# If scripts is not in allowed_types, we have to run the init by force now
self.do_init_all(build,
options,
do_test=do_test,
allowed_types=allowed_types)
return {}, True
all_results = {}
for variables in self.variables:
run = Run(variables)
if hasattr(self, 'late_variables'):
variables = self.late_variables.execute(variables, self)
r_status, r_out, r_err = self.test_require(variables, build)
if not r_status:
if not self.options.quiet:
print("Requirement not met for %s" %
run.format_variables(self.config["var_hide"]))
if r_out.strip():
print(output.strip())
if r_err.strip():
print(err.strip())
continue
if prev_results and prev_results is not None and not options.force_test:
run_results = prev_results.get(run, {})
if run_results is None:
run_results = {}
else:
run_results = {}
if not run_results and options.use_last:
for version in build.repo.method.get_history(build.version):
oldb = Build(build.repo, version)
r = oldb.load_results(self)
if r and run in r:
run_results = r[run]
break
for result_type in self.config['results_expect']:
if result_type not in run_results:
run_results = {}
have_new_results = False
n_runs = self.config["n_runs"] - (
0 if options.force_test or len(run_results) == 0 else min([
len(results)
for result_type, results in run_results.items()
]))
if n_runs > 0 and do_test:
if not init_done:
self.do_init_all(build,
options,
do_test,
allowed_types=allowed_types)
init_done = True
if not self.options.quiet:
print(run.format_variables(self.config["var_hide"]))
new_results, output, err = self.execute(
build,
run,
variables,
n_runs,
n_retry=self.config["n_retry"],
allowed_types={SectionScript.TYPE_SCRIPT})
if new_results:
if self.options.show_full:
print("stdout:")
print(output)
print("stderr:")
print(err)
for k, v in new_results.items():
run_results.setdefault(k, []).extend(v)
have_new_results = True
else:
if not self.options.quiet:
print(run.format_variables(self.config["var_hide"]))
if len(run_results) > 0:
if not self.options.quiet:
if len(run_results) == 1:
print(list(run_results.values())[0])
else:
print(run_results)
all_results[run] = run_results
else:
all_results[run] = {}
# Save results
if all_results and have_new_results:
if prev_results:
prev_results[run] = all_results[run]
build.writeversion(self, prev_results)
else:
build.writeversion(self, all_results)
return all_results, init_done