def print_match_dist(matches, non_matches, threshold, label, errors):
# try:
display("""
* Count of Matches: **%s**
* Count of Non-Matches: **%s**
""" % (
len(matches),
len(non_matches)
))
plot_match_dist_mpl(matches, non_matches, threshold=threshold)
plt.show()
figs.save_plot("Match Distribution " + f"({label})", height=9).display()
def print_performance(roc_results, ft, errors):
# try:
threshold = roc_results.threshold
if threshold is not None:
fpr_t = roc_results.fpr_thresh
tpr_t = roc_results.tpr_thresh
fnr_t = roc_results.fnr_thresh
display(dedent("""
## Performance at the threshold of *%.1f*
* False Positive Rate: *%2.2f* %%
* False Negative Rate: *%2.2f* %%
* True Positive Rate: *%2.2f* %%
""" % (threshold, fpr_t*100, fnr_t*100, tpr_t*100)))
def print_zoo_plot(zoo_results, label='', theshold=2000):
outliers = plot_Zoo_mpl(zoo_results, 100)
display("""
The zoo plot displays how different people perform based on their average match score and their average non-match score.
Each point represents a single individual and a good system will have few outliers.
It can be used to investigate which people or groups of people are causing more system errors when additional metadata is available.
In the tables below scores are shown in bold when they are above the threshold if a non-match and below the threshold for a match.
""")
figs.save_plot("Zoo Plot" + f"({label})", height=9).display()
out = outliers[["id", "zoo_class", "false_match_score",
"match_score", "false_match_score_max"]]
classes = {
"dove": "Individuals classified as doves work the best with this algorithm as they verify easily and are more difficult to impost",
"chameleon": "Indviduals classified as chameleons may have very generic features weighted heavily by the algorithm",
"worm": "Indviduals classified as worms work the worst with the algorithm as they have difficulty verifying and are easily imposted",
"phantom": "Indviduals classified as phantoms may have very unique features and match poorly in all cases"
}
for zoo_class in classes.keys():
filtered = out[out["zoo_class"] == zoo_class]
def fm_score(s, greater=True):
if greater:
return "**%r**" % s if s >= theshold else str(s)
else:
return "**%r**" % s if s <= theshold else str(s)
def highlight(col_name, greater):
return [fm_score(s, greater) for s in filtered[col_name]]
if len(filtered) > 0:
# filtered["match_score"]=highlight("match_score",False)
# filtered["false_match_score"]=highlight("false_match_score",True)
# filtered["false_match_score_max"]=highlight("false_match_score_max",True)
title = zoo_class+'s'
display(f"""
* **{title.title()}**
{classes[zoo_class]}. {proportion_to_description(len(filtered),len(zoo_results),title)}.
""")
tables.read_df(filtered).display(f"Zoo analysis {title} ({label})")
def build_md_doc(self, item_list, level=1):
""" Recursively builds a markdown document from a dict
Rules are:
* If value is a string treat as markdown
* If a key:
* contains a () then it is an analysis, params are in the dict
* **file**: load markdown file
"""
#raise_error(str(item_list))
for item_num, result in enumerate(item_list):
for key, val in result.items():
# if key =='doc_meta': # allow definition of local doc params
# self.doc_meta.update(val)
# continue
display('#' * level + f" {key}")
for cmds in val:
if type(cmds) == type(""):
cmds = {'md': cmds}
if type(cmds) == type({}):
for key1, val1 in cmds.items():
if key1.find('()') != -1:
self.analysis_func(key1, val1)
if key1 == 'md':
display("\n" + val1 + '\n')
elif key1 == 'py':
ccmpiled = compile(val1, 'py_vars', 'exec')
exec(ccmpiled, globals(), locals())
elif key1 == 'an':
self.analysis_func(val1)
elif key1 == 'file':
with open(val1) as f:
display("\n" + f.read() + '\n')
if type(val1) == type([]):
# recursive call for lower heading levels
self.build_md_doc([cmds], level + 1)
from px_build_doc.util import display, FigureManager, Checklist
display("""
# Performix Structure
The following discribes the performix structure.
""")
figs = FigureManager()
figs.set_uml("uml",
"The Performix Pipeline",
height=5,
uml=r"""
@startuml
start
repeat
:read data;
:generate diagrams;
repeat while (more data?) is (yes)
->no;
stop
@enduml
""").display()
display("""
def print_test_info(test_size, label, instance):
display(f"""
This section desacribes the performance parameters for {label} {instance}. There are {test_size} probes in this test set.
""")
# except Exception as ex:
# errors.append(f"For {ft} type, unable to output score boxplot (may be incomplete rank information) due to exception: {ex}")
finger_types = fetch_var("finger_types")
is_identification = True # TODO implement way to identify identification
is_verification = not is_identification
errors = []
# Main Analysis.
for ft in finger_types:
dres = results[ft]
plot_label = f'{dres.label} {dres.dtype}'
display(f"# {fetch_var('label')} type {ft}")
print_test_info(dres.nr_probes, fetch_var('label'), ft)
print_performance(dres.accuracy_results, ft, errors)
display("## Analysis")
display("### Match distribution")
display(dedent("""
The match distribution shows the frequency of matches and non matches versus the threshold.
"""))
print_match_dist(dres.match_scores, dres.non_match_scores,
dres.threshold, ft, errors)
display("### Accuracy")
from px_build_doc.util import fetch_var, News, display
display('# News\n')
display(r'\twocolumn')
News().query(fetch_var("news")).display()
display(r'\onecolumn')
from textwrap import dedent
import pandas as pd
from px_build_doc.util import display, fetch_var, TableManager
import data
_, results, _ = data.get()
tables = TableManager()
finger_types = fetch_var("finger_types")
#Conclusion
display(dedent(f"""
# Conclusion
Performed analysis on {len(finger_types)} finger types.
"""))
conclsn_table = pd.DataFrame(columns=['Finger Type','Test Size (Probes)', 'FPR', 'AUC'])
for ft in finger_types:
dres = results[ft]
conclsn_table.loc[ft] = [ft, f'{dres.nr_probes}', f'{dres.accuracy_results.fpr_thresh}', f'{dres.accuracy_results.auc:.3f}']
tables.read_df(conclsn_table).display(f"Summary of analysis performed.")
# display(dedent(f"""
# * Analysis completed with {len(errors)} errors and {missing_score_df.shape[0]} records with missing score data.
# """))
version = 1.0
def analysis_func(self, analysis_name, func_params=None):
""" Process the analysis function name """
tables = self.tables # define vars avaliable to any analysis function
figs = self.figs
def get_data(source_name):
""" return the data from a query"""
if not source_name in self.global_data_dict:
raise_error("Error data source %s is not definied." %
analysis.use_data)
return self.global_data_dict[source_name]
def split_params(name):
""" split function and parameters
match func(a,b,c..):use_data
a function can be called as so:
```yaml
- an: pie_test(24):stats_fingers
```
"""
regex = r"(.*)\((.*)\)(:.*)*"
matches = re.search(regex, name)
func_params = ""
use_data = None
if matches:
name = matches.group(1)
func_params = matches.group(2)
use_data = matches.group(3)
if use_data is not None:
use_data = use_data[1:] # strip first char as its a :
return name, func_params, use_data
def run_function(var_params_dict, analysis_code, analysis_name, df):
""" run the analysis function in the current context"""
# note df provdied as local to analysis function
tables = self.tables
figs = self.figs
get_df = get_data
def set_observation(key, info):
if not key in self.observation:
self.observation[key] = []
self.observation[key].append(info)
observations = self.observation
_local = locals().copy()
_local = deep_update(_local, var_params_dict)
if type(analysis.params) == type([]):
for param in analysis.params:
if param not in _local:
p_var = param.split('=')
if len(p_var) == 1:
_local[param] = None
else:
_local[p_var[0]] = p_var[1]
#_local.update(var_params_dict)
ccmpiled = compile(analysis_code, 'py_vars', 'exec')
try:
exec(ccmpiled, globals(), _local)
except:
raise_error("Syntax error excuting function [%s]" %
analysis_name)
analysis_name, func_params_str, use_data = split_params(analysis_name)
defaults = {
'enabled': False,
"use_data": False,
'table': False,
"before": '',
"after": '',
"uml": False,
"py": False,
"after_py": False,
"params": False,
"plot": {
"labels": False,
"caption": False,
"height": 7,
"format": self.doc_format
},
"table": {
"variable": False,
"caption": "",
"height": 8,
"floatfmt": ()
}
}
analysis_dict = self.bp['analysis'][analysis_name]
if func_params is not None:
#analysis_dict.update(func_params)
analysis_dict = deep_update(analysis_dict, func_params)
else:
func_params = {}
analysis = self.to_obj(analysis_dict, defaults=defaults)
if use_data is not None and 'use_data' not in func_params: # data is specified excplicitly in the function call
analysis.use_data = use_data
if analysis.use_data:
df = get_data(analysis.use_data)
else:
df = None
display("\n" + analysis.before + "\n")
if analysis.py:
run_function(func_params, analysis.py, analysis_name, df)
#raise_error(str(analysis.plot))
self.handle_plot(figs, analysis)
self.handle_table(tables, analysis)
if analysis.after_py:
run_function(func_params, analysis.after_py, analysis_name, df)
#raise_error(str(analysis.plot))
self.handle_uml(figs, analysis)
display("\n" + analysis.after + "\n")
"""@ UML Test"""
from px_build_doc.util import display, FigureManager, Checklist
display('# Figure Demo')
display('## WBS demo')
display('A demo of a long WBS diagram')
figs = FigureManager()
figs.set_uml("wbs",
"A long caption two",
height=5,
uml=r"""
<style>
wbsDiagram {
Linecolor black
BackGroundColor white
LineColor green
}
}
</style>
* World 2
** Cluster 0:
*** clearview
*** enforcement
*** 550m
** Cluster 1:
*** face
*** recognition