def test_healthcare_task_nb():
"""
Tests whether this task notebook works
"""
matplotlib.use(
"template") # Disable plt.show when executing nb as part of this test
Notebook.load(HEALTHCARE_TASK_NB)
def test_demo_nb():
"""
Tests whether the demo notebook works
"""
matplotlib.use(
"template") # Disable plt.show when executing nb as part of this test
Notebook.load(DEMO_NB_FILE)
def test_compas_task_nb():
"""
Tests whether this task notebook works
"""
matplotlib.use(
"template") # Disable plt.show when executing nb as part of this test
Notebook.load(COMPAS_TASK_NB)
def test_adult_simple_task_nb():
"""
Tests whether this task notebook works
"""
matplotlib.use(
"template") # Disable plt.show when executing nb as part of this test
Notebook.load(ADULT_SIMPLE_TASK_NB)
def test_experiment_nb():
"""
Tests whether the experiment notebook works
"""
matplotlib.use(
"template") # Disable plt.show when executing nb as part of this test
Notebook.load(EXPERIMENT_NB_FILE)
def test_kalman_filter(self):
"""
Test that the Kalman filter notebook runs successfully and computes the correct solution
"""
with Notebook():
import examples.Kalman_filter
position_posteriors = examples.Kalman_filter.position_posteriors
factors = examples.Kalman_filter.factors
evidence_dict = examples.Kalman_filter.evidence_dict
marginal_vars = [p.var_names for p in position_posteriors]
joint = factors[0]
for factor in factors[1:]:
joint = joint.absorb(factor)
joint = joint.reduce(vrs=list(evidence_dict.keys()), values=list(evidence_dict.values()))
correct_marginals = []
for vrs in marginal_vars:
correct_marginal = joint.marginalize(vrs, keep=True)
correct_marginals.append(correct_marginal)
for actual_marginal, expected_marginal in zip(position_posteriors, correct_marginals):
# TODO: see why log_weight (and g) parameters are so different between the actual and expected factors.
self.assertTrue(np.allclose(actual_marginal.get_prec(), expected_marginal.get_prec()))
self.assertTrue(np.allclose(actual_marginal.get_h(), expected_marginal.get_h(), rtol=1.0e-5, atol=1.0e-5))
def test_slip_on_grass(self):
"""
Test that the slip_on_grass notebook runs successfully and computes the correct solution (checked in notebook)
:return:
"""
with Notebook():
import examples.slip_on_grass
def save2(self,*args,**kwargs):
if self.notebook.path.endswith(".ipynb") and self.notebook_test.path.endswith(".ipynb"):
with Notebook(lazy=True):
#try:
path,tail=os.path.split(self.notebook.path)
tail=tail[:tail.index(".")]
module_temp=import_module(tail)
nb=nbformat.read(self.notebook.path, as_version=4)
(body, resources) = html_exporter.from_notebook_node(nb)
self.notebook_html=body
#imported_notebooks_0[tail]N=module_temp
self.description = module_temp.__doc__
func_dict={}
for func_name in list(module_temp.in_out_def().keys()):
func_dict[func_name]=getattr(module_temp, func_name).__doc__
self.functions = func_dict
path,tail=os.path.split(self.notebook_test.path)
#with open(os.devnull, "w") as f, contextlib.redirect_stdout(f):
var=pytest.main([str(self.notebook_test.path)])
print(tail,":",var,', value: ',var.value)
nb2=nbformat.read(self.notebook_test.path, as_version=4)
(body2, resources2) = html_exporter.from_notebook_node(nb2)
self.notebook_test_html=body2
if var.value==0:
super().save(*args, **kwargs)
print('success')
return "success"
#except:
# pass
super().delete()
print('fail')
return "fail"
def test_sudoku(self):
"""
Test that the sudoku notebook runs successfully and computes the correct solution.
:return:
"""
with Notebook():
import examples.sudoku
infered_solution_array = examples.sudoku.infered_solution_array
correct_solution_array = examples.sudoku.correct_solution_array
self.assertTrue(np.array_equal(infered_solution_array, correct_solution_array))
def viewPipeline(request, pipe_slug):
context = {'active_view': 'pipelines'}
template_name = "pipelines/viewPipeline.html"
form_dict = {
'file_browse': FileInput,
'files_browse': FilesInput,
'variable': Variable,
'choice': Variable,
'text_input': TextInput,
'float_input': FloatInput,
'int_input': IntInput,
'table_param': TableParam,
'boolean': BooleanInput
}
nb_script_groups = NotebookModel.objects.values_list('notebook_group',
flat=True).distinct()
notebooks = NotebookModel.objects.all()
imported_notebooks = {}
if notebooks.exists():
for fldr in nb_script_groups:
sys.path.append(os.path.join(dir, fldr.upper()))
for nb in notebooks:
path, tail = os.path.split(nb.notebook.path)
tail = tail[:tail.index(".")]
if tail not in sys.modules:
with Notebook():
module_temp = import_module(tail)
imported_notebooks[nb.name] = module_temp
#print(imported_notebooks)
else:
imported_notebooks[nb.name] = i = __import__(tail,
fromlist=[''])
pipe = Pipeline.objects.get(slug=pipe_slug)
pipe_steps = PipelineStep.objects.filter(pipeline_id=pipe.pk)
step_dicts = []
pre_pre_form = '<div><h6 class="text-muted my-0 toggle">Param '
pre_form = ' \/</h6><div class="my-1" style="display:none">'
post_form = '</div></div>'
pipeline_valid = True
pipeline_out = {}
outputs = {}
out_choices_list = []
context['content_dicts1'] = {"name": pipe.title}
tables = ModelDefinition.objects.all()
table_groups = [*set([t.table_group for t in tables])]
unique_table_groups = [(i, t_group)
for i, t_group in enumerate(table_groups)]
table_group_form = Variable(choices_param=unique_table_groups)
context['content_dicts1']['table_groups'] = table_group_form.as_p(
).replace('<label for="id_form_val">Form val:</label>',
'').replace('ajax-using_functions', 'table_group')
tables_in_group = [(i, t.name) for i, t in enumerate(tables)
if t.table_group == unique_table_groups[0][1]]
tables_in_group_form = Variable(choices_param=tables_in_group)
context['content_dicts1']['tables_in_group'] = tables_in_group_form.as_p(
).replace('<label for="id_form_val">Form val:</label>',
'').replace('ajax-using_functions', 'table_select')
columns_in_table = [(i, f['name']) for i, f in enumerate(
ModelDefinition.objects.get(
name=tables_in_group[0][1]).definition["fields"])]
columns_form = Variable(choices_param=columns_in_table)
context['content_dicts1']['columns_in_table'] = columns_form.as_p(
).replace('<label for="id_form_val">Form val:</label>',
'').replace('ajax-using_functions', 'search_column')
datas = Data.objects.filter(model_definition=ModelDefinition.objects.get(
name=tables_in_group[0][1]))
rows = []
fields = [
t['name'] for t in ModelDefinition.objects.get(
name=tables_in_group[0][1]).definition['fields']
]
for data in datas:
data_dict = {}
for field in fields:
if field in list(data.data.keys()):
data_dict[field] = data.data[field]
rows.append(data_dict)
context['content_dicts1']['rows'] = rows
context['content_dicts1']['table_data_columns'] = fields
#context['content_dicts1']=[context['content_dicts1']]
context['content_template_name1'] = "pipelines/FirstTableCard.html"
if request.is_ajax():
if request.POST.get('ajax_type') == "table_select":
table = request.POST.get('ajax_table')
table = ModelDefinition.objects.get(name=table)
datas = Data.objects.filter(model_definition=table)
rows = []
fields = [t['name'] for t in table.definition['fields']]
for data in datas:
data_dict = {}
for field in fields:
if field in list(data.data.keys()):
data_dict[field] = data.data[field]
rows.append(data_dict)
table_head_str = ""
table_body_str = ""
for field in fields:
table_head_str += '<th scope="col">' + field + '</th>'
search_cols = Variable(
choices_param=[(i, f) for i, f in enumerate(fields)])
search_cols = search_cols.as_p().replace(
'<label for="id_form_val">Form val:</label>',
'').replace('ajax-using_functions', 'search_column')
print(fields, rows, search_cols)
for row in rows:
table_body_str += "<tr>"
for field in fields:
table_body_str += "<td>" + str(row[field]) + "</td>"
table_body_str += "</tr>"
return JsonResponse(
{
'table_body': table_body_str,
"table_head": table_head_str,
"search_cols": search_cols
},
status=200)
elif request.POST.get('ajax_type') == 'table_group_choice':
table_group = request.POST.get('ajax_table_group')
tables_in_group = [(i, t.name) for i, t in enumerate(tables)
if t.table_group == table_group]
tables_in_group_form = Variable(
choices_param=tables_in_group).as_p().replace(
'<label for="id_form_val">Form val:</label>',
'').replace('ajax-using_functions', 'table_select')
columns_in_table = [(i, f['name']) for i, f in enumerate(
ModelDefinition.objects.get(
name=tables_in_group[0][1]).definition["fields"])]
columns_form = Variable(
choices_param=columns_in_table).as_p().replace(
'<label for="id_form_val">Form val:</label>',
'').replace('ajax-using_functions', 'search_column')
rows = []
fields = [
t['name'] for t in ModelDefinition.objects.get(
name=tables_in_group[0][1]).definition['fields']
]
datas = Data.objects.filter(
model_definition=ModelDefinition.objects.get(
name=tables_in_group[0][1]))
for data in datas:
data_dict = {}
for field in fields:
if field in list(data.data.keys()):
data_dict[field] = data.data[field]
rows.append(data_dict)
table_head_str = ""
table_body_str = ""
for field in fields:
table_head_str += '<th scope="col">' + field + '</th>'
search_cols = Variable(
choices_param=[(i, f) for i, f in enumerate(fields)])
search_cols = search_cols.as_p().replace(
'<label for="id_form_val">Form val:</label>',
'').replace('ajax-using_functions', 'search_column')
print(fields, rows, search_cols)
for row in rows:
table_body_str += "<tr>"
for field in fields:
table_body_str += "<td>" + str(row[field]) + "</td>"
table_body_str += "</tr>"
return JsonResponse(
{
'table_body': table_body_str,
"table_head": table_head_str,
'tables_in_group': tables_in_group_form,
"search_cols": columns_form
},
status=200)
step_output_names = []
for step_iter, step in enumerate(pipe_steps):
input_dict = imported_notebooks[step.module.name].in_out_def()[
step.function]['inputs']
output_dict = imported_notebooks[step.module.name].in_out_def()[
step.function]['outputs']
input_form_part = ""
outputs[step.step_num] = [step.module.name, step.function, step.output]
pipeline_out[step.step_num] = {}
print(request.FILES)
replace_name_include_outputs = str(
step.step_num) + '_' + step.module.name + '_' + 'include_output'
step_output_names.append(replace_name_include_outputs)
if request.method == 'POST':
inc_out_form = Include_Output(data={
"include_output":
request.POST.get(replace_name_include_outputs)
}).as_p().replace('name="include_output"',
'name="' + replace_name_include_outputs + '"')
else:
if step_iter + 1 == len(pipe_steps):
inc_out_form = Include_Output(data={
"include_output": 'on'
}).as_p().replace(
'name="include_output"',
'name="' + replace_name_include_outputs + '"')
else:
inc_out_form = Include_Output().as_p().replace(
'name="include_output"',
'name="' + replace_name_include_outputs + '"')
for param, type2 in input_dict.items():
input_var = str()
replace_name = str(
step.step_num
) + '_' + step.module.name + '_' + step.function + '_' + param + '_' + type2
if replace_name in list(request.FILES.keys()):
#print(type2)
if type2 == "files_browse":
pipeline_out[step.step_num][param] = 'FILES' + replace_name
else:
pipeline_out[step.step_num][param] = 'FILE' + replace_name
if type2 == "table_param":
form = form_dict[type2](choices_param=[])
elif 'choice' in type2:
choicelist = type2[type2.index("[") +
1:type2.index("]")].split(',')
#print(choicelist)
choicetuplelist = [(i, val)
for i, val in enumerate(choicelist)]
form = form_dict['choice'](choices_param=choicetuplelist)
elif type2 != "variable":
form = form_dict[type2]()
else:
out_choices_tuple = tuple(out_choices_list)
form = form_dict[type2](choices_param=out_choices_tuple)
if request.method == 'POST' and type2 not in [
'file_browse', 'files_browse'
]:
input_var = request.POST.get(replace_name)
if type2 not in ['files_browse']:
if type2 == "table_param":
tuple_param = str(
input_var)[str(input_var).index('--') + 2:]
out_choices_list_2 = [(input_var, tuple_param)]
out_choices_tuple_2 = tuple(out_choices_list_2)
form = form_dict[type2](
data={
'form_val': input_var
},
choices_param=out_choices_tuple_2)
elif 'choice' in type2:
choicelist = type2[type2.index("[") +
1:type2.index("]")].split(',')
#print(choicelist)
choicetuplelist = [(i, val)
for i, val in enumerate(choicelist)]
form = form_dict['choice'](
data={
'form_val': input_var
},
choices_param=choicetuplelist)
elif type2 != "variable":
form = form_dict[type2](data={'form_val': input_var})
else:
out_choices_tuple = tuple(out_choices_list)
form = form_dict[type2](
data={
'form_val': input_var
},
choices_param=out_choices_tuple)
if form.is_valid():
if type2 == "variable":
pipeline_out[
step.step_num][param] = out_choices_tuple[int(
form.cleaned_data['form_val'])][1]
elif 'choice' in type2:
pipeline_out[
step.step_num][param] = choicetuplelist[int(
form.cleaned_data['form_val'])][1]
elif type2 == "table_param":
pipeline_out[step.step_num][param] = tuple_param[
tuple_param.index('//') + 2:]
else:
pipeline_out[step.step_num][
param] = form.cleaned_data['form_val']
else:
pipeline_valid = False
input_form_part += pre_pre_form + param + pre_form + form.as_p(
).replace('<label for="id_form_val">Form val:</label>',
'').replace('name="form_val"',
'name="' + replace_name + '"') + post_form
for name_out, typ_out in output_dict.items():
if typ_out == "variable" or typ_out == 'graph':
out_choices_list.append((len(out_choices_list),
step.output + '_out/var_' + name_out))
step_dicts.append({
'form':
input_form_part,
'include_output':
inc_out_form,
'output':
step.output,
'module_function':
step.module.name + '/' + step.function,
'description':
step.description,
'step_class_title':
str(step.step_num).replace(" ", "")
})
#print(pipeline_out)
context['content_template_name'] = "pipelines/usePipeStep.html"
context['content_dicts'] = step_dicts
#print(step_dicts)
if request.method == 'POST' and pipeline_valid:
context2 = {}
context2['graphs'] = []
context2['graphs2'] = []
context2["vars"] = []
context2['table_var'] = []
context2['active'] = []
context2['graph'] = []
context2['table'] = []
context2["variable"] = []
context["out_zip"] = ""
graph_names = []
table_names = []
output_vals = {}
#print(pipeline_out)
#include_outputs=[]
for s_num in range(len(pipe_steps)):
#include_outputs.append(request.POST.get(step_output_names[s_num]))
for name, val in pipeline_out[s_num + 1].items():
if type(val) in [type(True), int, float]:
pass
elif "_out/var_" in val:
out_name = val[val.index("_out/var_"):].replace(
"_out/var_", "")
out_prefix = val[:val.index("_out/var_")]
step_to_use = 0
#outputs[step.step_num]=[step.module,step.function,step.output]
for step_number, li in outputs.items():
if li[2] == out_prefix:
#print('found step')
step_to_use = step_number
pipeline_out[s_num + 1][name] = output_vals[
outputs[step_to_use][2]]['values'][out_name]
elif "FILES" in val:
pipeline_out[s_num + 1][name] = request.FILES.getlist(
val[val.index("FILES") + 5:])
elif "FILE" in val:
pipeline_out[s_num + 1][name] = request.FILES[
val[val.index("FILE") + 4:]]
out_func = getattr(imported_notebooks[outputs[s_num + 1][0]],
outputs[s_num + 1][1])
output_vals[outputs[s_num + 1][2]] = {
"types":
imported_notebooks[outputs[s_num + 1][0]].in_out_def()[outputs[
s_num + 1][1]]['outputs'],
"values":
out_func(**pipeline_out[s_num + 1])
}
out_types_vars = output_vals[outputs[s_num +
1][2]]["types"].items()
if request.POST.get(step_output_names[s_num]
) == 'on': #s_num+1 == len(pipe_steps):
#print("Hello")
for name, typ in out_types_vars:
context2['graph'].append(False)
context2['table'].append(False)
context2["variable"].append(False)
context2['active'].append(True)
graph_names.append("")
table_names.append("")
context2['graphs'].append("")
context2['table_var'].append("")
context2["vars"].append("")
context2['graphs2'].append("")
if typ == "graph":
context2['graph'][-1] = True
graph_names[-1] = name
context2['graphs'][-1] = json.dumps(
output_vals[outputs[s_num + 1][2]]["values"][name])
#print(context['graphs'])
elif typ == "variable":
out_var = output_vals[outputs[s_num +
1][2]]["values"][name]
if isinstance(out_var, pd.DataFrame):
context2['table'][-1] = True
table_names[-1] = name
context2['table_var'][-1] = {
'cols': [*out_var.columns],
'rows': [
out_var.iloc[i, :].to_list()
for i in range(len(out_var.index))
]
}
#print([out_var.iloc[i,:].to_list() for i in range(len(out_var.index))])
else:
context2["variable"][-1] = True
context2["vars"][-1] = {
"value": str(out_var),
"name": name
}
else:
context2["active"][-1] = False
context2['table_var'] = zip(context2['table_var'], table_names)
context['graphs'] = [gr for gr in context2['graphs'] if gr != ""]
context["out_zip"] = zip(context2["active"], context2["table"],
context2["table_var"], context2["variable"],
context2["vars"], context2["graph"],
context2["graphs"], graph_names)
#print(*context["out_zip"])
return render(request, template_name, context)
# if temp.endswith('.ipynb'):
# path,tail=os.path.split(temp)
# if not tail.startswith('test'):
# tail=tail[:tail.index(".")]
# module_temp=import_module(tail)
# print(tail)
# imported_notebooks[tail]=module_temp
# reload(module_temp)
notebooks = NotebookModel.objects.all()
for fldr in nb_script_groups:
sys.path.append(os.path.join(dir, fldr.upper()))
for nb in notebooks:
path, tail = os.path.split(nb.notebook.path)
tail = tail[:tail.index(".")]
if tail not in sys.modules:
with Notebook(lazy=True):
module_temp = import_module(tail)
imported_notebooks[nb.name] = module_temp
imported_notebooks_functions = {}
for tail, module_temp in imported_notebooks.items():
func_dict = {}
for func_name in list(module_temp.in_out_def().keys()):
func_dict[func_name] = getattr(module_temp, func_name)
imported_notebooks_functions[tail] = {
'functions': func_dict,
}
def fig_to_html(fig):
buf = io.BytesIO()
fig.savefig(buf, format='png', bbox_inches="tight")
Path('informal_script.py').write_text(
exporter.from_filename(__file__)[0])
import informal_script
assert informal_script, """The script was not created."""
print('nbconvert is complete.')
(__name__ == '__main__' and
Path(__file__).parts[-1].startswith('informal.')) and test_nbconvert_script()
# In[9]:
if __name__ == '__main__':
from importnb import Notebook, reload
get_ipython().run_line_magic('reload_ext', 'pidgin')
get_ipython().run_line_magic('pidgin', 'conventions markdown')
with Notebook():
try:
import informal
except:
from . import informal
reload(informal)
assert informal.__file__.endswith('.ipynb')
# In[10]:
# 1. __[If a notebook is hypothesis then it tests something.](informal.ipynb)__
# 2. ~~[If a notebook is a hypothesis then it can be tested.](formal.ipynb)~~
# 3. ~~[If a notebook is a hypothesis then it will be tested.](continuous.ipynb)~~
#
def test_demo_nb():
"""
Tests whether the .py version of the inspector works
"""
Notebook.load(DEMO_NB_FILE)