def showdocs(uid, title=""):
if uid not in DOCS:
display(Markdown("No documentation found for `%s`"%uid))
return
d = DOCS[uid]
if len(title):
title += " "
display(Markdown("# %s%s" % (title, d['name'])))
display(Markdown(d['desc']))
if 'fn' in d:
# display the figures.
files = Path(BASEDIR, 'figures').glob("%s.png" % d['fn'])
for f in sorted(files,key=lambda x:x.name):
#print(f.name)
display(Image(filename=f))
# citations
md = []
if 'refs' in d and len(d['refs']):
md += ["# References"]
md += ["\n".join("+ %s" % z for z in d['refs'])]
display(Markdown("\n\n\n\n".join(md)))
def summary_stats():
df = pd.read_csv(output_docs['file'])
display(Markdown('#### Breakdown of mediums in the Local News Dataset'))
display(df.medium.value_counts().to_frame())
display(Markdown(''))
display(Markdown('#### Breakdown of data sources in the Local News Dataset'))
display(df.source.value_counts().to_frame())
display(Markdown('The `User Input` entires are custom additions added from the contents of [this JSON file](https://github.com/yinleon/LocalNewsDataset/blob/master/data/custom_additions.json) and added to the dataset in `merge.py`'))
display(Markdown(''))
def plot_county(county, state):
fips = county_to_fips[(county, state)]
pop = fips_to_pop[fips]
display(Markdown(f"## {county}, {state}"))
ll_plot(df_counties.query("total_cases > 50"),
geos=[(county, state)],
max_pop=(county, pop))
display(
Markdown(
f"{county}'s current decade time is {decade_time(county, state)} days"
))
display(Markdown(f"{county}'s population {pop/1e6:.2f} million"))
def generate_intro():
'''
Markdown for the Introduction of the Data Specs section
'''
display(Markdown(intro_markdown))
table_of_contents = '### Intermediates \n'
for dataset in docs:
filename = os.path.basename(dataset['file']).split('.')[0]
table_of_contents += f" - [{filename}.tsv](#{filename})\n"
table_of_contents += end_of_table_of_contents
display(Markdown(table_of_contents))
display(Markdown('<hr>'))
def show(self):
if self.notebook:
output = widgets.Output()
with output:
display(Markdown(f'## {self.name}'))
if self.prefix: display(Markdown(self.prefix))
display(self.content)
if self.appendix: display(Markdown(self.appendix))
return output
else:
print(f'\n{self.name}')
if self.prefix: print(self.prefix)
print(self.content)
if self.appendix: print(self.appendix)
def on_button_clicked(b):
output.clear_output()
with output:
# get password
pwd = myPwd if pwd_exists else wPwd.value
if not pwd:
print('Please write your password first !')
return
# connect to rafal
session.connect(url=wServer.value,
login=user,
pwd=pwd,
proxies=proxies)
environ['jpwd'] = pwd
del pwd
# Python version
print(f'Python version: {sys.version}')
print(f'Pandas version: {pd.__version__}')
swagger_link = Markdown(
f"[Swagger on {session.url}]({session.url}/docs/swagger-ui/index.html?url=/assets/swagger.json#/)"
)
display(swagger_link)
# list all modules
modules = session.request(endPoint='/modules',
http='GET',
verbose=True)
print('Modules found :\n', modules)
def jhelp(function, full=False):
"""
Print a nice looking help string based on the name of a declared function. By default print the function definition and description
* full
If True, the help string will included a description of all arguments
"""
try:
# For some reason signature is not aways importable. In these cases the build-in help in invoqued
from inspect import signature, isfunction, ismethod
if isfunction(function) or ismethod(function):
name = function.__name__.strip()
sig = str(signature(function)).strip()
display(HTML("<b>{}</b> {}".format(name, sig)))
if function.__doc__:
for line in function.__doc__.split("\n"):
line = line.strip()
if not full and line.startswith("*"):
break
display(Markdown(line.strip()))
else:
jprint("{} is not a function".format(function))
except Exception:
help(function)
def plot_text(model, doc, filter_category=None, raw_html=False):
indices, categories, length = model.processor(doc)
markdown = ''
for index, category in zip(indices, categories):
if index == 0:
continue
category = category.item()
token = model.processor.vocabulary.tokens[index]
color = None
weight = None
if filter_category is not None:
if filter_category == category:
color = CATEGORY_PALLETTE[category]
weight = round(model.processor.vocabulary.index2idf(index), 2)
else:
if category != -1:
color = CATEGORY_PALLETTE[category]
weight = round(model.processor.vocabulary.index2idf(index), 2)
if color:
markdown += f'<span style="color:{color}"><b>{token} [{weight}, C{category}]</b></span>'
else:
markdown += token
markdown += ' '
if raw_html:
return markdown
return display(Markdown(markdown))
def on_submit(self, button):
self._submit.disabled = True
self.form.reset()
try:
for widget in self.widgets:
value = widget.value
field = self.form[widget.field_name]
if isinstance(field, FileField):
try:
name, data = next(iter(value.items()))
except StopIteration:
value = None
else:
value = self.client.upload_file(
io.BytesIO(data['content']), name, field.media_type
or 'text/plain')
self.form.insert({field.name: value})
cleaned_data = self.form.validate()
job_id = self.form.submit()
except FormValidationError:
raise
else:
self.jobs[job_id] = {
'inputs': cleaned_data,
'state': JobState.PENDING,
'id': job_id,
'results': []
}
display(Markdown(f'job *{job_id}* submitted successfully.'))
asyncio.create_task(self.monitor_job_state(job_id))
finally:
self._submit.disabled = False
def respond_notebook(self):
display(
Markdown(
"Response Set as Notebook !",
metadata={"naas_api": True, "naas_type": t_notebook},
)
)
def show_test(elt, search=True, markdown: bool = True) -> str:
"Show associated tests for a fastai function/class"
fn_name = nbdoc.fn_name(elt)
md = ''
db_matches = [get_links(t) for t in lookup_db(elt)]
md += tests2markdown(ifnone(db_matches, []), 'This tests')
if search:
try:
direct, related = find_tests(elt)
direct = [get_links(t) for t in direct]
related = [get_links(t) for t in related]
direct = list(set(direct) - set(db_matches))
related = list(set(related) - set(db_matches) - set(direct))
md += tests2markdown(direct, 'Direct tests')
md += tests2markdown(related, 'Related tests')
except OSError as e:
print(
'Could not find fastai/tests folder. If you installed from conda, please install developer build instead.'
)
if len(md) == 0: md = f'No tests found for `{fn_name}`'
else: md = f'Tests found for `{fn_name}`:' + md
if markdown: display(Markdown(md))
else: return md
def show_doc(elt,
doc_string: bool = True,
full_name: str = None,
arg_comments: dict = None,
title_level=None,
alt_doc_string: str = '',
ignore_warn: bool = False,
markdown=True):
"Show documentation for element `elt`. Supported types: class, Callable, and enum."
arg_comments = ifnone(arg_comments, {})
anchor_id = full_name or get_anchor(elt)
elt = getattr(elt, '__func__', elt)
full_name = full_name or fn_name(elt)
if inspect.isclass(elt):
if is_enum(elt.__class__): name, args = get_enum_doc(elt, full_name)
else: name, args = get_cls_doc(elt, full_name)
elif isinstance(elt, Callable): name, args = format_ft_def(elt, full_name)
else: raise Exception(f'doc definition not supported for {full_name}')
source_link = get_function_source(elt) if is_fastai_class(elt) else ""
title_level = ifnone(title_level, 2 if inspect.isclass(elt) else 4)
doc = f'<h{title_level} id="{anchor_id}">{name}{source_link}</h{title_level}>'
doc += f'\n\n> {args}'
if doc_string and (inspect.getdoc(elt) or arg_comments):
doc += format_docstring(elt, arg_comments, alt_doc_string,
ignore_warn) + ' '
if markdown: display(Markdown(doc))
else: return doc
def show_doc(elt,
doc_string: bool = True,
full_name: str = None,
arg_comments: dict = None,
title_level=None,
alt_doc_string: str = '',
ignore_warn: bool = False,
markdown=True,
show_tests=False):
"Show documentation for element `elt`. Supported types: class, Callable, and enum."
arg_comments = ifnone(arg_comments, {})
anchor_id = full_name or get_anchor(elt)
elt = getattr(elt, '__func__', elt)
full_name = full_name or fn_name(elt)
if inspect.isclass(elt):
if is_enum(elt.__class__): name, args = get_enum_doc(elt, full_name)
else: name, args = get_cls_doc(elt, full_name)
elif isinstance(elt, Callable): name, args = format_ft_def(elt, full_name)
else: raise Exception(f'doc definition not supported for {full_name}')
source_link = get_function_source(elt) if is_fastai_class(elt) else ""
test_link, test_modal = get_pytest_html(
elt, anchor_id=anchor_id, inline=True) if show_tests else ('', '')
title_level = ifnone(title_level, 2 if inspect.isclass(elt) else 4)
doc = f'<h{title_level} id="{anchor_id}">{name}{source_link}{test_link}</h{title_level}>'
doc += f'\n\n> {args}'
if doc_string and (inspect.getdoc(elt) or arg_comments):
doc += format_docstring(elt, arg_comments, alt_doc_string,
ignore_warn) + ' '
doc += '\n\n'
doc += f'{test_modal}' # hidden popup for tests doc. appending separately so it doesn't inherit css from <h{title_level}>
if markdown: display(Markdown(doc))
else: return doc
def doctest(elt):
"Inline notebook popup for `show_test`"
md = show_test(elt, markdown=False)
output = HTMLExporter().markdown2html(md)
try:
page.page({'text/html': output})
except:
display(Markdown(md))
def display_moment_equations(equations, print_missing=True):
for (fM, dfMdt) in equations:
display_moment_equation(fM, dfMdt)
if print_missing:
missing = get_missing_moments(equations)
if missing:
display(Markdown('**The system is not closed!** Moment equations are missing for:'))
display(missing)
def doctest(elt):
"Inline notebook popup for `show_test`"
md = build_tests_markdown(elt)
output = nbdoc.md2html(md)
try:
page.page({"text/html": output})
except:
display(Markdown(md))
def printmd(string: str):
'''
Markdown printout in Jupyter
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
prints a ``string`` that contains markdown (or more precisely, can contain markdown) in Jupyter cell output
'''
display(Markdown(string))
def doctest(elt):
"Inline notebook popup for `show_test`"
md = ''.join(build_tests_markdown(elt))
output = HTMLExporter().markdown2html(md)
try:
page.page({'text/html': output})
except:
display(Markdown(md))
def plot_county_norm(county, state):
fips = county_to_fips[(county, state)]
pop = fips_to_pop[fips]
display(Markdown(f"## {county}, {state}"))
dft = df_counties.query(
f"state=='{state}' and county=='{county}' and new_zombies > 0"
).sort_values("date")
ll_plot2(
dft.zombies,
dft.new_zombies,
label=f"{county}, {state}",
)
display(
Markdown(
f"{county}'s current decade time is {decade_time(county, state)} days"
))
display(Markdown(f"{county}'s population {pop/1e6:.2f} million"))
plt.show()
def respond_file(self, path):
abs_path = os.path.abspath(path)
naas_type = mimetypes.guess_type(abs_path)[0]
display(Markdown("Response Set as File, preview below: "))
display(
JSON(
{"path": abs_path}, metadata={"naas_api": True, "naas_type": naas_type}
)
)
def list_watch_command(self):
header = "| ID | Expression | Value |\n"
split = "|---|---|---|\n"
template = "|{}|`{}`|`{!r}`|\n"
wpstr = header + split
for wp in self._debugger.watchpoints:
wpstr += template.format(*wp)
display(Markdown(wpstr))
async def monitor_job_state(self, job_id):
job = self.jobs[job_id]
job['state'] = self.client.get_job_state(job_id)
while not job['state'].is_finished():
await asyncio.sleep(1)
job['state'] = self.client.get_job_state(job_id)
job['results'] = self.client.get_job_results(job_id)
display(
Markdown(
f'Job *{job_id}* finished with state {job["state"].name}.'))
def show_doc(elt, doc_string=True, full_name=None, arg_comments={}, alt_doc_string=''):
if full_name is None: full_name = elt.__name__
if inspect.isclass(elt):
if is_enum(elt.__class__): doc = get_enum_doc(elt, full_name, arg_comments)
else: doc = get_cls_doc(elt, full_name, arg_comments)
elif inspect.isfunction(elt): doc = get_ft_doc(elt, full_name, arg_comments)
link = f'<a id={full_name}></a>'
if doc_string and inspect.getdoc(elt) is not None: doc += '\n' + inspect.getdoc(elt)
if len(alt_doc_string) != 0: doc += '\n\n' + alt_doc_string
display(Markdown(link + doc))
def breakpoints_command(self, arg=""):
""" List all breakpoints """
header = "| ID | Type | Location | Status | Condition |\n"
split = "|---|---|---|---|---|\n"
bpstr = header + split
for bp in self._debugger.breakpoints:
bpstr += "|" + "|".join(bp) + "|\n"
display(Markdown(bpstr))
def ShowListOfDemos(self):
data = "## " + self.conf['list']['header'] + "\n"
for lab in self.conf['list']['labs']:
lab_dir = os.path.dirname(lab)
with open(lab_dir + "/README.md", "r") as readme:
cont = readme.read()
readme.close()
data += cont
title = cont[0]
data += "\n<a href='" + lab + "' target='_blank' class='big-jupyter-button'>" + self.conf[
'list']['messages']['goto'] + ": " + lab + "</a>\n"
display(Markdown(data))
def show_decade(county, state, tail=5):
_cols = [
'date', 'total_cases', 'new_cases', 'deaths', 'new_deaths', 'pop',
'decade_time', 'zombies', 'new_zombies', 'fips'
]
display(
Markdown(
f"### {county} {state} Decade Times (Past Two Weeks)\n Zombies = total_cases/100,000 people"
))
display(
df_counties.query(f"state=='{state}' and county=='{county}'").tail(
tail)[_cols])
def print_rows(recods, limit=None):
"""
print_rows(['a', 'b'], [{'a':1, 'b':2}])
"""
cols = {c for record in recods for c in record.keys()}
txt = '|'.join(cols) + '\n'
txt += '|'.join(['---' for i in range(len(cols))]) + '\n'
for i, row in enumerate(recods):
txt += '|'.join([str(row.get(f)).replace('|', '|')
for f in cols]) + '\n'
if limit and limit <= i:
break
display(Markdown(txt))
def get_module_toc(mod_name):
mod = import_mod(mod_name)
ft_names = mod.__all__ if hasattr(mod,'__all__') else get_ft_names(mod)
ft_names.sort(key = str.lower)
tabmat = ''
for ft_name in ft_names:
tabmat += f'- [{ft_name}](#{ft_name})\n'
elt = getattr(mod, ft_name)
if inspect.isclass(elt) and not is_enum(elt.__class__):
in_ft_names = get_inner_fts(elt)
for name in in_ft_names:
tabmat += f' - [{name}](#{name})\n'
display(Markdown(tabmat))
def display_automated_moment_closure_details(details):
from IPython.core.display import display, Markdown
from sympy import latex, Symbol
def _print(expr):
N = Moment(*([0] * _getNumSpecies(expr)))
return f'${latex(Expectation(expr.subs(N, Symbol("N"))))}$'
def _print_all(expressions):
all_but_last = ", ".join([_print(expr) for expr in expressions[:-1]])
last = _print(expressions[-1])
if len(expressions) == 0:
return ''
elif len(expressions) == 1:
return last
elif len(expressions) == 2:
return f'{all_but_last} and {last}'
else:
return f'{all_but_last}, and {last}'
display(
Markdown(
f'Computed moment equations for desired moments {_print_all(details.desired)}.'
))
if details.added:
display(
Markdown(
f'Equations were iteratively added for {_print_all(details.added)}.'
))
if details.gamma:
display(
Markdown(
f'Gamma closures were substituted for {_print_all(details.gamma)}.'
))
if details.meanfield:
display(
Markdown(
f'Mean-field closures were substituted for {_print_all(details.meanfield)}.'
))
def render(self) -> object:
"""Renders SQL as markdown when called in IPython environment.
Useful when combining explanation of different components of a SQL
script in IPython or similar environment.
"""
if isinstance(self.sql, list):
self.sql = ';\n\n'.join(self.sql)
display(Markdown(f"```mysql\n{self.sql}\n```"))
return None
