def data_files(path):
if path.endswith('/'):
mimetype = request.accept_mimetypes.best_match([
'text/html',
'application/json',
'application/vnd.jupyter',
], 'text/html')
if mimetype == 'text/html':
fs = Filesystem(current_app.config['CWD'])
env = get_jinja2_env(config=current_app.config)
return env.get_template('landing.j2').render(_nb=os.path.basename(
current_app.config['IPYNB']), )
else:
data_fs = Filesystem(
Filesystem.join(current_app.config['DATA_DIR'], 'output'))
path += current_app.config['IPYNB']
if data_fs.exists(path):
return send_file(data_fs.open(path, 'rb'),
attachment_filename=os.path.basename(path))
else:
data_fs = Filesystem(
Filesystem.join(current_app.config['DATA_DIR'], 'output'))
if data_fs.exists(path):
return send_file(data_fs.open(path, 'rb'),
attachment_filename=os.path.basename(path))
abort(404)
def serve(app_path, **kwargs):
import os
import sys
import time
import appyter
import functools
import logging
logger = logging.getLogger(__name__)
from subprocess import Popen
from appyter.ext.fs import Filesystem
from appyter.context import get_env, get_jinja2_env, find_blueprints, get_appyter_directory
from appyter.util import join_routes
from appyter.profiles.default.filters.url_for import url_for
config = get_env(**kwargs)
logger.info(kwargs)
env = get_jinja2_env(config=config)
with Filesystem('tmpfs://') as tmp_fs:
logger.info(f"Working directory {tmp_fs.path()}")
#
logger.info(f"Pre-rendering pages...")
with Filesystem(config['CWD']).open(config['IPYNB']) as fr:
from appyter.parse.nb import nb_from_ipynb_io
nbtemplate = nb_from_ipynb_io(fr)
with tmp_fs.open('index.html', 'w') as fw:
from appyter.render.form import render_form_from_nbtemplate
fw.write(render_form_from_nbtemplate(env, nbtemplate))
with tmp_fs.open('index.json', 'w') as fw:
import json
from appyter.render.nbinspect import render_nbtemplate_json_from_nbtemplate
json.dump(render_nbtemplate_json_from_nbtemplate(env, nbtemplate),
fw)
with tmp_fs.open('landing.html', 'w') as fw:
env.get_template('landing.j2').stream(_nb=os.path.basename(
config['IPYNB']), ).dump(fw)
#
logger.info(f"Generating production config...")
with tmp_fs.open('supervisord.conf', 'w') as fw:
env.get_template('production/supervisord.conf.j2').stream(
_tmp_fs=tmp_fs, sys=sys, str=str).dump(fw)
with tmp_fs.open('nginx.conf', 'w') as fw:
env.get_template('production/nginx.conf.j2').stream(
_tmp_fs=tmp_fs,
os=os,
s3_to_url=s3_to_url,
get_appyter_directory=get_appyter_directory,
find_blueprints=find_blueprints,
).dump(fw)
logger.info(
f"Starting production instance at http://{config['HOST']}:{config['PORT']}{config['PREFIX']} ..."
)
with Popen(
['supervisord', '-n', '-c',
tmp_fs.path('supervisord.conf')]) as proc:
try:
sys.exit(proc.wait())
except KeyboardInterrupt:
proc.terminate()
sys.exit(proc.wait())
def nbconstruct(cwd, ipynb, context, output, **kwargs):
context = json.load(context)
env = get_jinja2_env(
config=get_env(cwd=cwd, ipynb=ipynb, mode='construct', **kwargs),
context=context,
)
nbtemplate = nb_from_ipynb_io(Filesystem(cwd).open(ipynb, 'r'))
nb = render_nb_from_nbtemplate(env, nbtemplate)
nb_to_ipynb_io(nb, output)
def get_fields():
''' Helper to get/cache fields even if we're on a different thread
'''
global _fields
if not _fields or current_app.config['DEBUG']:
fs = Filesystem(current_app.config['CWD'])
with fs.open(current_app.config['IPYNB'], 'r') as fr:
env = get_jinja2_env(config=current_app.config)
nbtemplate = nb_from_ipynb_io(fr)
_fields = render_nbtemplate_json_from_nbtemplate(env, nbtemplate)
return _fields
def get_index():
mimetype = request.accept_mimetypes.best_match([
'text/html',
'application/json',
'application/vnd.jupyter',
], 'text/html')
fs = Filesystem(current_app.config['CWD'])
if mimetype in {'text/html'}:
with fs.open(current_app.config['IPYNB'], 'r') as fr:
env = get_jinja2_env(config=current_app.config)
nbtemplate = nb_from_ipynb_io(fr)
return render_form_from_nbtemplate(env, nbtemplate)
elif mimetype in {'application/json'}:
with fs.open(current_app.config['IPYNB'], 'r') as fr:
env = get_jinja2_env(config=current_app.config)
nbtemplate = nb_from_ipynb_io(fr)
return jsonify(render_nbtemplate_json_from_nbtemplate(env, nbtemplate))
elif mimetype in {'application/vnd.jupyter'}:
return send_file(fs.open(current_app.config['IPYNB'], 'rb'), attachment_filename=current_app.config['IPYNB'], mimetype=mimetype)
else:
abort(404)
def prepare_results(data):
results_hash = sha1sum_dict(dict(ipynb=get_ipynb_hash(), data=data))
data_fs = Filesystem(current_app.config['DATA_DIR'])
results_path = Filesystem.join('output', results_hash)
if not data_fs.exists(
Filesystem.join(results_path, current_app.config['IPYNB'])):
# prepare files to be linked and update field to use filename
file_fields = {
field['args']['name']
for field in get_fields() if field['field'] == 'FileField'
}
links = []
files = {}
for file_field in file_fields:
if fdata := data.get(file_field):
content_hash, filename = fdata.split('/', maxsplit=1)
content_hash = sanitize_sha1sum(content_hash)
filename = secure_filepath(filename)
links.append((Filesystem.join('input', content_hash),
Filesystem.join(results_path, filename)))
files[filename] = filename
data[file_field] = filename
# construct notebook
env = get_jinja2_env(config=current_app.config,
context=data,
session=results_hash)
fs = Filesystem(current_app.config['CWD'])
with fs.open(current_app.config['IPYNB'], 'r') as fr:
nbtemplate = nb_from_ipynb_io(fr)
# in case of constraint failures, we'll fail here
nb = render_nb_from_nbtemplate(env, nbtemplate, files=files)
# actually link all input files into output directory
for src, dest in links:
data_fs.link(src, dest)
# write notebook
nbfile = Filesystem.join(results_path,
os.path.basename(current_app.config['IPYNB']))
with data_fs.open(nbfile, 'w') as fw:
nb_to_ipynb_io(nb, fw)
def dockerize(ipynb, cwd, output, **kwargs):
env = get_jinja2_env(config=get_env(cwd=cwd,
ipynb=ipynb,
mode='dockerize',
**kwargs), )
env.get_template('production/Dockerfile.j2').stream().dump(output)
import os, sys
sys.path.insert(0, os.path.realpath('..'))
from appyter.parse.nb import nb_from_ipynb_file
from appyter.render.nbconstruct import render_nb_from_nbtemplate
from appyter.render.form import render_form_from_nbtemplate
from appyter.context import get_jinja2_env, get_env
from appyter.parse.nbtemplate import parse_fields_from_nbtemplate
from appyter.parse.nb import parse_markdown
nbtemplate = nb_from_ipynb_file('example.ipynb')
config = get_env(cwd=os.path.dirname(__file__), ipynb='example.ipynb')
env = get_jinja2_env(config=config)
parse_fields_from_nbtemplate(env, nbtemplate)
env = get_jinja2_env(config=config)
render_form_from_nbtemplate(env, nbtemplate)
env = get_jinja2_env(
context={
'number_1': 8,
'operator': 'subtract',
},
config=config,
)
render_nb_from_nbtemplate(env, nbtemplate)
def init(_globals, verbose=False, ipynb='app.ipynb', mode='magic', **kwargs):
''' Initialize appyter magic.
Sets up a jinj2 environment and injects %%appyter magic into your environment.
:param _globals: (Dict[str, Any]) A callable with your globals for the purpose of injection, basically just: `lambda _=globals: _()`
:param verbose: (Optional[bool]) Expand exception reporting to be more verbose
'''
import os
import jinja2
import jinja2.meta
import traceback
from appyter.context import get_env, get_jinja2_env
env = get_jinja2_env(
config=get_env(verbose=verbose, ipynb=ipynb, mode=mode, **kwargs))
from IPython.core.magic import register_cell_magic
from IPython.display import display, Markdown, HTML
'''
register_cell_magic allows function to execute an entire cell with the following call structure:
```python
%%my_magic whatever
all
my
data
```
Results in a call:
```python
my_magic(
"whatever",
"""all
my
data"""
)
```
'''
@register_cell_magic
def appyter(line, cell):
''' Appyter Cell Magic: See Steps for more information.
Compile jinja2 into source code, and then evaluate that
source code.
'''
'''
Step 1. Render cell with jinja2, removing empty lines.
execute or display the results, modifying a copy of the
current python globals dict.
'''
global_internal = _globals()
cell_type = line.split('_')
try:
cell_lines = cell.splitlines()
try:
undeclared = jinja2.meta.find_undeclared_variables(
env.parse(cell))
if undeclared:
for lineno, cell_line in enumerate(cell_lines):
if any(v in cell_line for v in undeclared):
display(
HTML(
f"<div style='color: red'>{lineno}: {cell_line}</div>"
))
raise Exception(f"undeclared variable(s) {undeclared}")
template = env.from_string(cell)
template_rendered = template.render()
except Exception as e:
if getattr(e, 'lineno', None) is not None:
display(
HTML(
f"<div style='color: red'>{e.lineno}: {cell_lines[e.lineno-1]}</div>"
))
raise e
#
rendered_template_lines = list(
filter(None, map(str.rstrip, template_rendered.splitlines())))
try:
if len(rendered_template_lines) > 0:
if cell_type == ['markdown']:
display(Markdown(template_rendered))
elif 'code' in cell_type:
rendered = '\n'.join(rendered_template_lines[:-1])
rendered_last = rendered_template_lines[-1]
display(
Markdown('```python\n%s\n```' % ('\n'.join(
(rendered, rendered_last)))))
#
if 'eval' in cell_type:
exec(rendered, global_internal)
#
try:
display(eval(rendered_last, global_internal))
except Exception as e:
setattr(e, 'lineno',
len(rendered_template_lines))
raise e
elif 'exec' in cell_type:
exec('\n'.join((rendered, rendered_last)),
global_internal)
else:
raise Exception('Unrecognized appyter cell_type')
#
except Exception as e:
if getattr(e, 'lineno', None) is not None:
display(
HTML(
f"<div style='color: red'>{e.lineno}: {rendered_template_lines[e.lineno-1]}</div>"
))
raise e
except Exception as e:
display(HTML(f"<div style='color: red'>Error: {e}</div>"))
if verbose:
traceback.print_exc()
return
'''
Step 2. Check for new variables in the internal global
and pass them to the python global scope. Check for
new variables in the jinja2 template and pass them to the
template environment global so they are available in
future jinja2 calls.
'''
for k, v in global_internal.items():
if not k.startswith('_'):
_globals()[k] = v
for k, v in template.module.__dict__.items():
if not k.startswith('_'):
env.globals[k] = v
def nbinspect(cwd, ipynb, output, **kwargs):
env = get_jinja2_env(
config=get_env(cwd=cwd, ipynb=ipynb, mode='inspect', **kwargs))
nbtemplate = nb_from_ipynb_io(Filesystem(cwd).open(ipynb, 'r'))
fields = render_nbtemplate_json_from_nbtemplate(env, nbtemplate)
json.dump(fields, output)
