def to_hugo_safe_markdown(fpath):
notebook = nbformat.read(fpath, as_version=4)
pattern_display = re.compile(
r"(?:(?P<begin>\$\$)(?P<content>(?:.*?\r?\n?)*)(?P<end>\$\$))")
pattern_inline = re.compile(
r"(?:(?P<begin>\$)(?P<content>(?:.*?\r?\n?)*)(?P<end>\$))")
clean_newline = lambda x: re.sub(r"\\\\", r"\\newline", x.group())
clean_underline = lambda x: re.sub(r"\_", r"\\_", x.group())
for cell in notebook.cells:
if cell.cell_type == "markdown":
# clean display math
raw_s = cell.source
res = re.sub(pattern_display, clean_newline, raw_s)
res = re.sub(pattern_display, clean_underline, res)
res = re.sub(pattern_display, r"\\\[\g<content>\\\]", res)
# clean inline math
res = re.sub(pattern_inline, clean_underline, res)
cell.source = res
out_str, res = exporters.export(MarkdownExporter, notebook)
return out_str
def convert_notebook(cls, nb_path, dest, debug=False, **kwargs):
assert shutil.which("wkhtmltopdf") is not None, "Cannot export via HTML without wkhtmltopdf"
options = cls.default_options.copy()
options.update(kwargs)
nb = cls.load_notebook(nb_path, filtering=options["filtering"], pagebreaks=options["pagebreaks"])
if NBCONVERT_6:
nbconvert.TemplateExporter.extra_template_basedirs = [TEMPLATE_DIR]
orig_template_name = nbconvert.TemplateExporter.template_name
nbconvert.TemplateExporter.template_name = options["template"]
exporter = nbconvert.HTMLExporter()
if not NBCONVERT_6:
exporter.template_file = os.path.join(TEMPLATE_DIR, options["template"] + ".tpl")
if options["save_html"]:
html, _ = export(exporter, nb)
html_path = os.path.splitext(dest)[0] + ".html"
with open(html_path, "wb+") as f:
f.write(html.encode("utf-8"))
merger = PdfFileMerger()
for subnb in notebook_pdf_generator(nb):
html, _ = export(exporter, subnb)
pdfkit_options = {
'enable-local-file-access': None,
'quiet': '',
'print-media-type': '',
'javascript-delay': 2000
}
pdf_contents = pdfkit.from_string(html, False, options=pdfkit_options)
output = BytesIO()
output.write(pdf_contents)
output.seek(0)
merger.append(output, import_bookmarks=False)
merger.write(dest)
if NBCONVERT_6:
nbconvert.TemplateExporter.template_name = orig_template_name
def convert_single_notebook(self, notebook_filename, input_buffer=None):
notebook_pth = Path(notebook_filename)
self.log.info("Initializing single notebook resources for notebook: {}".format(notebook_filename)) #log
resources = self.init_single_notebook_resources(notebook_filename)
#calib
self.log.info("Calibrating notebook: {}".format(notebook_filename)) #log
from ipype.exporters import CalibratedNotebookExporter
resources.update(output_subdir=str(self._output / 'calib_notebooks'))
calib_output, resources = export(CalibratedNotebookExporter, notebook_filename, resources=resources)
self.writers['calib_writer'].write(calib_output, resources, notebook_name=notebook_pth.stem)
##############################################################
#exec ##########################################################
self.log.info("Executing notebook: {}".format(notebook_filename)) #log
results_subdir = Path(self._output / 'results')
results_subdir.mkdir(exist_ok=True) #create results subdir
from ipype.exporters import ExecutedNotebookExporter
exec_subdir = self._output / 'exec_notebooks'
resources.update(output_subdir=str(exec_subdir))
exec_output, resources = export(ExecutedNotebookExporter, notebook_filename, resources=resources)
self.writers['exec_writer'].write(exec_output, resources, notebook_name=notebook_pth.with_suffix('.exec').name)
executed_notebook_pth = exec_subdir / notebook_pth.with_suffix('.exec.ipynb').name
self.executed_notebooks.append(str(executed_notebook_pth))
##############################################################
#html
self.log.info("Exporting executed notebook {} to html..".format(str(executed_notebook_pth))) #log
from ipype.exporters import HTMLExporter
resources.update(output_subdir=str(self._output / 'html'), notebook_name=notebook_pth.stem)
exec_output_filelike = StringIO(exec_output)
html_output, resources = export(HTMLExporter, exec_output_filelike, resources=resources)
self.writers['html_writer'].write(html_output, resources, notebook_name=notebook_pth.stem)
def _tst_notebook(self, notebook_name):
notebook_filename = os.path.abspath(os.path.join(
self.this_file_directory, '..', '..', 'examples', notebook_name))
with open(notebook_filename) as f:
nb = nbformat.read(f, as_version=4)
python_nb, metadata = export(PythonExporter, nb)
# Remove magic lines manually
python_nb = '\n'.join([
line for line in python_nb.split('\n')
if 'get_ipython().run_line_magic(' not in line
])
exec(python_nb)
def _tst_notebook(self, notebook_name):
notebook_filename = os.path.abspath(
os.path.join(self.this_file_directory, '..', '..', 'examples',
notebook_name))
with open(notebook_filename) as f:
nb = nbformat.read(f, as_version=4)
python_nb, metadata = export(PythonExporter, nb)
# Remove magic lines manually
python_nb = '\n'.join([
line for line in python_nb.split('\n')
if 'get_ipython().run_line_magic(' not in line
])
exec(python_nb)
def init_notebooks(self):
filenames = []
pipeline_path = Path(self.config.pipeline)
output_path = Path(self.config.output_dir)
if pipeline_path.is_dir():
filenames = sorted(pipeline_path.glob('*.ipynb'))
elif is_zipfile(str(pipeline_path)):
filenames = get_notebooks_in_zip(str(pipeline_path))
elif pipeline_path.is_file():
if is_valid_notebook(str(pipeline_path)):
filenames = [pipeline_path] # list with one notebook
else:
raise Exception("Could not validate notebook")
_notebooks = [str(f) for f in filenames]
#copy notebooks to pipeline subfolder
copied_notebooks = []
for notebook in _notebooks:
notebook_pth = Path(notebook)
from nbconvert.exporters import NotebookExporter
pipeline_writer = FilesWriter(build_directory=str(output_path / 'pipeline'))
pipeline_output, resources = export(NotebookExporter, notebook, resources={})
pipeline_writer.write(pipeline_output, resources, notebook_name=notebook_pth.stem)
copied_notebook_pth = output_path / 'pipeline' / notebook_pth.name
assert copied_notebook_pth.exists()
copied_notebooks.append(str(copied_notebook_pth.absolute()))
self.notebooks = copied_notebooks
#add it into the Pipeline metadata
self.config.pipeline_notebooks = [nb for nb in self.notebooks]
self.config.executed_notebooks = []
from os import path
from sys import argv
from subprocess import run
in_file = argv[1]
param_file = argv[2]
f_path, f_name = path.split(in_file)
f_root, f_ext = path.splitext(f_name)
nb_out = path.join(f_path, f_root + '_processed.ipynb')
html_out = path.join(f_path, f_root + '_processed.html')
if (f_ext == '.py' and not path.exists(path.join(f_path, f_root + '.ipynb'))):
jupytext_call = ['jupytext', '--to', 'notebook', in_file]
run(jupytext_call)
in_file = path.join(f_path, f_root + '.ipynb')
with open(param_file, 'r') as f:
p = yaml.safe_load(f)
# inject params
nb = pm.execute_notebook(in_file, nb_out, parameters=p)
pp = ExecutePreprocessor()
pp.preprocess(nb, resources={'metadata': {'path': './'}})
html, resources = export(HTMLExporter, nb)
with open(html_out, 'w') as f:
f.write(html)
print(f'html written to {html_out}')
def prepare_deploy(ait_manifest: AITManifest,
ait_sdk_name: str,
base_dir: str,
requirements_path: str,
is_remote_deploy: bool = False):
"""
デプロイ準備をします。
Prepare for deployment.
Args:
ait_manifest (AITManifest) :
ait_manifestを指定します。
Specify the ait_manifest.
ait_sdk_name (str) :
ait_sdk_nameを指定します。
Specify ait_sdk_name.
base_dir (str) :
基点フォルダパスを指定します。
Specifies the base folder path.
requirements_path (str) :
requirements_pathを指定します。
Specify requirements_path.
is_remote_deploy (bool) :
aitのコンテナイメージをリモートから取得するか、ローカルでビルドするかを指定します。
Specifies whether the ait container image should be obtained remotely or built locally.
"""
base_path = Path(base_dir)
# convert my_ait.py
exporter = PythonExporter()
with open(str(base_path / 'my_ait.ipynb'), encoding='utf-8') as f:
nb = read(f, NO_CONVERT)
result = export(exporter, nb)
with open(str(base_path / 'my_ait.py'), mode='w', encoding='utf-8') as f:
f.write(result[0])
# move to deploy dir
shutil.move(str(base_path / 'my_ait.py'),
str(base_path / '../deploy/container/repository/my_ait.py'))
# copy ait.manifest to deploy dir
shutil.copyfile(
str(base_path / 'ait.manifest.json'),
str(base_path / '../deploy/container/repository/ait.manifest.json'))
# copy ait-sdk to deploy dir
shutil.copyfile(
str(base_path / f'./{ait_sdk_name}'),
str(base_path / f'../deploy/container/repository/{ait_sdk_name}'))
# copy requirements to deploy dir
requirements_file_name = Path(requirements_path).name
shutil.copyfile(
requirements_path,
str(base_path /
f'../deploy/container/repository/{requirements_file_name}'))
if is_remote_deploy:
keyword = 'remote'
else:
keyword = 'local'
# dag deploy
src_file = str(base_path / f'../template/dag_{keyword}.py')
dst_file = str(base_path / '../deploy/dag.py')
shutil.copyfile(src_file, dst_file)
_replace_template(ait_manifest.get_name(), ait_manifest.get_version(),
dst_file, 'utf-8')
# deploy script
src_file = str(base_path / f'../template/docker_deploy_{keyword}.bat')
dst_file = str(base_path / '../tool/docker_deploy.bat')
shutil.copyfile(src_file, dst_file)
_replace_template(ait_manifest.get_name(), ait_manifest.get_version(),
dst_file, 'shift-jis')
# license dockerfile
src_file = str(base_path / f'../template/dockerfile_license')
dst_file = str(base_path / '../deploy/container/dockerfile_license')
shutil.copyfile(src_file, dst_file)
_replace_template(ait_manifest.get_name(), ait_manifest.get_version(),
dst_file, 'utf-8')
# license script
src_file = str(base_path / f'../template/generate_thirdparty_notices.bat')
dst_file = str(base_path / '../tool/generate_thirdparty_notices.bat')
shutil.copyfile(src_file, dst_file)
_replace_template(ait_manifest.get_name(), ait_manifest.get_version(),
dst_file, 'shift-jis')
# ait deploy
if not is_remote_deploy:
# ait deploy(directory)
ait_deploy_name = ait_manifest.get_name(
) + '_' + ait_manifest.get_version()
ait_deploy_base_dir = str(
base_path /
'../tool/ait_deploy/container/ait') + '/' + ait_deploy_name
ait_deploy_archive_dir = str(
base_path / '../tool/ait_deploy/container/ait'
) + '/' + ait_deploy_name + '/' + ait_deploy_name + '/' + ait_deploy_name
copy_dir = str(base_path / '../deploy')
if os.path.exists(ait_deploy_base_dir):
shutil.rmtree(ait_deploy_base_dir)
shutil.copytree(copy_dir, ait_deploy_archive_dir)
shutil.copyfile(str(base_path / 'ait.manifest.json'),
ait_deploy_base_dir + '/ait.manifest.json')
delete_file = ait_deploy_archive_dir + '/container/dockerfile_license'
os.remove(delete_file)
# ait deploy(script)
src_file = str(base_path / f'../template/ait_deploy.bat')
dst_file = str(base_path / '../tool/ait_deploy.bat')
shutil.copyfile(src_file, dst_file)
_replace_template(ait_manifest.get_name(), ait_manifest.get_version(),
dst_file, 'shift-jis')
def run(notebook_filename, inps, reports_folder=None, key=None):
"""
:parameter notebook_filename:
:parameter inps:
A string with parameters replacing the content of the first cell
of the notebook
:parameter reports_folder:
The name of the folder where the report will be created
"""
with open(notebook_filename) as f:
nb = nbformat.read(f, as_version=4)
#
# replacing the content of the first cell
if len(inps):
nb['cells'][0]['source'] = inps
#
# prepare execution
ep = ExecutePreprocessor(timeout=100000, kernel_name='python')
ok = False
out = None
try:
#
# returns a 'nb node' and 'resources'
out = ep.preprocess(nb, {'metadata': {'path': './'}})
ok = True
except CellExecutionError as cee:
msg = 'Error executing the notebook "%s".\n\n' % notebook_filename
msg += 'See notebook for the traceback.'
if 'mpl_toolkits.basemap' in cee.traceback:
# TODO: remove this hack
raise unittest.SkipTest('Missing basemap')
elif 'conic lat_1 = -lat_2' in cee.traceback:
# TODO: remove this hack
raise unittest.SkipTest('Missing lat_1, lat_2 in proj=lcc')
else:
raise
finally:
#
# creating report
if reports_folder is not None and key is not None and out is not None:
#
# filtering cells
ocells = []
for cell in out[0]['cells']:
if cell['cell_type'] == 'code':
cell['source'] = ''
ocells.append(cell)
node = copy.deepcopy(out[0])
node['cells'] = ocells
#
# creating the exporter
# html_exporter = HTMLExporter()
html_exporter = HTMLExporter(html_exporter='nbextensions.tpl')
shtml = export(html_exporter, node)
#
#
filename = os.path.join(reports_folder, '%s.html' % key)
with open(filename, 'w') as f:
f.write(shtml[0])
print('Report in {:s}'.format(filename))
ok = True
return ok