def test_notebooks(MockDataset, tmpdir):
"""
Ensures all notebooks will run without error
"""
repo_root = os.path.join(os.path.dirname(__file__), "..")
notebooks = glob.glob(os.path.join(repo_root, "examples", "*.ipynb"))
logger.info(f"Found {len(notebooks)} notebooks to test.")
for notebook in notebooks:
logger.info(f"Testing notebook: {os.path.basename(notebook)}")
with open(notebook) as f:
nb = nbformat.read(f, as_version=4)
exporter = PythonExporter()
source, _meta = exporter.from_notebook_node(nb)
with open(os.path.join(tmpdir, "tmpmodule.py"), "w") as f:
f.writelines(source)
with open(os.path.join(tmpdir, "__init__.py"), "w") as f:
f.write("from .tmpmodule import *")
# Import this module to 'run' the code
module_dir = os.path.join(tmpdir, "..")
sys.path.insert(0, module_dir)
importlib.import_module(os.path.basename(tmpdir), ".")
sys.path.remove(module_dir)
def main(notebook_filename):
resources = {}
basename = os.path.basename(notebook_filename)
resources["unique_key"] = basename[:basename.rfind(".")]
exporter = PythonExporter()
output, resources = exporter.from_filename(notebook_filename,
resources=resources)
writer = StdoutWriter()
write_results = writer.write(output, resources)
def main(notebook_filename):
resources = {}
basename = os.path.basename(notebook_filename)
resources["unique_key"] = basename[:basename.rfind(".")]
exporter = PythonExporter()
output, resources = exporter.from_filename(notebook_filename,
resources=resources)
# Raise deprecations to errors
output = """
from pydrake.common.deprecation import DrakeDeprecationWarning
import warnings
warnings.simplefilter("error", DrakeDeprecationWarning)
""" + output
writer = StdoutWriter()
write_results = writer.write(output, resources)
def pytest_collect_file(file_path, path, parent):
"""
Collect tutorial files.
"""
if file_path.suffix == ".ipynb":
config = parent.config
if config.getoption("--flake8"):
# Convert .ipynb notebooks to plain .py files
def comment_lines(text, prefix="# "):
regex = re.compile(r".{1,80}(?:\s+|$)")
input_lines = text.split("\n")
output_lines = [
split_line.rstrip() for line in input_lines
for split_line in regex.findall(line)
]
output = prefix + ("\n" + prefix).join(output_lines)
return output.replace(prefix + "\n", prefix.rstrip(" ") + "\n")
def ipython2python(code):
return nbconvert.filters.ipython2python(code).rstrip(
"\n") + "\n"
filters = {
"comment_lines": comment_lines,
"ipython2python": ipython2python
}
exporter = PythonExporter(filters=filters)
exporter.exclude_input_prompt = True
code, _ = exporter.from_filename(file_path)
code = code.rstrip("\n") + "\n"
with open(file_path.with_suffix(".py"), "w",
encoding="utf-8") as f:
f.write(code)
# Collect the corresponding .py file
return pytest_flake8.pytest_collect_file(
file_path.with_suffix(".py"), None, parent)
else:
if not file_path.name.startswith("x"):
return TutorialFile.from_parent(parent=parent, path=file_path)
else:
return DoNothingFile.from_parent(parent=parent, path=file_path)
elif file_path.suffix == ".py":
assert not file_path.with_suffix(".ipynb").exists(), (
"Please run pytest on jupyter notebooks, not plain python files.")
return DoNothingFile.from_parent(parent=parent, path=file_path)
def pytest_collect_file(path, parent):
"""
Collect tutorial files.
"""
if path.ext == ".ipynb":
# Convert .ipynb notebooks to plain .py files
def comment_lines(text, prefix="# "):
regex = re.compile(r".{1,80}(?:\s+|$)")
input_lines = text.split("\n")
output_lines = [split_line.rstrip() for line in input_lines for split_line in regex.findall(line)]
output = prefix + ("\n" + prefix).join(output_lines)
return output.replace(prefix + "\n", prefix.rstrip(" ") + "\n")
def ipython2python(code):
return nbconvert.filters.ipython2python(code).rstrip("\n") + "\n"
filters = {
"comment_lines": comment_lines,
"ipython2python": ipython2python
}
exporter = PythonExporter(filters=filters)
exporter.exclude_input_prompt = True
code, _ = exporter.from_filename(path)
code = code.rstrip("\n") + "\n"
if MPI.COMM_WORLD.Get_rank() == 0:
with open(path.new(ext=".py"), "w", encoding="utf-8") as f:
f.write(code)
MPI.COMM_WORLD.Barrier()
# Collect the corresponding .py file
config = parent.config
if config.getoption("--flake8"):
return pytest_flake8.pytest_collect_file(path.new(ext=".py"), parent)
else:
if "data" not in path.dirname: # skip running mesh generation notebooks
if not path.basename.startswith("x"):
return TutorialFile.from_parent(parent=parent, fspath=path.new(ext=".py"))
else:
return DoNothingFile.from_parent(parent=parent, fspath=path.new(ext=".py"))
elif path.ext == ".py": # TODO remove after transition to ipynb is complete? assert never py files?
if (path.basename not in "conftest.py" # do not run pytest configuration file
or "data" not in path.dirname): # skip running mesh generation notebooks
if not path.basename.startswith("x"):
return TutorialFile.from_parent(parent=parent, fspath=path)
else:
return DoNothingFile.from_parent(parent=parent, fspath=path)
def convert_nb_to_script(path):
"""Convert notebook to Python script using the nbconvert API.
Before we were shelling out to call ``nbconvert``, but that
didn't always work because the cli tool is sometimes called
differently, e.g. ``nbconvert-3.6``. This should always work
and makes sure the right Python / nbconvert is used.
"""
# https://nbconvert.readthedocs.io/en/latest/execute_api.html#executing-notebooks-using-the-python-api-interface
# https://stackoverflow.com/a/38248141/498873
txt = path.read_text(encoding='utf-8')
nb = nbformat.reads(txt, nbformat.NO_CONVERT)
exporter = PythonExporter()
source, meta = exporter.from_notebook_node(nb)
path = path.with_suffix('.py')
log.info('Writing {}'.format(path))
path.write_text(source, encoding='utf-8')
def convert_nb_to_script(path):
"""Convert notebook to Python script using the nbconvert API.
Before we were shelling out to call ``nbconvert``, but that
didn't always work because the cli tool is sometimes called
differently, e.g. ``nbconvert-3.6``. This should always work
and makes sure the right Python / nbconvert is used.
"""
# https://nbconvert.readthedocs.io/en/latest/execute_api.html#executing-notebooks-using-the-python-api-interface
# https://stackoverflow.com/a/38248141/498873
txt = path.read_text(encoding='utf-8')
nb = nbformat.reads(txt, nbformat.NO_CONVERT)
exporter = PythonExporter()
source, meta = exporter.from_notebook_node(nb)
path = path.with_suffix('.py')
log.info('Writing {}'.format(path))
path.write_text(source, encoding='utf-8')
def nb_to_python(nb_path):
"""convert notebook to python script"""
exporter = PythonExporter()
output, resources = exporter.from_filename(nb_path)
return output
def nb_to_python(nb_path):
"""convert notebook to python script"""
exporter = PythonExporter()
output, resources = exporter.from_filename(nb_path)
return output
def prepare_deploy(ait_manifest: AITManifest,
ait_sdk_name: str,
base_dir: str,
requirements_path: str,
is_remote_deploy: bool = False):
"""
デプロイ準備をします。
Prepare for deployment.
Args:
ait_manifest (AITManifest) :
ait_manifestを指定します。
Specify the ait_manifest.
ait_sdk_name (str) :
ait_sdk_nameを指定します。
Specify ait_sdk_name.
base_dir (str) :
基点フォルダパスを指定します。
Specifies the base folder path.
requirements_path (str) :
requirements_pathを指定します。
Specify requirements_path.
is_remote_deploy (bool) :
aitのコンテナイメージをリモートから取得するか、ローカルでビルドするかを指定します。
Specifies whether the ait container image should be obtained remotely or built locally.
"""
base_path = Path(base_dir)
# convert my_ait.py
exporter = PythonExporter()
with open(str(base_path / 'my_ait.ipynb'), encoding='utf-8') as f:
nb = read(f, NO_CONVERT)
result = export(exporter, nb)
with open(str(base_path / 'my_ait.py'), mode='w', encoding='utf-8') as f:
f.write(result[0])
# move to deploy dir
shutil.move(str(base_path / 'my_ait.py'),
str(base_path / '../deploy/container/repository/my_ait.py'))
# copy ait.manifest to deploy dir
shutil.copyfile(
str(base_path / 'ait.manifest.json'),
str(base_path / '../deploy/container/repository/ait.manifest.json'))
# copy ait-sdk to deploy dir
shutil.copyfile(
str(base_path / f'./{ait_sdk_name}'),
str(base_path / f'../deploy/container/repository/{ait_sdk_name}'))
# copy requirements to deploy dir
requirements_file_name = Path(requirements_path).name
shutil.copyfile(
requirements_path,
str(base_path /
f'../deploy/container/repository/{requirements_file_name}'))
if is_remote_deploy:
keyword = 'remote'
else:
keyword = 'local'
# dag deploy
src_file = str(base_path / f'../template/dag_{keyword}.py')
dst_file = str(base_path / '../deploy/dag.py')
shutil.copyfile(src_file, dst_file)
_replace_template(ait_manifest.get_name(), ait_manifest.get_version(),
dst_file, 'utf-8')
# deploy script
src_file = str(base_path / f'../template/docker_deploy_{keyword}.bat')
dst_file = str(base_path / '../tool/docker_deploy.bat')
shutil.copyfile(src_file, dst_file)
_replace_template(ait_manifest.get_name(), ait_manifest.get_version(),
dst_file, 'shift-jis')
# license dockerfile
src_file = str(base_path / f'../template/dockerfile_license')
dst_file = str(base_path / '../deploy/container/dockerfile_license')
shutil.copyfile(src_file, dst_file)
_replace_template(ait_manifest.get_name(), ait_manifest.get_version(),
dst_file, 'utf-8')
# license script
src_file = str(base_path / f'../template/generate_thirdparty_notices.bat')
dst_file = str(base_path / '../tool/generate_thirdparty_notices.bat')
shutil.copyfile(src_file, dst_file)
_replace_template(ait_manifest.get_name(), ait_manifest.get_version(),
dst_file, 'shift-jis')
# ait deploy
if not is_remote_deploy:
# ait deploy(directory)
ait_deploy_name = ait_manifest.get_name(
) + '_' + ait_manifest.get_version()
ait_deploy_base_dir = str(
base_path /
'../tool/ait_deploy/container/ait') + '/' + ait_deploy_name
ait_deploy_archive_dir = str(
base_path / '../tool/ait_deploy/container/ait'
) + '/' + ait_deploy_name + '/' + ait_deploy_name + '/' + ait_deploy_name
copy_dir = str(base_path / '../deploy')
if os.path.exists(ait_deploy_base_dir):
shutil.rmtree(ait_deploy_base_dir)
shutil.copytree(copy_dir, ait_deploy_archive_dir)
shutil.copyfile(str(base_path / 'ait.manifest.json'),
ait_deploy_base_dir + '/ait.manifest.json')
delete_file = ait_deploy_archive_dir + '/container/dockerfile_license'
os.remove(delete_file)
# ait deploy(script)
src_file = str(base_path / f'../template/ait_deploy.bat')
dst_file = str(base_path / '../tool/ait_deploy.bat')
shutil.copyfile(src_file, dst_file)
_replace_template(ait_manifest.get_name(), ait_manifest.get_version(),
dst_file, 'shift-jis')
# configure exporter
config_for_converting.Exporter.preprocessors = [
"nbconvert.preprocessors.TagRemovePreprocessor"
]
# --------------------------------------------------
# loop over notebooks
path = os.path.abspath(os.path.dirname(__file__))
path_to_notebooks = os.path.join(path, '../docs/examples/')
# list notebooks to convert to Python scripts
list_notebooks = [
f for f in os.listdir(path_to_notebooks)
if os.path.splitext(f)[-1] == '.ipynb'
]
# folder to store converted files
output_directory = os.path.join(path_to_notebooks, './python_scripts')
for item in list_notebooks:
print(path_to_notebooks + item)
new_notebook = modify_notebook(path_to_notebooks + item)
python_exporter = PythonExporter(config=config_for_converting)
(body, resources) = python_exporter.from_notebook_node(new_notebook)
python_filename = os.path.splitext(item)[0] + '.py'
with open(os.path.join(output_directory, python_filename), 'w') as fout:
fout.write(body)