def test_should_generate_python_script_with_conf_auto_detect(work_dir, mocker):
"""
Convert a Jupyter Notebook to a Python 3 script using conf
"""
kept_cells, dropped_cells, docstring, notebook_path = generate_test_notebook(
work_dir=work_dir, notebook_name='test_nb.ipynb')
# Create conf with knew ignore cell keywords
conf_data = write_conf(work_dir=work_dir,
conf_path=join(work_dir, DEFAULT_CONF_FILENAME),
ignore_keys=['# Ignore', 'import'])
cmd_arguments = ['-n', notebook_path, '-w', work_dir]
IPynbToPython().run(*cmd_arguments)
# This path is generated using the conf script_dir and the notebook name
output_script_path = join(work_dir,
conf_data['path']['python_script_root_dir'],
'mlvtools_test_nb.py')
assert exists(output_script_path)
with open(output_script_path, 'r') as fd:
file_content = fd.read()
# With those knew keywords from conf the first code cell must be ignored (due to import)
# The second no effect cell must remain because "# No effect" is no more a keyword
assert not is_in(kept_cells.code[0][1], file_content)
assert is_in(dropped_cells.no_effect[1][1], file_content)
def test_should_generate_python_script_no_conf(work_dir):
"""
Convert a Jupyter Notebook to a Python 3 script using all parameters
- create right function name
- keep the docstring
- keep code cells
- remove no effect cells
- remove trailing cells
- call the function with right arguments
- python syntax is ok
- the script is executable
"""
kept_cells, dropped_cells, docstring, notebook_path = generate_test_notebook(
work_dir=work_dir, notebook_name='test_nb.ipynb')
output_path = join(work_dir, 'out.py')
cmd_arguments = ['-n', notebook_path, '-o', output_path, '-w', work_dir]
IPynbToPython().run(*cmd_arguments)
assert exists(output_path)
with open(output_path, 'r') as fd:
file_content = fd.read()
check_content(docstring, kept_cells, dropped_cells, file_content)
assert 'def mlvtools_test_nb(subset, rate):' in file_content
assert 'mlvtools_test_nb(args.subset, args.rate)' in file_content
# Ensure generated file syntax is right
compile(file_content, output_path, 'exec')
# Ensure script has exe right
assert stat.S_IMODE(os_stat(output_path).st_mode) == 0o755
def test_should_generate_python_script_with_conf_auto_detect(work_dir, mocker):
"""
Convert a Jupyter Notebook to a Python 3 script using conf
"""
mocked_check_output = mocker.patch('subprocess.check_output',
return_value=work_dir.encode())
kept_cells, dropped_cells, docstring, notebook_path = generate_test_notebook(
work_dir=work_dir, notebook_name='test_nb.ipynb')
# Create conf with knew ignore cell keywords
conf_data = write_conf(work_dir=work_dir,
conf_path=join(work_dir, DEFAULT_CONF_FILENAME),
ignore_keys=['# Ignore', 'import'])
cmd_arguments = ['-n', notebook_path]
IPynbToPython().run(*cmd_arguments)
# This path is generated using the conf script_dir and the notebook name
output_script_path = join(work_dir,
conf_data['path']['python_script_root_dir'],
'mlvtools_test_nb.py')
assert exists(output_script_path)
with open(output_script_path, 'r') as fd:
file_content = fd.read()
# With those knew keywords from conf the first code cell must be ignored (due to import)
# The second no effect cell must remain because "# No effect" is no more a keyword
assert not is_in(kept_cells.code[0][1], file_content)
assert is_in(dropped_cells.no_effect[1][1], file_content)
# Check conf file has been found using git rev-parse command
assert mocked_check_output.mock_calls == [
mocker.call(['git', 'rev-parse', '--show-toplevel'], cwd=work_dir)
]
def test_should_generate_valid_python_if_only_trailing_cells(work_dir):
"""
Convert a Jupyter Notebook with no 'code cell' to a Python 3 script
"""
cells = [('markdown', 'This is a trailing comment cell'),
('code', '# No effect\ntrailing = 2')]
notebook_path = gen_notebook(cells,
tmp_dir=work_dir,
file_name='test_nb.ipynb')
output_path = join(work_dir, 'out.py')
cmd_arguments = [
'-n', notebook_path, '-o', output_path, '--working-directory', work_dir
]
IPynbToPython().run(*cmd_arguments)
assert exists(output_path)
with open(output_path, 'r') as fd:
file_content = fd.read()
assert 'def mlvtools_test_nb():' in file_content
assert not is_in(cells[0][1], file_content)
assert not is_in(cells[1][1], file_content)
assert is_in('pass', file_content)
assert 'mlvtools_test_nb()' in file_content
def test_should_handle_notebook_with_invalid_python_name_with_conf(work_dir, mocker):
"""
Test invalid python filename are converted
"""
mocked_check_output = mocker.patch('subprocess.check_output', return_value=work_dir.encode())
notebook_path = gen_notebook(cells=[('code', 'pass')], tmp_dir=work_dir, file_name='01_(test) nb.ipynb')
# Create conf in a freshly init git repo
conf_data = write_conf(work_dir=work_dir, conf_path=join(work_dir, DEFAULT_CONF_FILENAME),
ignore_keys=['# Ignore', 'remove='])
cmd_arguments = ['-n', notebook_path]
IPynbToPython().run(*cmd_arguments)
# This path is generated using the conf script_dir and the notebook name
output_script_path = join(work_dir, conf_data['path']['python_script_root_dir'], 'mlvtools_01__test_nb.py')
assert exists(output_script_path)
with open(output_script_path, 'r') as fd:
file_content = fd.read()
# Ensure generated file syntax is right
compile(file_content, output_script_path, 'exec')
assert mocked_check_output.mock_calls == [mocker.call(
['git', 'rev-parse', '--show-toplevel'],
cwd=work_dir)]
def test_should_raise_if_missing_output_path_argument_and_no_conf():
"""
Test command raise if output path is not provided when no conf
"""
arguments = ['-n', './test.ipynb', '--working-directory', './']
with pytest.raises(MlVToolException):
IPynbToPython().run(*arguments)
def test_should_handle_notebook_with_invalid_python_name_with_conf(
work_dir, mocker):
"""
Test invalid python filename are converted
"""
notebook_path = gen_notebook(cells=[('code', 'pass')],
tmp_dir=work_dir,
file_name='01_(test) nb.ipynb')
# Create conf in working directory
conf_data = write_conf(work_dir=work_dir,
conf_path=join(work_dir, DEFAULT_CONF_FILENAME),
ignore_keys=['# Ignore', 'remove='])
cmd_arguments = ['-n', notebook_path, '-w', work_dir]
IPynbToPython().run(*cmd_arguments)
# This path is generated using the conf script_dir and the notebook name
output_script_path = join(work_dir,
conf_data['path']['python_script_root_dir'],
'mlvtools_01__test_nb.py')
assert exists(output_script_path)
with open(output_script_path, 'r') as fd:
file_content = fd.read()
# Ensure generated file syntax is right
compile(file_content, output_script_path, 'exec')
def test_should_raise_if_output_path_exist_and_no_force(work_dir):
"""
Test command raise if output path already exists and no force argument
"""
output_path = join(work_dir, 'py_script')
with open(output_path, 'w') as fd:
fd.write('')
arguments = [
'-n', './test.ipynb', '--working-directory', work_dir, '-o',
output_path
]
with pytest.raises(MlVToolException):
IPynbToPython().run(*arguments)
def test_should_get_output_path_from_auto_detected_conf(work_dir):
"""
Test output python script path is generated from notebook name and auto detected configuration
"""
conf_path, notebook_path = setup_with_conf(work_dir,
conf_path=join(
work_dir,
DEFAULT_CONF_FILENAME))
arguments = ['-n', notebook_path, '--working-directory', work_dir]
IPynbToPython().run(*arguments)
# Those path are generated using conf path and the notebook name
script_path = join(work_dir, 'test_scripts', 'mlvtools_nb_test.py')
assert exists(script_path)
def test_should_overwrite_with_force_argument(work_dir):
"""
Test output paths are overwritten with force argument
"""
notebook_path = gen_notebook(cells=[('code', 'pass')],
tmp_dir=work_dir,
file_name='test_nb.ipynb')
output_path = join(work_dir, 'py_script')
with open(output_path, 'w') as fd:
fd.write('')
arguments = [
'-n', notebook_path, '--working-directory', work_dir, '-o',
output_path, '--force'
]
IPynbToPython().run(*arguments)
with open(output_path, 'r') as fd:
assert fd.read()
def test_should_overwrite_conf(work_dir):
"""
Test output path argument overwrite conf
"""
conf_path, notebook_path = setup_with_conf(work_dir,
conf_path=join(
work_dir,
DEFAULT_CONF_FILENAME))
arg_script_path = join(work_dir, 'custom_script.py')
arguments = [
'-n', notebook_path, '--working-directory', work_dir, '-o',
arg_script_path
]
IPynbToPython().run(*arguments)
# Assert output path is the one given as command argument not the one generated from conf
assert exists(arg_script_path)
conf_script_path = join(work_dir, 'test_scripts', 'nb_test.py')
assert not exists(conf_script_path)