def test_notebooks(notebook_path, base_env):
"""Ensures testing the notebooks in isolated virtual environments."""
tmpdir, proto_dir = base_env
notebook_file = os.path.basename(notebook_path)
dir_name = notebook_file.rstrip(".ipynb")
notebook_env = os.path.join(tmpdir, f"{dir_name}")
cmd = f"""
{proto_dir}/bin/virtualenv-clone {proto_dir} {notebook_env}
cd {notebook_env}
. ./bin/activate
papermill {notebook_path}"""
stdout, stderr, status = shell_tools.run_shell(
cmd=cmd,
log_run_to_stderr=False,
raise_on_fail=False,
out=shell_tools.TeeCapture(),
err=shell_tools.TeeCapture(),
)
if status != 0:
print(stdout)
print(stderr)
pytest.fail(f"Notebook failure: {notebook_file}")
def test_notebooks_against_released_cirq(notebook_path, base_env):
"""Tests the notebooks in isolated virtual environments."""
notebook_file = os.path.basename(notebook_path)
notebook_rel_dir = os.path.dirname(os.path.relpath(notebook_path, "."))
out_path = f"out/{notebook_rel_dir}/{notebook_file[:-6]}.out.ipynb"
tmpdir, proto_dir = base_env
notebook_file = os.path.basename(notebook_path)
dir_name = notebook_file.rstrip(".ipynb")
notebook_env = os.path.join(tmpdir, f"{dir_name}")
cmd = f"""
mkdir -p out/{notebook_rel_dir}
{proto_dir}/bin/virtualenv-clone {proto_dir} {notebook_env}
cd {notebook_env}
. ./bin/activate
papermill {notebook_path} {os.getcwd()}/{out_path}"""
_, stderr, status = shell_tools.run_shell(
cmd=cmd,
log_run_to_stderr=False,
raise_on_fail=False,
out=shell_tools.TeeCapture(),
err=shell_tools.TeeCapture(),
)
if status != 0:
print(stderr)
pytest.fail(
f"Notebook failure: {notebook_file}, please see {out_path} for the output "
f"notebook (in Github Actions, you can download it from the workflow artifact"
f" 'notebook-outputs')"
)
def test_notebooks_against_released_cirq(partition, notebook_path, base_env):
"""Tests the notebooks in isolated virtual environments.
In order to speed up the execution of these tests an auxiliary file may be supplied which
performs substitutions on the notebook to make it faster.
Specifically for a notebook file notebook.ipynb, one can supply a file notebook.tst which
contains the substitutes. The substitutions are provide in the form `pattern->replacement`
where the pattern is what is matched and replaced. While the pattern is compiled as a
regular expression, it is considered best practice to not use complicated regular expressions.
Lines in this file that do not have `->` are ignored.
"""
notebook_file = os.path.basename(notebook_path)
notebook_rel_dir = os.path.dirname(os.path.relpath(notebook_path, "."))
out_path = f"out/{notebook_rel_dir}/{notebook_file[:-6]}.out.ipynb"
tmpdir, proto_dir = base_env
notebook_file = os.path.basename(notebook_path)
dir_name = notebook_file.rstrip(".ipynb")
notebook_env = os.path.join(tmpdir, f"{dir_name}")
rewritten_notebook_descriptor, rewritten_notebook_path = rewrite_notebook(
notebook_path)
cmd = f"""
mkdir -p out/{notebook_rel_dir}
{proto_dir}/bin/virtualenv-clone {proto_dir} {notebook_env}
cd {notebook_env}
. ./bin/activate
papermill {rewritten_notebook_path} {os.getcwd()}/{out_path}"""
_, stderr, status = shell_tools.run_shell(
cmd=cmd,
log_run_to_stderr=False,
raise_on_fail=False,
out=shell_tools.TeeCapture(),
err=shell_tools.TeeCapture(),
# important to get rid of PYTHONPATH specifically, which contains
# the Cirq repo path due to check/pytest
env={},
)
if status != 0:
print(stderr)
pytest.fail(
f"Notebook failure: {notebook_file}, please see {out_path} for the output "
f"notebook (in Github Actions, you can download it from the workflow artifact"
f" 'notebook-outputs'). \n"
f"If this is a new failure in this notebook due to a new change, "
f"that is only available in master for now, consider adding `pip install --pre cirq` "
f"instead of `pip install cirq` to this notebook, and exclude it from "
f"dev_tools/notebooks/isolated_notebook_test.py.")
if rewritten_notebook_descriptor:
os.close(rewritten_notebook_descriptor)
def run(
*,
script_file: str,
tmpdir_factory: '_pytest.tmpdir.TempdirFactory',
arg: str = '',
setup: str = '',
additional_intercepts: Iterable[str] = (),
) -> shell_tools.CommandOutput:
"""Invokes the given script within a temporary test environment."""
with open(script_file) as f:
script_lines = f.readlines()
# Create a unique temporary directory
dir_path = tmpdir_factory.mktemp('tmp', numbered=True)
file_path = os.path.join(dir_path, 'test-script.sh')
intercepted = [
'python',
'python3',
'pylint',
'pytest',
'mypy',
'black',
*additional_intercepts,
]
assert script_lines[0] == '#!/usr/bin/env bash\n'
for e in intercepted:
script_lines.insert(1, e + '() {\n echo INTERCEPTED ' + e + ' $@\n}\n')
with open(file_path, 'w') as f:
f.writelines(script_lines)
cmd = r"""
dir=$(git rev-parse --show-toplevel)
cd {}
git init --quiet --initial-branch master
git config --local user.name 'Me'
git config --local user.email '<>'
git commit -m init --allow-empty --quiet --no-gpg-sign
{}
mkdir -p dev_tools
touch dev_tools/pypath
chmod +x ./test-script.sh
./test-script.sh {}
""".format(
dir_path, setup, arg
)
return shell_tools.run_shell(
cmd=cmd,
log_run_to_stderr=False,
raise_on_fail=False,
out=shell_tools.TeeCapture(),
err=shell_tools.TeeCapture(),
)
def perform_check(self, env: env_tools.PreparedEnv, verbose: bool):
do_coverage = True
base_path = cast(str, env.destination_directory)
rc_path = os.path.join(base_path, 'continuous-integration',
'.coveragerc')
target_path = base_path
result = shell_tools.run_cmd(env.bin('pytest'),
target_path,
None if verbose else '--quiet',
*([
'--cov', '--cov-report=annotate',
'--cov-config={}'.format(rc_path)
] if do_coverage else []),
out=shell_tools.TeeCapture(sys.stdout),
raise_on_fail=False,
log_run_to_stderr=verbose)
output = cast(str, result[0])
passed = result[2] == 0
if passed:
return True, 'Tests passed!'
last_line = [e for e in output.split('\n') if e.strip()][-1]
fail_match = re.match('.+=== (\d+) failed', last_line)
if fail_match is None:
return False, 'Tests failed.'
return False, '{} tests failed.'.format(fail_match.group(1))
def _common_run_helper(env: env_tools.PreparedEnv, coverage: bool,
verbose: bool) -> Tuple[bool, str]:
base_path = cast(str, env.destination_directory)
target_path = base_path
result = shell_tools.run_cmd(
env.bin('pytest'),
target_path,
None if verbose else '--quiet',
'--cov' if coverage else '',
'--cov-report=annotate' if coverage else '',
out=shell_tools.TeeCapture(sys.stdout),
raise_on_fail=False,
log_run_to_stderr=verbose,
abbreviate_non_option_arguments=True)
output = cast(str, result[0])
passed = result[2] == 0
if passed:
return True, 'Converted tests passed!'
last_line = [e for e in output.split('\n') if e.strip()][-1]
fail_match = re.match('.+=== (\d+) failed', last_line)
if fail_match is None:
return False, 'Tests failed.'
return False, '{} tests failed.'.format(fail_match.group(1))
def perform_check(self, env: env_tools.PreparedEnv, verbose: bool):
base_path = cast(str, env.destination_directory)
rc_path = os.path.join(base_path, 'continuous-integration',
'.pylintrc')
files = list(
env_tools.get_unhidden_ungenerated_python_files(base_path))
result = shell_tools.run_cmd(env.bin('pylint'),
'--reports=no',
'--score=no',
'--output-format=colorized',
'--rcfile={}'.format(rc_path),
*files,
out=shell_tools.TeeCapture(sys.stdout),
raise_on_fail=False,
log_run_to_stderr=verbose,
abbreviate_non_option_arguments=True)
output = cast(str, result[0])
passed = result[2] == 0
if passed:
return True, 'No lint here!'
file_line_count = len(re.findall(r'\*' * 10, output))
total_line_count = len([e for e in output.split('\n') if e.strip()])
issue_count = total_line_count - file_line_count
return False, '{} issues'.format(issue_count)
def test_isolated_env_cloning(cloned_env, param):
env = cloned_env("test_isolated", "flynt==0.64")
assert (env / "bin" / "pip").is_file()
result = shell_tools.run_cmd(
*f"{env}/bin/pip list --format=json".split(), out=shell_tools.TeeCapture()
)
packages = json.loads(result.out)
assert {"name": "flynt", "version": "0.64"} in packages
assert {"astor", "flynt", "pip", "setuptools", "wheel"} == set(p['name'] for p in packages)
def test_notebooks_against_released_cirq(notebook_path):
notebook_file = os.path.basename(notebook_path)
notebook_rel_dir = os.path.dirname(os.path.relpath(notebook_path, "."))
out_path = f"out/{notebook_rel_dir}/{notebook_file[:-6]}.out.ipynb"
cmd = f"""mkdir -p out/{notebook_rel_dir}
papermill {notebook_path} {out_path}"""
stdout, stderr, status = shell_tools.run_shell(
cmd=cmd,
log_run_to_stderr=False,
raise_on_fail=False,
out=shell_tools.TeeCapture(),
err=shell_tools.TeeCapture(),
)
if status != 0:
print(stderr)
pytest.fail(
f"Notebook failure: {notebook_file}, please see {out_path}")
def test_isolated_packages(cloned_env, module):
env = cloned_env("isolated_packages", *PACKAGES)
if str(module.root) != "cirq-core":
assert f'cirq-core=={module.version}' in module.install_requires
result = shell_tools.run_cmd(
*f"{env}/bin/pip install ./{module.root} ./cirq-core".split(),
err=shell_tools.TeeCapture(),
raise_on_fail=False,
)
assert result.exit_code == 0, f"Failed to install {module.name}:\n{result.err}"
result = shell_tools.run_cmd(
*f"{env}/bin/pytest ./{module.root} --ignore ./cirq-core/cirq/contrib".split(),
out=shell_tools.TeeCapture(),
err=shell_tools.TeeCapture(),
raise_on_fail=False,
)
assert result.exit_code == 0, f"Failed isolated tests for {module.name}:\n{result.stdout}"
def test_notebooks_against_released_cirq(notebook_path):
notebook_file = os.path.basename(notebook_path)
notebook_rel_dir = os.path.dirname(os.path.relpath(notebook_path, "."))
out_path = f"out/{notebook_rel_dir}/{notebook_file[:-6]}.out.ipynb"
cmd = f"""mkdir -p out/{notebook_rel_dir}
papermill {notebook_path} {out_path}"""
_, stderr, status = shell_tools.run_shell(
cmd=cmd,
log_run_to_stderr=False,
raise_on_fail=False,
out=shell_tools.TeeCapture(),
err=shell_tools.TeeCapture(),
)
if status != 0:
print(stderr)
pytest.fail(
f"Notebook failure: {notebook_file}, please see {out_path} for the output "
f"notebook (in Github Actions, you can download it from the workflow artifact"
f" 'notebook-outputs')")
def run(*,
script_file: str,
arg: str = '',
setup: str = '') -> shell_tools.CommandOutput:
"""Invokes the given script within a temporary test environment."""
with open(script_file) as f:
script_lines = f.readlines()
intercepted = [
'python',
'python2',
'python3',
'pylint',
'pytest',
'mypy',
]
assert script_lines[0] == '#!/usr/bin/env bash\n'
for e in intercepted:
script_lines.insert(1,
e + '() {\n echo INTERCEPTED ' + e + ' $@\n}\n')
with cirq.testing.TempDirectoryPath() as dir_path:
with open(os.path.join(dir_path, 'test-script'), 'w') as f:
f.writelines(script_lines)
cmd = r"""
dir=$(git rev-parse --show-toplevel)
cd {}
git init --quiet
git commit -m init --allow-empty --quiet --no-gpg-sign
{}
chmod +x ./test-script
./test-script {}
""".format(dir_path, setup, arg)
return shell_tools.run_shell(cmd=cmd,
log_run_to_stderr=False,
raise_on_fail=False,
out=shell_tools.TeeCapture(),
err=shell_tools.TeeCapture())
def test_notebooks_against_released_cirq(notebook_path):
"""Test that jupyter notebooks execute.
In order to speed up the execution of these tests an auxiliary file may be supplied which
performs substitutions on the notebook to make it faster.
Specifically for a notebook file notebook.ipynb, one can supply a file notebook.tst which
contains the substitutes. The substitutions are provide in the form `pattern->replacement`
where the pattern is what is matched and replaced. While the pattern is compiled as a
regular expression, it is considered best practice to not use complicated regular expressions.
Lines in this file that do not have `->` are ignored.
"""
notebook_file = os.path.basename(notebook_path)
notebook_rel_dir = os.path.dirname(os.path.relpath(notebook_path, "."))
out_path = f"out/{notebook_rel_dir}/{notebook_file[:-6]}.out.ipynb"
rewritten_notebook_descriptor, rewritten_notebook_path = rewrite_notebook(notebook_path)
papermill_flags = "--no-request-save-on-cell-execute --autosave-cell-every 0"
cmd = f"""mkdir -p out/{notebook_rel_dir}
papermill {rewritten_notebook_path} {out_path} {papermill_flags}"""
_, stderr, status = shell_tools.run_shell(
cmd=cmd,
log_run_to_stderr=False,
raise_on_fail=False,
out=shell_tools.TeeCapture(),
err=shell_tools.TeeCapture(),
)
if status != 0:
print(stderr)
pytest.fail(
f"Notebook failure: {notebook_file}, please see {out_path} for the output "
f"notebook (in Github Actions, you can download it from the workflow artifact"
f" 'notebook-outputs')"
)
if rewritten_notebook_descriptor:
os.close(rewritten_notebook_descriptor)
def perform_check(self, env: env_tools.PreparedEnv, verbose: bool):
base_path = cast(str, env.destination_directory)
config_path = os.path.join(base_path, 'dev_tools', 'conf', 'mypy.ini')
files = list(
env_tools.get_unhidden_ungenerated_python_files(base_path))
result = shell_tools.run_cmd(env.bin('mypy'),
'--config-file={}'.format(config_path),
*files,
out=shell_tools.TeeCapture(sys.stdout),
raise_on_fail=False,
log_run_to_stderr=verbose,
abbreviate_non_option_arguments=True)
output = cast(str, result[0])
passed = result[2] == 0
if passed:
return True, 'Types look good!'
issue_count = len([e for e in output.split('\n') if e.strip()])
return False, '{} issues'.format(issue_count)
def test_run_shell_capture():
assert run_shell('echo test 1>&2', err=None) == (None, None, 0)
assert run_shell('echo test 1>&2',
err=shell_tools.TeeCapture()) == (None, 'test\n', 0)
assert run_shell('echo test 1>&2', err=None,
out=shell_tools.TeeCapture()) == ('', None, 0)
def test_run_cmd_capture():
assert run_cmd('echo', 'test', out=None) == (None, None, 0)
assert run_cmd('echo', 'test',
out=shell_tools.TeeCapture()) == ('test\n', None, 0)
assert run_cmd('echo', 'test', out=None,
err=shell_tools.TeeCapture()) == (None, '', 0)
