def create_virtual_env(venv_path: str,
requirements_paths: Iterable[str],
python_path: str,
verbose: bool) -> None:
"""Creates a new virtual environment and then installs dependencies.
Args:
venv_path: Where to put the virtual environment's state.
requirements_paths: Location of requirements files to -r install.
python_path: The python binary to use.
verbose: When set, more progress output is produced.
"""
shell_tools.run_cmd('virtualenv',
None if verbose else '--quiet',
'-p',
python_path,
venv_path,
out=sys.stderr)
pip_path = os.path.join(venv_path, 'bin', 'pip')
for req_path in requirements_paths:
shell_tools.run_cmd(pip_path,
'install',
None if verbose else '--quiet',
'-r',
req_path,
out=sys.stderr)
def create_virtual_env(venv_path: str,
requirements_path: str,
python_path: str,
verbose: bool) -> None:
"""Creates a new virtual environment and then installs dependencies.
Args:
venv_path: Where to put the virtual environment's state.
requirements_path: Location of the requirements file to -r install.
python_path: The python binary to use.
verbose: When set, more progress output is produced.
"""
shell_tools.run_cmd('virtualenv',
None if verbose else '--quiet',
'-p',
python_path,
venv_path,
out=sys.stderr)
pip_path = os.path.join(venv_path, 'bin', 'pip')
shell_tools.run_cmd(pip_path,
'install',
None if verbose else '--quiet',
'-r',
requirements_path,
out=sys.stderr)
def _git_fetch_for_comparison(remote: str,
actual_branch: str,
compare_branch: str,
verbose: bool) -> prepared_env.PreparedEnv:
"""Fetches two branches including their common ancestor.
Limits the depth of the fetch to avoid unnecessary work. Scales up the
depth exponentially and tries again when the initial guess is not deep
enough.
Args:
remote: The location of the remote repository, in a format that the
git command will understand.
actual_branch: A remote branch or ref to fetch,
compare_branch: Another remote branch or ref to fetch,
verbose: When set, more progress output is produced.
Returns:
A ComparableCommits containing the commit id of the actual branch and
a the id of a commit to compare against (e.g. for when doing incremental
checks).
"""
actual_id = ''
base_id = ''
for depth in [10, 100, 1000, None]:
depth_str = '' if depth is None else '--depth={}'.format(depth)
shell_tools.run_cmd(
'git',
'fetch',
None if verbose else '--quiet',
remote,
actual_branch,
depth_str,
log_run_to_stderr=verbose)
actual_id = shell_tools.output_of('git', 'rev-parse', 'FETCH_HEAD')
shell_tools.run_cmd('git',
'fetch',
None if verbose else '--quiet',
remote,
compare_branch,
depth_str,
log_run_to_stderr=verbose)
base_id = shell_tools.output_of('git', 'rev-parse', 'FETCH_HEAD')
try:
base_id = shell_tools.output_of(
'git',
'merge-base',
actual_id,
base_id)
break
except subprocess.CalledProcessError:
# No common ancestor. We need to dig deeper.
pass
return prepared_env.PreparedEnv(None, actual_id, base_id, None, None)
def derive_temporary_python2_environment(
destination_directory: str,
python3_environment: PreparedEnv,
verbose: bool,
env_name: str = '.test_virtualenv_py2',
python_path: str = "/usr/bin/python2.7") -> PreparedEnv:
"""Creates a python 2.7 environment starting from a prepared python 3 one.
Args:
destination_directory: Where to put the python 2 environment.
python3_environment: The prepared environment to start from.
verbose: When set, more progress output is produced.
env_name: The name to use for the virtualenv directory.
python_path: The python binary to use.
Returns:
A description of the environment that was prepared.
"""
shutil.rmtree(destination_directory)
input_directory = cast(str, python3_environment.destination_directory)
os.chdir(input_directory)
conversion_script_path = os.path.join(
input_directory,
'dev_tools',
'python2.7-generate.sh')
shell_tools.run_cmd('bash',
conversion_script_path,
destination_directory,
input_directory,
python3_environment.virtual_env_path,
out=sys.stderr)
os.chdir(destination_directory)
# Create virtual environment.
env_path = os.path.join(destination_directory, env_name)
# (These files are output by dev_tools/python2.7-generate.sh.)
req_path = os.path.join(destination_directory, 'requirements.txt')
dev_req_path = os.path.join(destination_directory,
'pip-list-test-tools.txt')
contrib_req_path = os.path.join(destination_directory,
'cirq',
'contrib',
'contrib-requirements.txt')
req_paths = [req_path, dev_req_path, contrib_req_path]
create_virtual_env(venv_path=env_path,
python_path=python_path,
requirements_paths=req_paths,
verbose=verbose)
return PreparedEnv(github_repo=python3_environment.repository,
actual_commit_id=python3_environment.actual_commit_id,
compare_commit_id=python3_environment.compare_commit_id,
destination_directory=destination_directory,
virtual_env_path=env_path)
def _create_base_env(base_dir: Path, pip_install_args: Tuple[str, ...]):
try:
create_virtual_env(str(base_dir), [], sys.executable, True)
with open(base_dir / "testrun.uid", mode="w") as f:
f.write(testrun_uid)
if pip_install_args:
shell_tools.run_cmd(f"{base_dir}/bin/pip", "install", *pip_install_args)
except BaseException as ex:
# cleanup on failure
if base_dir.is_dir():
print(f"Removing {base_dir}, due to error: {ex}")
shutil.rmtree(base_dir)
raise
def _common_run_helper(env: env_tools.PreparedEnv,
coverage: bool,
verbose: bool) -> Tuple[bool, str]:
base_path = cast(str, env.destination_directory)
target_path = base_path
result = shell_tools.run_cmd(
env.bin('pytest'),
target_path,
None if verbose else '--quiet',
'--cov' if coverage else '',
'--cov-report=annotate' if coverage else '',
out=shell_tools.TeeCapture(sys.stdout),
raise_on_fail=False,
log_run_to_stderr=verbose,
abbreviate_non_option_arguments=True)
output = cast(str, result[0])
passed = result[2] == 0
if passed:
return True, 'Converted tests passed!'
last_line = [e for e in output.split('\n') if e.strip()][-1]
fail_match = re.match('.+=== (\d+) failed', last_line)
if fail_match is None:
return False, 'Tests failed.'
return False, '{} tests failed.'.format(fail_match.group(1))
def _common_run_helper(env: env_tools.PreparedEnv, coverage: bool,
verbose: bool) -> Tuple[bool, str]:
base_path = cast(str, env.destination_directory)
target_path = base_path
result = shell_tools.run_cmd(
env.bin('pytest'),
target_path,
None if verbose else '--quiet',
'--cov' if coverage else '',
'--cov-report=annotate' if coverage else '',
out=shell_tools.TeeCapture(sys.stdout),
raise_on_fail=False,
log_run_to_stderr=verbose,
abbreviate_non_option_arguments=True)
output = cast(str, result[0])
passed = result[2] == 0
if passed:
return True, 'Converted tests passed!'
last_line = [e for e in output.split('\n') if e.strip()][-1]
fail_match = re.match('.+=== (\d+) failed', last_line)
if fail_match is None:
return False, 'Tests failed.'
return False, '{} tests failed.'.format(fail_match.group(1))
def perform_check(self, env: env_tools.PreparedEnv, verbose: bool):
do_coverage = True
base_path = cast(str, env.destination_directory)
rc_path = os.path.join(base_path, 'continuous-integration',
'.coveragerc')
target_path = base_path
result = shell_tools.run_cmd(env.bin('pytest'),
target_path,
None if verbose else '--quiet',
*([
'--cov', '--cov-report=annotate',
'--cov-config={}'.format(rc_path)
] if do_coverage else []),
out=shell_tools.TeeCapture(sys.stdout),
raise_on_fail=False,
log_run_to_stderr=verbose)
output = cast(str, result[0])
passed = result[2] == 0
if passed:
return True, 'Tests passed!'
last_line = [e for e in output.split('\n') if e.strip()][-1]
fail_match = re.match('.+=== (\d+) failed', last_line)
if fail_match is None:
return False, 'Tests failed.'
return False, '{} tests failed.'.format(fail_match.group(1))
def perform_check(self, env: env_tools.PreparedEnv, verbose: bool):
base_path = cast(str, env.destination_directory)
rc_path = os.path.join(base_path, 'continuous-integration',
'.pylintrc')
files = list(
env_tools.get_unhidden_ungenerated_python_files(base_path))
result = shell_tools.run_cmd(env.bin('pylint'),
'--reports=no',
'--score=no',
'--output-format=colorized',
'--rcfile={}'.format(rc_path),
*files,
out=shell_tools.TeeCapture(sys.stdout),
raise_on_fail=False,
log_run_to_stderr=verbose,
abbreviate_non_option_arguments=True)
output = cast(str, result[0])
passed = result[2] == 0
if passed:
return True, 'No lint here!'
file_line_count = len(re.findall(r'\*' * 10, output))
total_line_count = len([e for e in output.split('\n') if e.strip()])
issue_count = total_line_count - file_line_count
return False, '{} issues'.format(issue_count)
def perform_check(self, env: env_tools.PreparedEnv, verbose: bool):
base_path = cast(str, env.destination_directory)
rc_path = os.path.join(base_path,
'continuous-integration',
'.pylintrc')
files = list(
env_tools.get_unhidden_ungenerated_python_files(base_path))
result = shell_tools.run_cmd(
env.bin('pylint'),
'--rcfile={}'.format(rc_path),
*files,
out=shell_tools.TeeCapture(sys.stdout),
raise_on_fail=False,
log_run_to_stderr=verbose,
abbreviate_non_option_arguments=True)
output = cast(str, result[0])
passed = result[2] == 0
if passed:
return True, 'No lint here!'
file_line_count = len(re.findall(r'\*' * 10, output))
total_line_count = len([e for e in output.split('\n') if e.strip()])
issue_count = total_line_count - file_line_count
return False, '{} issues'.format(issue_count)
def perform_check(self, env: env_tools.PreparedEnv, verbose: bool):
do_coverage = True
base_path = cast(str, env.destination_directory)
rc_path = os.path.join(base_path,
'continuous-integration',
'.coveragerc')
target_path = base_path
result = shell_tools.run_cmd(
env.bin('pytest'),
target_path,
None if verbose else '--quiet',
*([
'--cov',
'--cov-report=annotate',
'--cov-config={}'.format(rc_path)
] if do_coverage else []),
out=shell_tools.TeeCapture(sys.stdout),
raise_on_fail=False,
log_run_to_stderr=verbose)
output = cast(str, result[0])
passed = result[2] == 0
if passed:
return True, 'Tests passed!'
last_line = [e for e in output.split('\n') if e.strip()][-1]
fail_match = re.match('.+=== (\d+) failed', last_line)
if fail_match is None:
return False, 'Tests failed.'
return False, '{} tests failed.'.format(fail_match.group(1))
def derive_temporary_python2_environment(
destination_directory: str,
python3_environment: PreparedEnv,
verbose: bool,
env_name: str = '.test_virtualenv_py2',
python_path: str = "/usr/bin/python2.7") -> PreparedEnv:
"""Creates a python 2.7 environment starting from a prepared python 3 one.
Args:
destination_directory: Where to put the python 2 environment.
python3_environment: The prepared environment to start from.
verbose: When set, more progress output is produced.
env_name: The name to use for the virtualenv directory.
python_path: The python binary to use.
Returns:
A description of the environment that was prepared.
"""
shutil.rmtree(destination_directory)
input_directory = cast(str, python3_environment.destination_directory)
os.chdir(input_directory)
conversion_script_path = os.path.join(
input_directory,
'python2.7-generate.sh')
shell_tools.run_cmd('bash',
conversion_script_path,
destination_directory,
input_directory,
python3_environment.virtual_env_path,
out=sys.stderr)
os.chdir(destination_directory)
# Create virtual environment.
env_path = os.path.join(destination_directory, env_name)
req_path = os.path.join(destination_directory, 'dev-requirements.txt')
create_virtual_env(venv_path=env_path,
python_path=python_path,
requirements_path=req_path,
verbose=verbose)
return PreparedEnv(github_repo=python3_environment.repository,
actual_commit_id=python3_environment.actual_commit_id,
compare_commit_id=python3_environment.compare_commit_id,
destination_directory=destination_directory,
virtual_env_path=env_path)
def base_env_creator(env_dir_name: str, *pip_install_args: str) -> Path:
"""The function to create a cloned base environment."""
# get/create a temp directory shared by all workers
base_temp_path = Path(tempfile.gettempdir()) / "cirq-pytest"
os.makedirs(name=base_temp_path, exist_ok=True)
nonlocal base_dir
base_dir = base_temp_path / env_dir_name
with FileLock(str(base_dir) + ".lock"):
if _check_for_reuse_or_recreate(base_dir):
print(f"Pytest worker [{worker_id}] is reusing {base_dir} for '{env_dir_name}'.")
else:
print(f"Pytest worker [{worker_id}] is creating {base_dir} for '{env_dir_name}'.")
_create_base_env(base_dir, pip_install_args)
clone_dir = base_temp_path / str(uuid.uuid4())
shell_tools.run_cmd("virtualenv-clone", str(base_dir), str(clone_dir))
return clone_dir
def test_isolated_env_cloning(cloned_env, param):
env = cloned_env("test_isolated", "flynt==0.64")
assert (env / "bin" / "pip").is_file()
result = shell_tools.run_cmd(
*f"{env}/bin/pip list --format=json".split(), out=shell_tools.TeeCapture()
)
packages = json.loads(result.out)
assert {"name": "flynt", "version": "0.64"} in packages
assert {"astor", "flynt", "pip", "setuptools", "wheel"} == set(p['name'] for p in packages)
def test_isolated_packages(cloned_env, module):
env = cloned_env("isolated_packages", *PACKAGES)
if str(module.root) != "cirq-core":
assert f'cirq-core=={module.version}' in module.install_requires
result = shell_tools.run_cmd(
*f"{env}/bin/pip install ./{module.root} ./cirq-core".split(),
err=shell_tools.TeeCapture(),
raise_on_fail=False,
)
assert result.exit_code == 0, f"Failed to install {module.name}:\n{result.err}"
result = shell_tools.run_cmd(
*f"{env}/bin/pytest ./{module.root} --ignore ./cirq-core/cirq/contrib".split(),
out=shell_tools.TeeCapture(),
err=shell_tools.TeeCapture(),
raise_on_fail=False,
)
assert result.exit_code == 0, f"Failed isolated tests for {module.name}:\n{result.stdout}"
def fetch_github_pull_request(
destination_directory: str,
repository: github_repository.GithubRepository,
pull_request_number: int,
verbose: bool,
) -> prepared_env.PreparedEnv:
"""Uses content from github to create a dir for testing and comparisons.
Args:
destination_directory: The location to fetch the contents into.
repository: The github repository that the commit lives under.
pull_request_number: The id of the pull request to clone. If None, then
the master branch is cloned instead.
verbose: When set, more progress output is produced.
Returns:
Commit ids corresponding to content to test/compare.
"""
branch = 'pull/{}/head'.format(pull_request_number)
os.chdir(destination_directory)
print('chdir', destination_directory, file=sys.stderr)
shell_tools.run_cmd('git',
'init',
None if verbose else '--quiet',
out=sys.stderr)
result = _git_fetch_for_comparison(
remote=repository.as_remote(),
actual_branch=branch,
compare_branch='master',
verbose=verbose,
)
shell_tools.run_cmd(
'git',
'branch',
None if verbose else '--quiet',
'compare_commit',
result.compare_commit_id,
log_run_to_stderr=verbose,
)
shell_tools.run_cmd(
'git',
'checkout',
None if verbose else '--quiet',
'-b',
'actual_commit',
result.actual_commit_id,
log_run_to_stderr=verbose,
)
return prepared_env.PreparedEnv(
github_repo=repository,
actual_commit_id=result.actual_commit_id,
compare_commit_id=result.compare_commit_id,
destination_directory=destination_directory,
virtual_env_path=None,
)
def fetch_github_pull_request(destination_directory: str,
repository: github_repository.GithubRepository,
pull_request_number: int,
verbose: bool
) -> prepared_env.PreparedEnv:
"""Uses content from github to create a dir for testing and comparisons.
Args:
destination_directory: The location to fetch the contents into.
repository: The github repository that the commit lives under.
pull_request_number: The id of the pull request to clone. If None, then
the master branch is cloned instead.
verbose: When set, more progress output is produced.
Returns:
Commit ids corresponding to content to test/compare.
"""
branch = 'pull/{}/head'.format(pull_request_number)
os.chdir(destination_directory)
print('chdir', destination_directory, file=sys.stderr)
shell_tools.run_cmd(
'git',
'init',
None if verbose else '--quiet',
out=sys.stderr)
result = _git_fetch_for_comparison(remote=repository.as_remote(),
actual_branch=branch,
compare_branch='master',
verbose=verbose)
shell_tools.run_cmd(
'git',
'branch',
None if verbose else '--quiet',
'compare_commit',
result.compare_commit_id,
log_run_to_stderr=verbose)
shell_tools.run_cmd(
'git',
'checkout',
None if verbose else '--quiet',
'-b',
'actual_commit',
result.actual_commit_id,
log_run_to_stderr=verbose)
return prepared_env.PreparedEnv(
github_repo=repository,
actual_commit_id=result.actual_commit_id,
compare_commit_id=result.compare_commit_id,
destination_directory=destination_directory,
virtual_env_path=None)
def perform_check(self, env: env_tools.PreparedEnv, verbose: bool):
base_path = cast(str, env.destination_directory)
config_path = os.path.join(base_path, 'dev_tools', 'conf', 'mypy.ini')
files = list(
env_tools.get_unhidden_ungenerated_python_files(base_path))
result = shell_tools.run_cmd(env.bin('mypy'),
'--config-file={}'.format(config_path),
*files,
out=shell_tools.TeeCapture(sys.stdout),
raise_on_fail=False,
log_run_to_stderr=verbose,
abbreviate_non_option_arguments=True)
output = cast(str, result[0])
passed = result[2] == 0
if passed:
return True, 'Types look good!'
issue_count = len([e for e in output.split('\n') if e.strip()])
return False, '{} issues'.format(issue_count)
def perform_check(self, env: env_tools.PreparedEnv, verbose: bool):
base_path = cast(str, env.destination_directory)
config_path = os.path.join(base_path,
'continuous-integration',
'mypy.ini')
files = list(env_tools.get_unhidden_ungenerated_python_files(
base_path))
result = shell_tools.run_cmd(
env.bin('mypy'),
'--config-file={}'.format(config_path),
*files,
out=shell_tools.TeeCapture(sys.stdout),
raise_on_fail=False,
log_run_to_stderr=verbose,
abbreviate_non_option_arguments=True)
output = cast(str, result[0])
passed = result[2] == 0
if passed:
return True, 'Types look good!'
issue_count = len([e for e in output.split('\n') if e.strip()])
return False, '{} issues'.format(issue_count)
def _create_base_env(proto_dir):
create_virtual_env(str(proto_dir), [], sys.executable, True)
pip_path = str(proto_dir / "bin" / "pip")
shell_tools.run_cmd(pip_path, "install", *PACKAGES)
def run_cmd(*args, **kwargs):
return shell_tools.run_cmd(*args, log_run_to_stderr=False, **kwargs)
def fetch_local_files(destination_directory: str,
verbose: bool) -> prepared_env.PreparedEnv:
"""Uses local files to create a directory for testing and comparisons.
Args:
destination_directory: The directory where the copied files should go.
verbose: When set, more progress output is produced.
Returns:
Commit ids corresponding to content to test/compare.
"""
staging_dir = destination_directory + '-staging'
try:
shutil.copytree(get_repo_root(), staging_dir)
os.chdir(staging_dir)
if verbose:
print('chdir', staging_dir, file=sys.stderr)
shell_tools.run_cmd(
'git',
'add',
'--all',
out=sys.stderr,
log_run_to_stderr=verbose)
shell_tools.run_cmd(
'git',
'commit',
'-m', 'working changes',
'--allow-empty',
None if verbose else '--quiet',
out=sys.stderr,
log_run_to_stderr=verbose)
cur_commit = shell_tools.output_of('git', 'rev-parse', 'HEAD')
os.chdir(destination_directory)
if verbose:
print('chdir', destination_directory, file=sys.stderr)
shell_tools.run_cmd('git',
'init',
None if verbose else '--quiet',
out=sys.stderr,
log_run_to_stderr=verbose)
result = _git_fetch_for_comparison(staging_dir,
cur_commit,
'master',
verbose=verbose)
finally:
shutil.rmtree(staging_dir, ignore_errors=True)
shell_tools.run_cmd(
'git',
'branch',
None if verbose else '--quiet',
'compare_commit',
result.compare_commit_id,
log_run_to_stderr=verbose)
shell_tools.run_cmd(
'git',
'checkout',
None if verbose else '--quiet',
'-b',
'actual_commit',
result.actual_commit_id,
log_run_to_stderr=verbose)
return prepared_env.PreparedEnv(
github_repo=None,
actual_commit_id=result.actual_commit_id,
compare_commit_id=result.compare_commit_id,
destination_directory=destination_directory,
virtual_env_path=None)
def fetch_local_files(destination_directory: str,
verbose: bool) -> prepared_env.PreparedEnv:
"""Uses local files to create a directory for testing and comparisons.
Args:
destination_directory: The directory where the copied files should go.
verbose: When set, more progress output is produced.
Returns:
Commit ids corresponding to content to test/compare.
"""
staging_dir = destination_directory + '-staging'
try:
shutil.copytree(get_repo_root(), staging_dir)
os.chdir(staging_dir)
if verbose:
print('chdir', staging_dir, file=sys.stderr)
shell_tools.run_cmd('git',
'add',
'--all',
out=sys.stderr,
log_run_to_stderr=verbose)
shell_tools.run_cmd('git',
'commit',
'-m',
'working changes',
'--allow-empty',
'--no-gpg-sign',
None if verbose else '--quiet',
out=sys.stderr,
log_run_to_stderr=verbose)
cur_commit = shell_tools.output_of('git', 'rev-parse', 'HEAD')
os.chdir(destination_directory)
if verbose:
print('chdir', destination_directory, file=sys.stderr)
shell_tools.run_cmd('git',
'init',
None if verbose else '--quiet',
out=sys.stderr,
log_run_to_stderr=verbose)
result = _git_fetch_for_comparison(staging_dir,
cur_commit,
'master',
verbose=verbose)
finally:
shutil.rmtree(staging_dir, ignore_errors=True)
shell_tools.run_cmd('git',
'branch',
None if verbose else '--quiet',
'compare_commit',
result.compare_commit_id,
log_run_to_stderr=verbose)
shell_tools.run_cmd('git',
'checkout',
None if verbose else '--quiet',
'-b',
'actual_commit',
result.actual_commit_id,
log_run_to_stderr=verbose)
return prepared_env.PreparedEnv(
github_repo=None,
actual_commit_id=result.actual_commit_id,
compare_commit_id=result.compare_commit_id,
destination_directory=destination_directory,
virtual_env_path=None)
def run_cmd(*args, **kwargs):
return shell_tools.run_cmd(*args, log_run_to_stderr=False, **kwargs)
