def update_changelog(self, new_version, changes, skip_title_change=False):
"""
:param str new_version: New version
:param list[str] changes: List of changes
:param bool skip_title_change: Skip title change
:returns: Path to changelog file
"""
docs_dir = os.path.join(repo_path(), 'docs')
if not os.path.isdir(docs_dir):
os.makedirs(docs_dir)
changelog_file = os.path.join(docs_dir, 'CHANGELOG.rst')
existing_changes = os.path.exists(changelog_file) and open(changelog_file).read()
major_title = '=' * 80
minor_title = '-' * 80
with open(changelog_file, 'w') as fp:
fp.write('Version %s' % new_version + '\n')
fp.write(major_title + '\n\n')
for change in changes:
fp.write('* %s\n' % (change.replace('\n', '\n ')))
if existing_changes:
fp.write('\n')
if not skip_title_change:
existing_changes = existing_changes.replace(major_title, minor_title, 1)
fp.write(existing_changes)
return changelog_file
def update_changelog(self, new_version, changes, skip_title_change=False):
"""
:param str new_version: New version
:param list[str] changes: List of changes
:param bool skip_title_change: Skip title change
:returns: Path to changelog file
"""
docs_dir = os.path.join(repo_path(), 'docs')
if not os.path.isdir(docs_dir):
os.makedirs(docs_dir)
changelog_file = os.path.join(docs_dir, 'CHANGELOG.rst')
existing_changes = os.path.exists(changelog_file) and open(changelog_file).read()
major_title = '=' * 80
minor_title = '-' * 80
with open(changelog_file, 'w') as fp:
fp.write('Version %s' % new_version + '\n')
fp.write(major_title + '\n\n')
for change in changes:
fp.write('* %s\n' % (change.replace('\n', '\n ')))
if existing_changes:
fp.write('\n')
if not skip_title_change:
existing_changes = existing_changes.replace(major_title, minor_title, 1)
fp.write(existing_changes)
return changelog_file
def run(self):
repo = repo_path()
if repo:
click.echo('Removing build/dist folders')
silent_run("rm -rf build dist docs/_build */activate",
cwd=repo,
shell=True)
click.echo('Removing *.pyc files')
silent_run(
"find . -type d \( -path '*/.tox' -o -path '*/mppy-*' \) -prune -o -name *.pyc -exec rm {} \;",
cwd=repo,
shell=True)
if self.force:
click.echo('Removing untracked/ignored files')
silent_run('git clean -fdx')
else:
path = workspace_path()
click.echo('Cleaning {}'.format(path))
if config.clean.remove_products_older_than_days or config.clean.remove_all_products_except:
keep_time = 0
keep_products = []
if config.clean.remove_all_products_except:
click.echo('Removing all products except: %s' %
config.clean.remove_all_products_except)
keep_products = expand_product_groups(
config.clean.remove_all_products_except.split())
if config.clean.remove_products_older_than_days:
click.echo('Removing products older than %s days' %
config.clean.remove_products_older_than_days)
keep_time = time(
) - config.clean.remove_products_older_than_days * 86400
removed_products = []
for repo in repos(path):
name = product_name(repo)
modified_time = os.stat(repo).st_mtime
if keep_products and name not in keep_products or keep_time and modified_time < keep_time:
status = stat_repo(repo, return_output=True)
if (not status or 'nothing to commit' in status and
('working directory clean' in status
or 'working tree clean' in status)
and len(all_branches(repo)) <= 1):
shutil.rmtree(repo)
removed_products.append(name)
else:
click.echo(
' - Skipping "%s" as it has changes that may not be committed'
% name)
if removed_products:
click.echo('Removed ' + ', '.join(removed_products))
def get_version(self):
""" Get current version and setup.py file """
setup_file = os.path.join(repo_path(), 'setup.py')
match = VERSION_RE.search(open(setup_file).read())
if not match:
log.error('Failed to find "version=" in setup.py to get version')
sys.exit(1)
return match.group(2), setup_file
def get_version(self):
""" Get current version and setup.py file """
setup_file = os.path.join(repo_path(), 'setup.py')
match = VERSION_RE.search(open(setup_file).read())
if not match:
log.error('Failed to find "version=" in setup.py to get version')
sys.exit(1)
return match.group(2), setup_file
def changes_since_last_publish(self):
commit_msgs = extract_commit_msgs(commit_logs(limit=100, repo=repo_path()), True)
changes = []
published_version = None
for msg in commit_msgs:
if msg.startswith(PUBLISH_VERSION_PREFIX):
published_version = msg.split(PUBLISH_VERSION_PREFIX)[-1]
break
if len(msg) < 7 or IGNORE_CHANGE_RE.match(msg):
continue
changes.append(msg)
return published_version, changes
def changes_since_last_publish(self):
commit_msgs = extract_commit_msgs(commit_logs(limit=100, repo=repo_path()), True)
changes = []
published_version = None
for msg in commit_msgs:
if msg.startswith(PUBLISH_VERSION_PREFIX):
published_version = msg.split(PUBLISH_VERSION_PREFIX)[-1]
break
if len(msg) < 7 or IGNORE_CHANGE_RE.match(msg):
continue
changes.append(msg)
return published_version, changes
def run(self):
repo = repo_path()
if repo:
click.echo('Removing build/dist folders')
silent_run("rm -rf build dist docs/_build */activate", cwd=repo, shell=True)
click.echo('Removing *.pyc files')
silent_run("find . -type d \( -path '*/.tox' -o -path '*/mppy-*' \) -prune -o -name *.pyc -exec rm {} \;", cwd=repo, shell=True)
if self.force:
click.echo('Removing untracked/ignored files')
silent_run('git clean -fdx')
else:
path = workspace_path()
click.echo('Cleaning {}'.format(path))
if config.clean.remove_products_older_than_days or config.clean.remove_all_products_except:
keep_time = 0
keep_products = []
if config.clean.remove_all_products_except:
click.echo('Removing all products except: %s' % config.clean.remove_all_products_except)
keep_products = expand_product_groups(config.clean.remove_all_products_except.split())
if config.clean.remove_products_older_than_days:
click.echo('Removing products older than %s days' % config.clean.remove_products_older_than_days)
keep_time = time() - config.clean.remove_products_older_than_days * 86400
removed_products = []
for repo in repos(path):
name = product_name(repo)
modified_time = os.stat(repo).st_mtime
if keep_products and name not in keep_products or keep_time and modified_time < keep_time:
status = stat_repo(repo, return_output=True)
if (not status or 'nothing to commit' in status and
('working directory clean' in status or 'working tree clean' in status) and
len(all_branches(repo)) <= 1):
shutil.rmtree(repo)
removed_products.append(name)
else:
click.echo(' - Skipping "%s" as it has changes that may not be committed' % name)
if removed_products:
click.echo('Removed ' + ', '.join(removed_products))
def bump_version(self, major=False, minor=False):
"""
Bump the version (defaults to patch) in setup.py
:param bool major: Bump major version only
:param bool minor: Bump minor version only
"""
setup_file = os.path.join(repo_path(), 'setup.py')
if not os.path.exists(setup_file):
log.error(setup_file + ' does not exist.')
sys.exit(1)
def replace_version(match):
global new_version
version_parts = match.group(2).split('.')
i = 0 if major else (1 if minor else 2)
while len(version_parts) < i + 1:
version_parts.append(0)
for j in range(i + 1, len(version_parts)):
version_parts[j] = '0'
version_parts[i] = str(int(version_parts[i]) + 1)
new_version = '.'.join(version_parts)
return 'version=' + match.group(1) + new_version + match.group(1)
content = VERSION_RE.sub(replace_version, open(setup_file).read())
with open(setup_file, 'w') as fp:
fp.write(content)
if not new_version:
log.error('Failed to find "version=" in setup.py to bump version')
sys.exit(1)
return new_version, setup_file
def bump_version(self, major=False, minor=False):
"""
Bump the version (defaults to patch) in setup.py
:param bool major: Bump major version only
:param bool minor: Bump minor version only
"""
setup_file = os.path.join(repo_path(), 'setup.py')
if not os.path.exists(setup_file):
log.error(setup_file + ' does not exist.')
sys.exit(1)
def replace_version(match):
global new_version
version_parts = match.group(2).split('.')
i = 0 if major else (1 if minor else 2)
while len(version_parts) < i + 1:
version_parts.append(0)
for j in range(i+1, len(version_parts)):
version_parts[j] = '0'
version_parts[i] = str(int(version_parts[i]) + 1)
new_version = '.'.join(version_parts)
return 'version=' + match.group(1) + new_version + match.group(1)
content = VERSION_RE.sub(replace_version, open(setup_file).read())
with open(setup_file, 'w') as fp:
fp.write(content)
if not new_version:
log.error('Failed to find "version=" in setup.py to bump version')
sys.exit(1)
return new_version, setup_file
def run(self):
current = current_branch()
repo = git.Repo(path=repo_path())
if self.branch and self.downstreams:
log.error('Branch and --downstreams are mutually exclusive. Please use one or the other.')
sys.exit(1)
if repo.is_dirty(untracked_files=True):
log.error(
'Your repo has untracked or modified files in working dir or in staging index. Please cleanup before doing merge')
sys.exit(1)
if not self.skip_update:
self.commander.run('update', quiet=True)
if self.branch:
click.echo('Merging {} into {}'.format(self.branch, current))
if not self.skip_update:
checkout_branch(self.branch)
self.commander.run('update', quiet=True)
checkout_branch(current)
all_commits = self.get_unmerged_commits(repo, self.branch, current)
self.merge_commits(self.branch, all_commits, self.skip_commits)
elif self.downstreams:
if not self.merge_branches:
self.merge_branches = config.merge.branches
if not self.merge_branches:
log.error('Config merge.branches must be configured with a list of branches to merge to, or '
'use --merge-branches to provide a list')
sys.exit(1)
branches = self.merge_branches.split()
if current not in branches:
log.error('Current branch %s not found in config merge.branches (%s)', current, self.merge_branches)
sys.exit(1)
last = current
downstream_branches = branches[branches.index(last) + 1:]
if not downstream_branches:
click.echo('You are currently on the last branch, so no downstream branches to merge.')
click.echo('Switch to the branch that you want to merge from first, and then re-run')
sys.exit(0)
for branch in downstream_branches:
checkout_branch(branch)
commits = self._unmerged_commits(repo, last, branch)
if self.quiet and not commits:
last = branch
continue
click.echo('Merging {} into {}'.format(last, branch))
if not self.skip_update:
self.commander.run('update', quiet=True)
if self.dry_run:
self.get_unmerged_commits(repo, last, branch)
else:
if self.allow_commits:
if commits:
for commit in commits.split('\n'):
# Not performant / ok as # of allow_commits should be low
allowed_commit = (' Merge branch ' in commit
or ' Merge pull request ' in commit
or any(allow_commit in commit for allow_commit in self.allow_commits))
if not allowed_commit:
click.echo('Found a commit that was not allowed to be merged:'.format(last))
click.echo(' {}'.format(commit))
raise NotAllowedCommit(commit)
self.merge_commits(last, commits, self.skip_commits)
if self.validation:
process_run(self.validation)
if self.validation:
process_run(self.validation)
self.commander.run('push', all_remotes=True, skip_style_check=True)
last = branch
else:
log.error(
'Please specify either a branch to merge from or --downstreams to merge to all downstream branches')
sys.exit(1)
def run(self):
if self.minor and self.major:
log.error('--minor and --major are mutually exclusive, please use only one.')
return
repo_check()
pypirc = LocalConfig('~/.pypirc')
repository = pypirc.get(self.repo, 'repository')
username = pypirc.get(self.repo, 'username')
password = pypirc.get(self.repo, 'password')
repo_title = 'PyPI' if self.repo == 'pypi' else self.repo.title()
if not repository:
log.error('Please add repository / username to [%s] section in ~/.pypirc', self.repo)
sys.exit(1)
if not username:
username = getpass.getuser('{} Username: '******'{} Password: '******'update')
published_version, changes = self.changes_since_last_publish()
if not changes:
click.echo('There are no changes since last publish')
sys.exit(0)
silent_run('rm -rf dist/*', shell=True, cwd=repo_path())
if self.major or self.minor:
new_version, setup_file = self.bump_version(major=self.major, minor=self.minor)
major_minor = 'major' if self.major else 'minor'
self.commander.run('commit', msg=f'Bump {major_minor} version', files=[setup_file], push=2,
skip_style_check=True)
else:
current_version, setup_file = self.get_version()
# Previously, we publish the current version in setup.py, so need to bump it first before
# we can publish a new version.
if published_version == current_version:
new_version, setup_file = self.bump_version()
else:
new_version = current_version
changelog_file = self.update_changelog(new_version, changes, self.minor or self.major)
tox = ToxIni()
envs = [e for e in tox.envlist if e != 'style']
if envs:
env = envs[0]
if len(envs) > 1:
log.debug('Found multiple default envs in tox.ini, will use first one to build: %s', env)
else:
click.echo('Odd, there are no default envs in tox.ini, so we can not build.')
sys.exit(1)
envdir = tox.envdir(env)
python = os.path.join(envdir, 'bin', 'python')
click.echo('Building source/built distribution')
silent_run(f'{python} setup.py sdist bdist_wheel', cwd=repo_path())
click.echo('Uploading to ' + repo_title)
try:
run('twine upload -r "{repo}" -u "{username}" -p "{password}" dist/*'.format(
repo=self.repo,
username=username,
password=password), shell=True, cwd=repo_path(), silent=2)
except Exception:
sys.exit(1)
self.bump_version()
self.commander.run('commit', msg=PUBLISH_VERSION_PREFIX + new_version, push=2,
files=[setup_file, changelog_file],
skip_style_check=True)
def run(self):
current = current_branch()
repo = git.Repo(path=repo_path())
if self.branch and self.downstreams:
log.error('Branch and --downstreams are mutually exclusive. Please use one or the other.')
sys.exit(1)
if repo.is_dirty(untracked_files=True):
log.error('Your repo has untracked or modified files in working dir or in staging index. Please cleanup before doing merge')
sys.exit(1)
if not self.skip_update:
self.commander.run('update', quiet=True)
if self.branch:
click.echo('Merging {} into {}'.format(self.branch, current))
if self.dry_run:
self.show_unmerged_commits(repo, self.branch, current)
else:
if not self.skip_update:
checkout_branch(self.branch)
self.commander.run('update', quiet=True)
checkout_branch(current)
merge_branch(self.branch, strategy=self.strategy)
elif self.downstreams:
if not self.merge_branches:
self.merge_branches = config.merge.branches
if not self.merge_branches:
log.error('Config merge.branches must be configured with a list of branches to merge to, or '
'use --merge-branches to provide a list')
sys.exit(1)
branches = self.merge_branches.split()
if current not in branches:
log.error('Current branch %s not found in config merge.branches (%s)', current, self.merge_branches)
sys.exit(1)
last = current
downstream_branches = branches[branches.index(last)+1:]
if not downstream_branches:
click.echo('You are currently on the last branch, so no downstream branches to merge.')
click.echo('Switch to the branch that you want to merge from first, and then re-run')
sys.exit(0)
for branch in downstream_branches:
checkout_branch(branch)
commits = self._unmerged_commits(repo, last, branch)
if self.quiet and not commits:
last = branch
continue
click.echo('Merging {} into {}'.format(last, branch))
if not self.skip_update:
self.commander.run('update', quiet=True)
if self.dry_run:
self.show_unmerged_commits(repo, last, branch)
else:
if self.allow_commits:
if commits:
for commit in commits.split('\n'):
# Not performant / ok as # of allow_commits should be low
allowed_commit = (' Merge branch ' in commit
or ' Merge pull request ' in commit
or any(allow_commit in commit for allow_commit in self.allow_commits))
if not allowed_commit:
click.echo('Found a commit that was not allowed to be merged:'.format(last))
click.echo(' {}'.format(commit))
raise NotAllowedCommit(commit)
merge_branch(last, strategy=self.strategy)
if self.validation:
process_run(self.validation)
self.commander.run('push', all_remotes=True, skip_style_check=True)
last = branch
else:
log.error('Please specify either a branch to merge from or --downstreams to merge to all downstream branches')
sys.exit(1)
def run(self):
if self.test_dependents:
name = product_name()
# Convert None to list for tuple([])
if not self.env_or_file:
self.env_or_file = []
if not self.extra_args:
self.extra_args = []
test_args = (
('env_or_file', tuple(self.env_or_file)),
('return_output', True),
('num_processes', self.num_processes),
('silent', True),
('debug', self.debug),
('extra_args', tuple(self.extra_args))
)
test_repos = [repo_path()]
test_repos.extend(r for r in repos(workspace_path()) if self.product_depends_on(r, name) and r not in test_repos)
test_args = [(r, test_args, self.__class__) for r in test_repos]
def test_done(result):
name, output = result
success, summary = self.summarize(output)
if success:
click.echo('{}: {}'.format(name, summary))
else:
temp_output_file = os.path.join(tempfile.gettempdir(), 'test-%s.out' % name)
with open(temp_output_file, 'w') as fp:
fp.write(output)
temp_output_file = 'See ' + temp_output_file
log.error('%s: %s', name, '\n\t'.join([summary, temp_output_file]))
def show_remaining(completed, all_args):
completed_repos = set(product_name(r) for r, _, _ in completed)
all_repos = set(product_name(r) for r, _, _ in all_args)
remaining_repos = sorted(list(all_repos - completed_repos))
if len(remaining_repos):
repo = remaining_repos.pop()
more = '& %d more' % len(remaining_repos) if remaining_repos else ''
return '%s %s' % (repo, more)
else:
return 'None'
repo_results = parallel_call(test_repo, test_args, callback=test_done, show_progress=show_remaining, progress_title='Remaining')
for result in list(repo_results.values()):
if isinstance(result, tuple):
_, result = result
success, _ = self.summarize(result)
if not (success or self.return_output):
sys.exit(1)
return dict(list(repo_results.values()))
if not self.repo:
self.repo = project_path()
# Strip out venv bin path to python to avoid issues with it being removed when running tox
if 'VIRTUAL_ENV' in os.environ:
venv_bin = os.environ['VIRTUAL_ENV']
os.environ['PATH'] = os.pathsep.join([p for p in os.environ['PATH'].split(os.pathsep)
if os.path.exists(p) and not p.startswith(venv_bin)])
envs = []
files = []
if self.env_or_file:
for ef in self.env_or_file:
if os.path.exists(ef):
files.append(os.path.abspath(ef))
else:
envs.append(ef)
pytest_args = ''
if self.match_test or self.num_processes is not None or files or self.extra_args:
pytest_args = []
if self.match_test:
pytest_args.append('-k ' + self.match_test)
if self.num_processes is None: # Skip parallel for targeted test run / works better with pdb
self.num_processes = 0
if self.num_processes is not None:
pytest_args.append('-n ' + str(self.num_processes))
if self.extra_args:
pytest_args.extend(self.extra_args)
if files:
pytest_args.extend(files)
pytest_args = ' '.join(pytest_args)
os.environ['PYTESTARGS'] = pytest_args
tox = ToxIni(self.repo, self.tox_ini)
if not envs:
envs = tox.envlist
# Prefer 'test' over 'cover' when there are pytest args as cover is likely to fail and distract from
# test results. And also remove style as user is focused on fixing a test, and style for the whole project
# isn't interesting yet.
if pytest_args:
if 'cover' in envs:
python = tox.get(tox.envsection('cover'), 'basepython')
version = ''.join(python.strip('python').split('.')) if python else '36'
envs[envs.index('cover')] = 'py' + version
if 'style' in envs:
envs.remove('style')
env_commands = {}
if self.install_only and not self.redevelop:
self.redevelop = 1
if self.show_dependencies:
if 'style' in envs:
envs.remove('style')
for env in envs:
self.show_installed_dependencies(tox, env, filter_name=self.show_dependencies)
elif self.install_editable:
if 'style' in envs:
envs.remove('style')
for env in envs:
if len(envs) > 1:
print(env + ':')
self.install_editable_dependencies(tox, env, editable_products=self.install_editable)
elif self.redevelop:
if self.tox_cmd:
cmd = self.tox_cmd
else:
cmd = ['tox', '-c', tox.tox_ini]
if envs:
cmd.extend(['-e', ','.join(envs)])
if self.redevelop > 1:
cmd.append('-r')
if self.install_only:
cmd.append('--notest')
output = run(cmd, cwd=self.repo, raises=not self.return_output, silent=self.silent, return_output=self.return_output)
if not output:
if self.return_output:
return False
else:
sys.exit(1)
for env in envs:
env_commands[env] = ' '.join(cmd)
# Touch envdir
envdir = tox.envdir(env)
if os.path.exists(envdir):
os.utime(envdir, None)
# Strip entry version
self._strip_version_from_entry_scripts(tox, env)
if self.return_output:
return output
else:
for env in envs:
envdir = tox.envdir(env)
def requirements_updated():
req_mtime = 0
requirements_files = ['requirements.txt', 'pinned.txt', 'tox.ini']
for req_file in requirements_files:
req_path = os.path.join(self.repo, req_file)
if os.path.exists(req_path):
req_mtime = max(req_mtime, os.stat(req_path).st_mtime)
return req_mtime > os.stat(envdir).st_mtime
if not os.path.exists(envdir) or requirements_updated():
env_commands.update(
self.commander.run('test', env_or_file=[env], repo=self.repo, redevelop=True, tox_cmd=self.tox_cmd,
tox_ini=self.tox_ini, tox_commands=self.tox_commands, match_test=self.match_test,
num_processes=self.num_processes, silent=self.silent,
debug=self.debug, extra_args=self.extra_args))
continue
commands = self.tox_commands.get(env) or tox.commands(env)
env_commands[env] = '\n'.join(commands)
for command in commands:
full_command = os.path.join(envdir, 'bin', command)
command_path = full_command.split()[0]
if os.path.exists(command_path):
if 'pytest' in full_command or 'py.test' in full_command:
if 'PYTESTARGS' in full_command:
full_command = full_command.replace('{env:PYTESTARGS:}', pytest_args)
else:
full_command += ' ' + pytest_args
activate = '. ' + os.path.join(envdir, 'bin', 'activate')
output = run(activate + '; ' + full_command, shell=True, cwd=self.repo, raises=False, silent=self.silent,
return_output=self.return_output)
if not output:
if self.return_output:
return False
else:
sys.exit(1)
if not self.silent and (len(envs) > 1 or env == 'style'):
click.secho(f'{env}: OK', fg='green')
if self.return_output:
return output
else:
log.error('%s does not exist', command_path)
if self.return_output:
return False
else:
sys.exit(1)
return env_commands
def run(self):
if self.minor and self.major:
log.error('--minor and --major are mutually exclusive, please use only one.')
return
repo_check()
pypirc = LocalConfig('~/.pypirc')
repository = pypirc.get(self.repo, 'repository')
username = pypirc.get(self.repo, 'username')
password = pypirc.get(self.repo, 'password')
repo_title = 'PyPI' if self.repo == 'pypi' else self.repo.title()
if not repository:
log.error('Please add repository / username to [%s] section in ~/.pypirc', self.repo)
sys.exit(1)
if not username:
username = getpass.getuser('{} Username: '******'{} Password: '******'update')
published_version, changes = self.changes_since_last_publish()
if not changes:
click.echo('There are no changes since last publish')
sys.exit(0)
silent_run('rm -rf dist/*', shell=True, cwd=repo_path())
if self.major or self.minor:
new_version, setup_file = self.bump_version(major=self.major, minor=self.minor)
major_minor = 'major' if self.major else 'minor'
self.commander.run('commit', msg=f'Bump {major_minor} version', files=[setup_file], push=2,
skip_style_check=True)
else:
current_version, setup_file = self.get_version()
# Previously, we publish the current version in setup.py, so need to bump it first before
# we can publish a new version.
if published_version == current_version:
new_version, setup_file = self.bump_version()
else:
new_version = current_version
changelog_file = self.update_changelog(new_version, changes, self.minor or self.major)
tox = ToxIni()
envs = [e for e in tox.envlist if e != 'style']
if envs:
env = envs[0]
if len(envs) > 1:
log.debug('Found multiple default envs in tox.ini, will use first one to build: %s', env)
else:
click.echo('Odd, there are no default envs in tox.ini, so we can not build.')
sys.exit(1)
envdir = tox.envdir(env)
python = os.path.join(envdir, 'bin', 'python')
click.echo('Building source/built distribution')
silent_run(f'{python} setup.py sdist bdist_wheel', cwd=repo_path())
click.echo('Uploading to ' + repo_title)
try:
run('twine upload -r "{repo}" -u "{username}" -p "{password}" dist/*'.format(
repo=self.repo,
username=username,
password=password), shell=True, cwd=repo_path(), silent=2)
except Exception:
sys.exit(1)
self.bump_version()
self.commander.run('commit', msg=PUBLISH_VERSION_PREFIX + new_version, push=2, files=[setup_file, changelog_file],
skip_style_check=True)
def run(self):
if self.test_dependents:
name = product_name()
# Convert None to list for tuple([])
if not self.env_or_file:
self.env_or_file = []
if not self.extra_args:
self.extra_args = []
test_args = (('env_or_file', tuple(self.env_or_file)),
('return_output', True), ('num_processes',
self.num_processes),
('silent', True), ('debug', self.debug),
('extra_args', tuple(self.extra_args)))
test_repos = [repo_path()]
test_repos.extend(
r for r in repos(workspace_path())
if self.product_depends_on(r, name) and r not in test_repos)
test_args = [(r, test_args, self.__class__) for r in test_repos]
def test_done(result):
name, output = result
success, summary = self.summarize(output)
if success:
click.echo('{}: {}'.format(name, summary))
else:
temp_output_file = os.path.join(tempfile.gettempdir(),
'test-%s.out' % name)
with open(temp_output_file, 'w') as fp:
fp.write(output)
temp_output_file = 'See ' + temp_output_file
log.error('%s: %s', name,
'\n\t'.join([summary, temp_output_file]))
def show_remaining(completed, all_args):
completed_repos = set(product_name(r) for r, _, _ in completed)
all_repos = set(product_name(r) for r, _, _ in all_args)
remaining_repos = sorted(list(all_repos - completed_repos))
if len(remaining_repos):
repo = remaining_repos.pop()
more = '& %d more' % len(
remaining_repos) if remaining_repos else ''
return '%s %s' % (repo, more)
else:
return 'None'
repo_results = parallel_call(test_repo,
test_args,
callback=test_done,
show_progress=show_remaining,
progress_title='Remaining')
for result in list(repo_results.values()):
if isinstance(result, tuple):
_, result = result
success, _ = self.summarize(result)
if not (success or self.return_output):
sys.exit(1)
return dict(list(repo_results.values()))
if not self.repo:
self.repo = project_path()
# Strip out venv bin path to python to avoid issues with it being removed when running tox
if 'VIRTUAL_ENV' in os.environ:
venv_bin = os.environ['VIRTUAL_ENV']
os.environ['PATH'] = os.pathsep.join([
p for p in os.environ['PATH'].split(os.pathsep)
if os.path.exists(p) and not p.startswith(venv_bin)
])
envs = []
files = []
if self.env_or_file:
for ef in self.env_or_file:
if os.path.exists(ef):
files.append(os.path.abspath(ef))
else:
envs.append(ef)
pytest_args = ''
if self.match_test or self.num_processes is not None or files or self.extra_args:
pytest_args = []
if self.match_test:
pytest_args.append('-k ' + self.match_test)
if self.num_processes is None: # Skip parallel for targeted test run / works better with pdb
self.num_processes = 0
if self.num_processes is not None:
pytest_args.append('-n ' + str(self.num_processes))
if self.extra_args:
pytest_args.extend(self.extra_args)
if files:
pytest_args.extend(files)
pytest_args = ' '.join(pytest_args)
os.environ['PYTESTARGS'] = pytest_args
tox = ToxIni(self.repo, self.tox_ini)
if not envs:
envs = tox.envlist
# Prefer 'test' over 'cover' when there are pytest args as cover is likely to fail and distract from
# test results. And also remove style as user is focused on fixing a test, and style for the whole project
# isn't interesting yet.
if pytest_args:
if 'cover' in envs:
python = tox.get(tox.envsection('cover'), 'basepython')
version = ''.join(
python.strip('python').split('.')) if python else '36'
envs[envs.index('cover')] = 'py' + version
if 'style' in envs:
envs.remove('style')
env_commands = {}
if self.install_only and not self.redevelop:
self.redevelop = 1
if self.show_dependencies:
if 'style' in envs:
envs.remove('style')
for env in envs:
self.show_installed_dependencies(
tox, env, filter_name=self.show_dependencies)
elif self.install_editable:
if 'style' in envs:
envs.remove('style')
for env in envs:
if len(envs) > 1:
print(env + ':')
self.install_editable_dependencies(
tox, env, editable_products=self.install_editable)
elif self.redevelop:
if self.tox_cmd:
cmd = self.tox_cmd
else:
cmd = ['tox', '-c', tox.tox_ini]
if envs:
cmd.extend(['-e', ','.join(envs)])
if self.redevelop > 1:
cmd.append('-r')
if self.install_only:
cmd.append('--notest')
output = run(cmd,
cwd=self.repo,
raises=not self.return_output,
silent=self.silent,
return_output=self.return_output)
if not output:
if self.return_output:
return False
else:
sys.exit(1)
for env in envs:
env_commands[env] = ' '.join(cmd)
# Touch envdir
envdir = tox.envdir(env)
if os.path.exists(envdir):
os.utime(envdir, None)
# Strip entry version
self._strip_version_from_entry_scripts(tox, env)
if self.return_output:
return output
else:
for env in envs:
envdir = tox.envdir(env)
def requirements_updated():
req_mtime = 0
requirements_files = [
'requirements.txt', 'pinned.txt', 'tox.ini'
]
for req_file in requirements_files:
req_path = os.path.join(self.repo, req_file)
if os.path.exists(req_path):
req_mtime = max(req_mtime,
os.stat(req_path).st_mtime)
return req_mtime > os.stat(envdir).st_mtime
if not os.path.exists(envdir) or requirements_updated():
env_commands.update(
self.commander.run('test',
env_or_file=[env],
repo=self.repo,
redevelop=True,
tox_cmd=self.tox_cmd,
tox_ini=self.tox_ini,
tox_commands=self.tox_commands,
match_test=self.match_test,
num_processes=self.num_processes,
silent=self.silent,
debug=self.debug,
extra_args=self.extra_args))
continue
commands = self.tox_commands.get(env) or tox.commands(env)
env_commands[env] = '\n'.join(commands)
for command in commands:
full_command = os.path.join(envdir, 'bin', command)
command_path = full_command.split()[0]
if os.path.exists(command_path):
if 'pytest' in full_command or 'py.test' in full_command:
if 'PYTESTARGS' in full_command:
full_command = full_command.replace(
'{env:PYTESTARGS:}', pytest_args)
else:
full_command += ' ' + pytest_args
activate = '. ' + os.path.join(envdir, 'bin',
'activate')
output = run(activate + '; ' + full_command,
shell=True,
cwd=self.repo,
raises=False,
silent=self.silent,
return_output=self.return_output)
if not output:
if self.return_output:
return False
else:
sys.exit(1)
if not self.silent and (len(envs) > 1
or env == 'style'):
click.secho(f'{env}: OK', fg='green')
if self.return_output:
return output
else:
log.error('%s does not exist', command_path)
if self.return_output:
return False
else:
sys.exit(1)
return env_commands
