def show_remaining(completed, all_args):
completed_repos = set(product_name(r) for r, _, _ in completed)
all_repos = set(product_name(r) for r, _, _ in all_args)
remaining_repos = sorted(list(all_repos - completed_repos))
if len(remaining_repos):
repo = remaining_repos.pop()
more = '& %d more' % len(remaining_repos) if remaining_repos else ''
return '%s %s' % (repo, more)
else:
return 'None'
def show_remaining(completed, all_args):
completed_repos = set(product_name(r) for r, _, _ in completed)
all_repos = set(product_name(r) for r, _, _ in all_args)
remaining_repos = sorted(list(all_repos - completed_repos))
if len(remaining_repos):
repo = remaining_repos.pop()
more = '& %d more' % len(
remaining_repos) if remaining_repos else ''
return '%s %s' % (repo, more)
else:
return 'None'
def _update_repo(repo, raises=False, verbose=1):
name = product_name(repo)
try:
if verbose == 1:
click.echo('Updating ' + name)
branch = current_branch(repo)
parent = parent_branch(branch)
if parent:
checkout_branch(parent, repo)
update_repo(repo, quiet=verbose != 2)
if parent:
if verbose == 2:
click.echo('Rebasing ' + branch)
checkout_branch(branch, repo_path=repo)
update_branch(repo=repo, parent=parent)
return True
except Exception as e:
if raises:
raise
else:
log.error('%s: %s', name, e)
return False
def _strip_version_from_entry_scripts(self, tox, env):
""" Strip out version spec "==1.2.3" from entry scripts as they require re-develop when version is changed in develop mode. """
name = product_name(tox.path)
script_bin = tox.bindir(env)
if os.path.exists(script_bin):
name_version_re = re.compile('%s==[0-9\.]+' % name)
removed_from = []
for script in os.listdir(script_bin):
script_path = os.path.join(script_bin, script)
if os.path.isfile(script_path):
try:
with open(script_path) as fp:
script = fp.read()
except Exception:
continue # Binary files
if name_version_re.search(script):
new_script = name_version_re.sub(name, script)
with open(script_path, 'w') as fp:
fp.write(new_script)
removed_from.append(os.path.basename(script_path))
if removed_from:
log.debug('Removed version spec from entry script(s): %s', ', '.join(removed_from))
def _update_repo(repo, raises=False, verbose=1):
name = product_name(repo)
try:
if verbose == 1:
click.echo('Updating ' + name)
branch = current_branch(repo)
parent = parent_branch(branch)
if parent:
checkout_branch(parent, repo)
update_repo(repo, quiet=verbose != 2)
if parent:
if verbose == 2:
click.echo('Rebasing ' + branch)
checkout_branch(branch, repo_path=repo)
update_branch(repo=repo, parent=parent)
return True
except Exception as e:
if raises:
raise
else:
log.error('%s: %s', name, e)
return False
def run(self):
try:
scm_repos = repos()
in_repo = is_repo(os.getcwd())
optional = len(scm_repos) == 1
pager = ProductPager(optional=optional)
for repo in scm_repos:
stat_path = os.getcwd() if in_repo else repo
output = stat_repo(stat_path, return_output=True, with_color=True)
nothing_to_commit = ('nothing to commit' in output and
'Your branch is ahead of' not in output and
'Your branch is behind' not in output)
branches = all_branches(repo, verbose=True)
child_branches = [b for b in branches if '@' in b]
if len(child_branches) >= 1 or len(scm_repos) == 1:
show_branches = branches if len(scm_repos) == 1 else child_branches
remotes = all_remotes() if len(scm_repos) == 1 else []
remotes = '\n# Remotes: {}'.format(' '.join(remotes)) if len(remotes) > 1 else ''
if nothing_to_commit:
output = '# Branches: {}{}'.format(' '.join(show_branches), remotes)
nothing_to_commit = False
elif len(show_branches) > 1:
output = '# Branches: {}{}\n#\n{}'.format(' '.join(show_branches), remotes, output)
if output and not nothing_to_commit:
pager.write(product_name(repo), output)
finally:
pager.close_and_wait()
def run(self):
if self.context:
scm_repos = [os.getcwd()]
else:
scm_repos = repos()
optional = len(scm_repos) == 1
pager = ProductPager(optional=optional)
for repo in scm_repos:
with log_exception():
cur_branch = current_branch(repo)
branch = (parent_branch(cur_branch)
or 'master') if self.parent else None
color = not pager.pager or 'less' in pager.pager.args
output = diff_repo(repo,
branch=branch,
context=self.context,
return_output=True,
name_only=self.name_only,
color=color)
if output:
pager.write(product_name(repo), output, cur_branch)
pager.close_and_wait()
def _strip_version_from_entry_scripts(self, tox, env):
""" Strip out version spec "==1.2.3" from entry scripts as they require re-develop when version is changed in develop mode. """
name = product_name(tox.path)
script_bin = tox.bindir(env)
if os.path.exists(script_bin):
name_version_re = re.compile('%s==[0-9\.]+' % name)
removed_from = []
for script in os.listdir(script_bin):
script_path = os.path.join(script_bin, script)
if os.path.isfile(script_path):
try:
with open(script_path) as fp:
script = fp.read()
except Exception:
continue # Binary files
if name_version_re.search(script):
new_script = name_version_re.sub(name, script)
with open(script_path, 'w') as fp:
fp.write(new_script)
removed_from.append(os.path.basename(script_path))
if removed_from:
log.debug('Removed version spec from entry script(s): %s',
', '.join(removed_from))
def summarize(cls, tests, include_no_tests=True):
"""
Summarize the test results
:param dict|str tests: Map of product name to test result, or the test result of the current prod.
:param bool include_no_tests: Include "No tests" results when there are no tests found.
:return: A tuple of (success, list(summaries)) where success is True if all tests pass and summaries
is a list of passed/failed summary of each test or just str if 'tests' param is str.
"""
prod_name = product_name()
if isinstance(tests, dict):
product_tests = tests
else:
product_tests = {}
product_tests[prod_name] = tests
success = True
summaries = []
def append_summary(summary, name=None):
if len(product_tests) == 1:
summaries.append(summary)
else:
summaries.append("%s: %s" % (name, summary))
for name in sorted(product_tests, key=lambda n: n == prod_name or n):
if not product_tests[name]:
success = False
append_summary('Test failed / No output', name)
elif product_tests[name] is True:
append_summary('Test successful / No output', name)
elif 'collected 0 items' in product_tests[
name] and 'error' not in product_tests[name]:
append_summary('No tests')
else:
match = TEST_RE.search(product_tests[name])
if not match: # Fall back to build if there are no tests.
match = BUILD_RE.search(product_tests[name])
if match:
append_summary(match.group(0), name)
else:
append_summary('No test summary found in output', name)
summary_lines = [
l for l in product_tests[name].replace('xfailed',
'').split('\n')
if l.startswith('===') and 'warnings summary' not in l
]
if not len(summary_lines) == 2 or 'failed' in summary_lines[
-1] or 'error' in summary_lines[-1]:
success = False
return success, summaries if isinstance(tests, dict) else summaries[0]
def run(self):
repo = repo_path()
if repo:
click.echo('Removing build/dist folders')
silent_run("rm -rf build dist docs/_build */activate",
cwd=repo,
shell=True)
click.echo('Removing *.pyc files')
silent_run(
"find . -type d \( -path '*/.tox' -o -path '*/mppy-*' \) -prune -o -name *.pyc -exec rm {} \;",
cwd=repo,
shell=True)
if self.force:
click.echo('Removing untracked/ignored files')
silent_run('git clean -fdx')
else:
path = workspace_path()
click.echo('Cleaning {}'.format(path))
if config.clean.remove_products_older_than_days or config.clean.remove_all_products_except:
keep_time = 0
keep_products = []
if config.clean.remove_all_products_except:
click.echo('Removing all products except: %s' %
config.clean.remove_all_products_except)
keep_products = expand_product_groups(
config.clean.remove_all_products_except.split())
if config.clean.remove_products_older_than_days:
click.echo('Removing products older than %s days' %
config.clean.remove_products_older_than_days)
keep_time = time(
) - config.clean.remove_products_older_than_days * 86400
removed_products = []
for repo in repos(path):
name = product_name(repo)
modified_time = os.stat(repo).st_mtime
if keep_products and name not in keep_products or keep_time and modified_time < keep_time:
status = stat_repo(repo, return_output=True)
if (not status or 'nothing to commit' in status and
('working directory clean' in status
or 'working tree clean' in status)
and len(all_branches(repo)) <= 1):
shutil.rmtree(repo)
removed_products.append(name)
else:
click.echo(
' - Skipping "%s" as it has changes that may not be committed'
% name)
if removed_products:
click.echo('Removed ' + ', '.join(removed_products))
def test_repo(repo, test_args, test_class):
name = product_name(repo)
branch = current_branch(repo)
on_branch = '#' + branch if branch != 'master' and branch is not None else ''
click.echo('Testing {} {}'.format(name, on_branch))
return name, test_class(repo=repo, **dict(test_args)).run()
def test_repo(repo, test_args, test_class):
name = product_name(repo)
branch = current_branch(repo)
on_branch = '#' + branch if branch != 'master' and branch is not None else ''
click.echo('Testing {} {}'.format(name, on_branch))
return name, test_class(repo=repo, **dict(test_args)).run()
def summarize(cls, tests, include_no_tests=True):
"""
Summarize the test results
:param dict|str tests: Map of product name to test result, or the test result of the current prod.
:param bool include_no_tests: Include "No tests" results when there are no tests found.
:return: A tuple of (success, list(summaries)) where success is True if all tests pass and summaries
is a list of passed/failed summary of each test or just str if 'tests' param is str.
"""
prod_name = product_name()
if isinstance(tests, dict):
product_tests = tests
else:
product_tests = {}
product_tests[prod_name] = tests
success = True
summaries = []
def append_summary(summary, name=None):
if len(product_tests) == 1:
summaries.append(summary)
else:
summaries.append("%s: %s" % (name, summary))
for name in sorted(product_tests, key=lambda n: n == prod_name or n):
if not product_tests[name]:
success = False
append_summary('Test failed / No output', name)
elif product_tests[name] is True:
append_summary('Test successful / No output', name)
elif 'collected 0 items' in product_tests[name] and 'error' not in product_tests[name]:
append_summary('No tests')
else:
match = TEST_RE.search(product_tests[name])
if not match: # Fall back to build if there are no tests.
match = BUILD_RE.search(product_tests[name])
if match:
append_summary(match.group(0), name)
else:
append_summary('No test summary found in output', name)
summary_lines = [l for l in product_tests[name].replace('xfailed', '').split('\n')
if l.startswith('===') and 'warnings summary' not in l]
if not len(summary_lines) == 2 or 'failed' in summary_lines[-1] or 'error' in summary_lines[-1]:
success = False
return success, summaries if isinstance(tests, dict) else summaries[0]
def install_editable_dependencies(self, tox, env, editable_products):
name = product_name(tox.path)
editable_products = expand_product_groups(editable_products)
dependencies_output = self.show_installed_dependencies(tox, env, return_output=True)
if not dependencies_output:
log.debug('%s is not installed or there is no dependencies - skipping editable mode changes', name)
return
try:
product_dependencies_list = json.loads(dependencies_output)
except Exception as e:
log.debug('Failed to get installed dependencies - skipping editable mode changes: %s', e)
return
product_dependencies = {}
for dep, _, path in product_dependencies_list:
product_dependencies[dep] = path
available_products = [os.path.basename(r) for r in product_repos()]
libs = [d for d in editable_products if d in available_products and d in product_dependencies and
tox.envdir(env) in product_dependencies[d]]
already_editable = [d for d in editable_products if d in product_dependencies and
tox.envdir(env) not in product_dependencies[d]]
for lib in already_editable:
click.echo('{} is already installed in editable mode.'.format(lib))
not_dependent = [d for d in editable_products if d not in product_dependencies]
for lib in not_dependent:
log.debug('%s is not currently installed (not a dependency) and will be ignored.', lib)
not_available = [d for d in editable_products if d not in not_dependent and d not in available_products]
for lib in not_available:
click.echo('{} is a dependency but not checked out in workspace, and so can not be installed in editable mode.'.format(lib))
pip = tox.bindir(env, 'pip')
for lib in libs:
if not self.silent or self.debug:
click.echo('{}: Installing {} in editable mode'.format(env, lib))
with log_exception('An error occurred when installing %s in editable mode' % lib):
run([pip, 'uninstall', lib, '-y'], raises=False, silent=not self.debug)
lib_path = product_path(lib)
if os.path.exists(os.path.join(lib_path, lib, 'setup.py')):
lib_path = os.path.join(lib_path, lib)
run([pip, 'install', '--editable', lib_path], silent=not self.debug)
def run(self):
if is_repo():
if len(self.target) == 1:
if self.target[0] in all_branches():
checkout_branch(self.target[0])
click.echo('Switched to branch ' + self.target[0])
return
else:
possible_remotes = {upstream_remote()}
if '/' in self.target[0]:
possible_remotes.add(self.target[0].split('/')[0])
for pull_tags in [False, True]:
if pull_tags:
for remote in possible_remotes & set(
all_remotes()):
update_tags(remote)
if '/' in self.target[0] and 'remotes/{}'.format(
self.target[0]) in all_branches(remotes=True):
checkout_branch(self.target[0])
click.echo('Switched to branch ' +
self.target[0].split('/')[-1])
return
if 'remotes/{}/{}'.format(
upstream_remote(),
self.target[0]) in all_branches(remotes=True):
checkout_branch("{}/{}".format(
upstream_remote(), self.target[0]))
click.echo('Switched to branch ' + self.target[0])
return
checkout_files(self.target)
return
product_urls = expand_product_groups(self.target)
for product_url in product_urls:
product_url = product_url.strip('/')
product_path = product_checkout_path(product_url)
if os.path.exists(product_path):
click.echo('Updating ' + product_name(product_path))
else:
click.echo('Checking out ' + product_url)
checkout_product(product_url, product_path)
def run(self):
repo = repo_path()
if repo:
click.echo('Removing build/dist folders')
silent_run("rm -rf build dist docs/_build */activate", cwd=repo, shell=True)
click.echo('Removing *.pyc files')
silent_run("find . -type d \( -path '*/.tox' -o -path '*/mppy-*' \) -prune -o -name *.pyc -exec rm {} \;", cwd=repo, shell=True)
if self.force:
click.echo('Removing untracked/ignored files')
silent_run('git clean -fdx')
else:
path = workspace_path()
click.echo('Cleaning {}'.format(path))
if config.clean.remove_products_older_than_days or config.clean.remove_all_products_except:
keep_time = 0
keep_products = []
if config.clean.remove_all_products_except:
click.echo('Removing all products except: %s' % config.clean.remove_all_products_except)
keep_products = expand_product_groups(config.clean.remove_all_products_except.split())
if config.clean.remove_products_older_than_days:
click.echo('Removing products older than %s days' % config.clean.remove_products_older_than_days)
keep_time = time() - config.clean.remove_products_older_than_days * 86400
removed_products = []
for repo in repos(path):
name = product_name(repo)
modified_time = os.stat(repo).st_mtime
if keep_products and name not in keep_products or keep_time and modified_time < keep_time:
status = stat_repo(repo, return_output=True)
if (not status or 'nothing to commit' in status and
('working directory clean' in status or 'working tree clean' in status) and
len(all_branches(repo)) <= 1):
shutil.rmtree(repo)
removed_products.append(name)
else:
click.echo(' - Skipping "%s" as it has changes that may not be committed' % name)
if removed_products:
click.echo('Removed ' + ', '.join(removed_products))
def run(self):
if is_repo():
if len(self.target) == 1:
if self.target[0] in all_branches():
checkout_branch(self.target[0])
click.echo('Switched to branch ' + self.target[0])
return
else:
possible_remotes = {upstream_remote()}
if '/' in self.target[0]:
possible_remotes.add(self.target[0].split('/')[0])
for pull_tags in [False, True]:
if pull_tags:
for remote in possible_remotes & set(all_remotes()):
update_tags(remote)
if '/' in self.target[0] and 'remotes/{}'.format(self.target[0]) in all_branches(remotes=True):
checkout_branch(self.target[0])
click.echo('Switched to branch ' + self.target[0].split('/')[-1])
return
if 'remotes/{}/{}'.format(upstream_remote(), self.target[0]) in all_branches(remotes=True):
checkout_branch("{}/{}".format(upstream_remote(), self.target[0]))
click.echo('Switched to branch ' + self.target[0])
return
checkout_files(self.target)
return
product_urls = expand_product_groups(self.target)
for product_url in product_urls:
product_url = product_url.strip('/')
product_path = product_checkout_path(product_url)
if os.path.exists(product_path):
click.echo('Updating ' + product_name(product_path))
else:
click.echo('Checking out ' + product_url)
checkout_product(product_url, product_path)
def run(self):
if self.context:
scm_repos = [os.getcwd()]
else:
scm_repos = repos()
optional = len(scm_repos) == 1
pager = ProductPager(optional=optional)
for repo in scm_repos:
with log_exception():
cur_branch = current_branch(repo)
branch = (parent_branch(cur_branch) or 'master') if self.parent else None
color = not pager.pager or 'less' in pager.pager.args
output = diff_repo(repo, branch=branch, context=self.context, return_output=True,
name_only=self.name_only, color=color)
if output:
pager.write(product_name(repo), output, cur_branch)
pager.close_and_wait()
def run(self):
if self.products:
self.products = expand_product_groups(self.products)
select_repos = [
repo for repo in repos() if not self.products
or self.products and product_name(repo) in self.products
]
if not select_repos:
click.echo('No product found')
elif len(select_repos) == 1:
_update_repo(select_repos[0],
raises=self.raises,
verbose=0 if self.quiet else 2)
else:
if not all(parallel_call(_update_repo, select_repos).values()):
sys.exit(1)
def run(self):
try:
scm_repos = repos()
in_repo = is_repo(os.getcwd())
optional = len(scm_repos) == 1
pager = ProductPager(optional=optional)
for repo in scm_repos:
stat_path = os.getcwd() if in_repo else repo
output = stat_repo(stat_path,
return_output=True,
with_color=True)
nothing_to_commit = ('nothing to commit' in output and
'Your branch is ahead of' not in output
and 'Your branch is behind' not in output)
branches = all_branches(repo, verbose=True)
child_branches = [b for b in branches if '@' in b]
if len(child_branches) >= 1 or len(scm_repos) == 1:
show_branches = branches if len(
scm_repos) == 1 else child_branches
remotes = all_remotes() if len(scm_repos) == 1 else []
remotes = '\n# Remotes: {}'.format(
' '.join(remotes)) if len(remotes) > 1 else ''
if nothing_to_commit:
output = '# Branches: {}{}'.format(
' '.join(show_branches), remotes)
nothing_to_commit = False
elif len(show_branches) > 1:
output = '# Branches: {}{}\n#\n{}'.format(
' '.join(show_branches), remotes, output)
if output and not nothing_to_commit:
pager.write(product_name(repo), output)
finally:
pager.close_and_wait()
def run(self):
if self.test_dependents:
name = product_name()
# Convert None to list for tuple([])
if not self.env_or_file:
self.env_or_file = []
if not self.extra_args:
self.extra_args = []
test_args = (('env_or_file', tuple(self.env_or_file)),
('return_output', True), ('num_processes',
self.num_processes),
('silent', True), ('debug', self.debug),
('extra_args', tuple(self.extra_args)))
test_repos = [repo_path()]
test_repos.extend(
r for r in repos(workspace_path())
if self.product_depends_on(r, name) and r not in test_repos)
test_args = [(r, test_args, self.__class__) for r in test_repos]
def test_done(result):
name, output = result
success, summary = self.summarize(output)
if success:
click.echo('{}: {}'.format(name, summary))
else:
temp_output_file = os.path.join(tempfile.gettempdir(),
'test-%s.out' % name)
with open(temp_output_file, 'w') as fp:
fp.write(output)
temp_output_file = 'See ' + temp_output_file
log.error('%s: %s', name,
'\n\t'.join([summary, temp_output_file]))
def show_remaining(completed, all_args):
completed_repos = set(product_name(r) for r, _, _ in completed)
all_repos = set(product_name(r) for r, _, _ in all_args)
remaining_repos = sorted(list(all_repos - completed_repos))
if len(remaining_repos):
repo = remaining_repos.pop()
more = '& %d more' % len(
remaining_repos) if remaining_repos else ''
return '%s %s' % (repo, more)
else:
return 'None'
repo_results = parallel_call(test_repo,
test_args,
callback=test_done,
show_progress=show_remaining,
progress_title='Remaining')
for result in list(repo_results.values()):
if isinstance(result, tuple):
_, result = result
success, _ = self.summarize(result)
if not (success or self.return_output):
sys.exit(1)
return dict(list(repo_results.values()))
if not self.repo:
self.repo = project_path()
# Strip out venv bin path to python to avoid issues with it being removed when running tox
if 'VIRTUAL_ENV' in os.environ:
venv_bin = os.environ['VIRTUAL_ENV']
os.environ['PATH'] = os.pathsep.join([
p for p in os.environ['PATH'].split(os.pathsep)
if os.path.exists(p) and not p.startswith(venv_bin)
])
envs = []
files = []
if self.env_or_file:
for ef in self.env_or_file:
if os.path.exists(ef):
files.append(os.path.abspath(ef))
else:
envs.append(ef)
pytest_args = ''
if self.match_test or self.num_processes is not None or files or self.extra_args:
pytest_args = []
if self.match_test:
pytest_args.append('-k ' + self.match_test)
if self.num_processes is None: # Skip parallel for targeted test run / works better with pdb
self.num_processes = 0
if self.num_processes is not None:
pytest_args.append('-n ' + str(self.num_processes))
if self.extra_args:
pytest_args.extend(self.extra_args)
if files:
pytest_args.extend(files)
pytest_args = ' '.join(pytest_args)
os.environ['PYTESTARGS'] = pytest_args
tox = ToxIni(self.repo, self.tox_ini)
if not envs:
envs = tox.envlist
# Prefer 'test' over 'cover' when there are pytest args as cover is likely to fail and distract from
# test results. And also remove style as user is focused on fixing a test, and style for the whole project
# isn't interesting yet.
if pytest_args:
if 'cover' in envs:
python = tox.get(tox.envsection('cover'), 'basepython')
version = ''.join(
python.strip('python').split('.')) if python else '36'
envs[envs.index('cover')] = 'py' + version
if 'style' in envs:
envs.remove('style')
env_commands = {}
if self.install_only and not self.redevelop:
self.redevelop = 1
if self.show_dependencies:
if 'style' in envs:
envs.remove('style')
for env in envs:
self.show_installed_dependencies(
tox, env, filter_name=self.show_dependencies)
elif self.install_editable:
if 'style' in envs:
envs.remove('style')
for env in envs:
if len(envs) > 1:
print(env + ':')
self.install_editable_dependencies(
tox, env, editable_products=self.install_editable)
elif self.redevelop:
if self.tox_cmd:
cmd = self.tox_cmd
else:
cmd = ['tox', '-c', tox.tox_ini]
if envs:
cmd.extend(['-e', ','.join(envs)])
if self.redevelop > 1:
cmd.append('-r')
if self.install_only:
cmd.append('--notest')
output = run(cmd,
cwd=self.repo,
raises=not self.return_output,
silent=self.silent,
return_output=self.return_output)
if not output:
if self.return_output:
return False
else:
sys.exit(1)
for env in envs:
env_commands[env] = ' '.join(cmd)
# Touch envdir
envdir = tox.envdir(env)
if os.path.exists(envdir):
os.utime(envdir, None)
# Strip entry version
self._strip_version_from_entry_scripts(tox, env)
if self.return_output:
return output
else:
for env in envs:
envdir = tox.envdir(env)
def requirements_updated():
req_mtime = 0
requirements_files = [
'requirements.txt', 'pinned.txt', 'tox.ini'
]
for req_file in requirements_files:
req_path = os.path.join(self.repo, req_file)
if os.path.exists(req_path):
req_mtime = max(req_mtime,
os.stat(req_path).st_mtime)
return req_mtime > os.stat(envdir).st_mtime
if not os.path.exists(envdir) or requirements_updated():
env_commands.update(
self.commander.run('test',
env_or_file=[env],
repo=self.repo,
redevelop=True,
tox_cmd=self.tox_cmd,
tox_ini=self.tox_ini,
tox_commands=self.tox_commands,
match_test=self.match_test,
num_processes=self.num_processes,
silent=self.silent,
debug=self.debug,
extra_args=self.extra_args))
continue
commands = self.tox_commands.get(env) or tox.commands(env)
env_commands[env] = '\n'.join(commands)
for command in commands:
full_command = os.path.join(envdir, 'bin', command)
command_path = full_command.split()[0]
if os.path.exists(command_path):
if 'pytest' in full_command or 'py.test' in full_command:
if 'PYTESTARGS' in full_command:
full_command = full_command.replace(
'{env:PYTESTARGS:}', pytest_args)
else:
full_command += ' ' + pytest_args
activate = '. ' + os.path.join(envdir, 'bin',
'activate')
output = run(activate + '; ' + full_command,
shell=True,
cwd=self.repo,
raises=False,
silent=self.silent,
return_output=self.return_output)
if not output:
if self.return_output:
return False
else:
sys.exit(1)
if not self.silent and (len(envs) > 1
or env == 'style'):
click.secho(f'{env}: OK', fg='green')
if self.return_output:
return output
else:
log.error('%s does not exist', command_path)
if self.return_output:
return False
else:
sys.exit(1)
return env_commands
def show_installed_dependencies(self,
tox,
env,
return_output=False,
filter_name=None):
script_template = """
import json
import os
import sys
try:
from pip._internal.utils.misc import get_installed_distributions
# pip < 10
except Exception:
from pip import get_installed_distributions
# Required params to run this script
package = '%s'
json_output = %s
env = '%s'
filter_name = '%s'
cwd = os.getcwd()
workspace_dir = os.path.dirname(cwd)
try:
libs = [(p.key, p.version, p.location) for p in get_installed_distributions()]
except Exception as e:
print(e)
sys.exit(1)
output = []
if not json_output:
print(env + ':')
def strip_cwd(dir):
if dir.startswith(cwd + '/'):
dir = dir[len(cwd):].lstrip('/')
elif dir.startswith(workspace_dir):
dir = os.path.join('..', dir[len(workspace_dir):].lstrip('/'))
return dir
for lib, version, location in sorted(libs):
if filter_name and filter_name not in lib:
continue
if json_output:
output.append((lib, version, location))
else:
output.append(' %%-25s %%-10s %%s' %% (lib, version, strip_cwd(location)))
if json_output:
print(json.dumps(output))
else:
print('\\n'.join(output))
"""
name = product_name(tox.path)
filter_name = isinstance(filter_name, str) and filter_name or ''
script = script_template % (name, return_output, env, filter_name)
python = tox.bindir(env, 'python')
if not os.path.exists(python):
log.error(
'Test environment %s is not installed. Please run without -d / --show-dependencies to install it first.',
env)
sys.exit(1)
return run([python, '-c', script],
return_output=return_output,
raises=False)
def install_editable_dependencies(self, tox, env, editable_products):
name = product_name(tox.path)
editable_products = expand_product_groups(editable_products)
dependencies_output = self.show_installed_dependencies(
tox, env, return_output=True)
if not dependencies_output:
log.debug(
'%s is not installed or there is no dependencies - skipping editable mode changes',
name)
return
try:
product_dependencies_list = json.loads(dependencies_output)
except Exception as e:
log.debug(
'Failed to get installed dependencies - skipping editable mode changes: %s',
e)
return
product_dependencies = {}
for dep, _, path in product_dependencies_list:
product_dependencies[dep] = path
available_products = [os.path.basename(r) for r in product_repos()]
libs = [
d for d in editable_products
if d in available_products and d in product_dependencies
and tox.envdir(env) in product_dependencies[d]
]
already_editable = [
d for d in editable_products if d in product_dependencies
and tox.envdir(env) not in product_dependencies[d]
]
for lib in already_editable:
click.echo('{} is already installed in editable mode.'.format(lib))
not_dependent = [
d for d in editable_products if d not in product_dependencies
]
for lib in not_dependent:
log.debug(
'%s is not currently installed (not a dependency) and will be ignored.',
lib)
not_available = [
d for d in editable_products
if d not in not_dependent and d not in available_products
]
for lib in not_available:
click.echo(
'{} is a dependency but not checked out in workspace, and so can not be installed in editable mode.'
.format(lib))
pip = tox.bindir(env, 'pip')
for lib in libs:
if not self.silent or self.debug:
click.echo('{}: Installing {} in editable mode'.format(
env, lib))
with log_exception(
'An error occurred when installing %s in editable mode' %
lib):
run([pip, 'uninstall', lib, '-y'],
raises=False,
silent=not self.debug)
lib_path = product_path(lib)
if os.path.exists(os.path.join(lib_path, lib, 'setup.py')):
lib_path = os.path.join(lib_path, lib)
run([pip, 'install', '--editable', lib_path],
silent=not self.debug)
def setup_product(self):
project_path = os.getcwd()
name = product_name(project_path)
sanitized_name = re.sub('[^A-Za-z]', '_', name)
placeholder_info = '- please update <PLACEHOLDER> with appropriate value'
tox_ini = TOX_INI_TMPL.replace('{name}', name)
tox_ini_file = os.path.join(project_path, TOX_INI_FILE)
self._create_or_update_file(tox_ini_file, tox_ini)
readme_files = glob(os.path.join(project_path, 'README*'))
if readme_files:
readme_file = readme_files[0]
else:
readme_file = os.path.join(project_path, 'README.rst')
with open(readme_file, 'w') as fp:
fp.write(README_TMPL.format(name=name))
click.echo('Created {} {}'.format(self._relative_path(readme_file),
placeholder_info))
coveragerc_file = os.path.join(project_path, '.coveragerc')
self._create_or_update_file(coveragerc_file, COVERAGERC_TMPL)
gitignore_file = os.path.join(project_path, '.gitignore')
resp = requests.get(
'https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore'
)
gitignore_content = resp.text.replace('htmlcov', 'htmlcov/\ntextcov')
self._create_or_update_file(gitignore_file, gitignore_content)
setup_py_file = os.path.join(project_path, 'setup.py')
if os.path.exists(setup_py_file):
setup_content = open(setup_py_file).read()
if 'wheel' not in setup_content:
setup_content = setup_content.replace(
"setup_requires=['setuptools-git'],",
"setup_requires=['setuptools-git', 'wheel'],")
self._create_or_update_file(setup_py_file, setup_content)
else:
requirements_file = os.path.join(project_path, 'requirements.txt')
if not os.path.exists(requirements_file):
with open(requirements_file, 'w') as fp:
pass
click.echo('Created ' + self._relative_path(requirements_file))
readme_name = os.path.basename(readme_file)
requirements_name = os.path.basename(requirements_file)
with open(setup_py_file, 'w') as fp:
fp.write(SETUP_PY_TMPL %
(name, readme_name, requirements_name))
click.echo('Created {} {}'.format(
self._relative_path(setup_py_file), placeholder_info))
package_dir = os.path.join(project_path, sanitized_name)
if not os.path.exists(package_dir):
os.makedirs(package_dir)
init_file = os.path.join(package_dir, '__init__.py')
open(init_file, 'w').close()
click.echo('Created ' + self._relative_path(init_file))
test_dir = os.path.join(project_path, 'tests')
if not os.path.exists(test_dir) and not os.path.exists(
os.path.join(project_path, 'test')):
os.makedirs(test_dir)
test_file = os.path.join(test_dir, 'test_%s.py' % sanitized_name)
with open(test_file, 'w') as fp:
fp.write(
'def test_{}():\n """ Test is code\'s best friend. ^_^ """'
.format(sanitized_name))
click.echo('Created ' + self._relative_path(test_file))
def run(self):
if self.products:
self.products = expand_product_groups(self.products)
select_repos = [repo for repo in repos() if not self.products or self.products and product_name(repo)
in self.products]
if not select_repos:
click.echo('No product found')
elif len(select_repos) == 1:
_update_repo(select_repos[0], raises=self.raises, verbose=0 if self.quiet else 2)
else:
if not all(parallel_call(_update_repo, select_repos).values()):
sys.exit(1)
def show_installed_dependencies(self, tox, env, return_output=False, filter_name=None):
script_template = """
import json
import os
import sys
try:
from pip._internal.utils.misc import get_installed_distributions
# pip < 10
except Exception:
from pip import get_installed_distributions
# Required params to run this script
package = '%s'
json_output = %s
env = '%s'
filter_name = '%s'
cwd = os.getcwd()
workspace_dir = os.path.dirname(cwd)
try:
libs = [(p.key, p.version, p.location) for p in get_installed_distributions()]
except Exception as e:
print(e)
sys.exit(1)
output = []
if not json_output:
print(env + ':')
def strip_cwd(dir):
if dir.startswith(cwd + '/'):
dir = dir[len(cwd):].lstrip('/')
elif dir.startswith(workspace_dir):
dir = os.path.join('..', dir[len(workspace_dir):].lstrip('/'))
return dir
for lib, version, location in sorted(libs):
if filter_name and filter_name not in lib:
continue
if json_output:
output.append((lib, version, location))
else:
output.append(' %%-25s %%-10s %%s' %% (lib, version, strip_cwd(location)))
if json_output:
print(json.dumps(output))
else:
print('\\n'.join(output))
"""
name = product_name(tox.path)
filter_name = isinstance(filter_name, str) and filter_name or ''
script = script_template % (name, return_output, env, filter_name)
python = tox.bindir(env, 'python')
if not os.path.exists(python):
log.error('Test environment %s is not installed. Please run without -d / --show-dependencies to install it first.', env)
sys.exit(1)
return run([python, '-c', script], return_output=return_output, raises=False)
def setup_product(self):
project_path = os.getcwd()
name = product_name(project_path)
sanitized_name = re.sub('[^A-Za-z]', '_', name)
placeholder_info = '- please update <PLACEHOLDER> with appropriate value'
tox_ini = TOX_INI_TMPL.replace('{name}', name)
tox_ini_file = os.path.join(project_path, TOX_INI_FILE)
self._create_or_update_file(tox_ini_file, tox_ini)
readme_files = glob(os.path.join(project_path, 'README*'))
if readme_files:
readme_file = readme_files[0]
else:
readme_file = os.path.join(project_path, 'README.rst')
with open(readme_file, 'w') as fp:
fp.write(README_TMPL.format(name=name))
click.echo('Created {} {}'.format(self._relative_path(readme_file), placeholder_info))
coveragerc_file = os.path.join(project_path, '.coveragerc')
self._create_or_update_file(coveragerc_file, COVERAGERC_TMPL)
gitignore_file = os.path.join(project_path, '.gitignore')
resp = requests.get('https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore')
gitignore_content = resp.text.replace('htmlcov', 'htmlcov/\ntextcov')
self._create_or_update_file(gitignore_file, gitignore_content)
setup_py_file = os.path.join(project_path, 'setup.py')
if os.path.exists(setup_py_file):
setup_content = open(setup_py_file).read()
if 'wheel' not in setup_content:
setup_content = setup_content.replace("setup_requires=['setuptools-git'],",
"setup_requires=['setuptools-git', 'wheel'],")
self._create_or_update_file(setup_py_file, setup_content)
else:
requirements_file = os.path.join(project_path, 'requirements.txt')
if not os.path.exists(requirements_file):
with open(requirements_file, 'w') as fp:
pass
click.echo('Created ' + self._relative_path(requirements_file))
readme_name = os.path.basename(readme_file)
requirements_name = os.path.basename(requirements_file)
with open(setup_py_file, 'w') as fp:
fp.write(SETUP_PY_TMPL % (name, readme_name, requirements_name))
click.echo('Created {} {}'.format(self._relative_path(setup_py_file), placeholder_info))
package_dir = os.path.join(project_path, sanitized_name)
if not os.path.exists(package_dir):
os.makedirs(package_dir)
init_file = os.path.join(package_dir, '__init__.py')
open(init_file, 'w').close()
click.echo('Created ' + self._relative_path(init_file))
test_dir = os.path.join(project_path, 'tests')
if not os.path.exists(test_dir) and not os.path.exists(os.path.join(project_path, 'test')):
os.makedirs(test_dir)
test_file = os.path.join(test_dir, 'test_%s.py' % sanitized_name)
with open(test_file, 'w') as fp:
fp.write('def test_{}():\n """ Test is code\'s best friend. ^_^ """'.format(sanitized_name))
click.echo('Created ' + self._relative_path(test_file))
def run(self):
if self.test_dependents:
name = product_name()
# Convert None to list for tuple([])
if not self.env_or_file:
self.env_or_file = []
if not self.extra_args:
self.extra_args = []
test_args = (
('env_or_file', tuple(self.env_or_file)),
('return_output', True),
('num_processes', self.num_processes),
('silent', True),
('debug', self.debug),
('extra_args', tuple(self.extra_args))
)
test_repos = [repo_path()]
test_repos.extend(r for r in repos(workspace_path()) if self.product_depends_on(r, name) and r not in test_repos)
test_args = [(r, test_args, self.__class__) for r in test_repos]
def test_done(result):
name, output = result
success, summary = self.summarize(output)
if success:
click.echo('{}: {}'.format(name, summary))
else:
temp_output_file = os.path.join(tempfile.gettempdir(), 'test-%s.out' % name)
with open(temp_output_file, 'w') as fp:
fp.write(output)
temp_output_file = 'See ' + temp_output_file
log.error('%s: %s', name, '\n\t'.join([summary, temp_output_file]))
def show_remaining(completed, all_args):
completed_repos = set(product_name(r) for r, _, _ in completed)
all_repos = set(product_name(r) for r, _, _ in all_args)
remaining_repos = sorted(list(all_repos - completed_repos))
if len(remaining_repos):
repo = remaining_repos.pop()
more = '& %d more' % len(remaining_repos) if remaining_repos else ''
return '%s %s' % (repo, more)
else:
return 'None'
repo_results = parallel_call(test_repo, test_args, callback=test_done, show_progress=show_remaining, progress_title='Remaining')
for result in list(repo_results.values()):
if isinstance(result, tuple):
_, result = result
success, _ = self.summarize(result)
if not (success or self.return_output):
sys.exit(1)
return dict(list(repo_results.values()))
if not self.repo:
self.repo = project_path()
# Strip out venv bin path to python to avoid issues with it being removed when running tox
if 'VIRTUAL_ENV' in os.environ:
venv_bin = os.environ['VIRTUAL_ENV']
os.environ['PATH'] = os.pathsep.join([p for p in os.environ['PATH'].split(os.pathsep)
if os.path.exists(p) and not p.startswith(venv_bin)])
envs = []
files = []
if self.env_or_file:
for ef in self.env_or_file:
if os.path.exists(ef):
files.append(os.path.abspath(ef))
else:
envs.append(ef)
pytest_args = ''
if self.match_test or self.num_processes is not None or files or self.extra_args:
pytest_args = []
if self.match_test:
pytest_args.append('-k ' + self.match_test)
if self.num_processes is None: # Skip parallel for targeted test run / works better with pdb
self.num_processes = 0
if self.num_processes is not None:
pytest_args.append('-n ' + str(self.num_processes))
if self.extra_args:
pytest_args.extend(self.extra_args)
if files:
pytest_args.extend(files)
pytest_args = ' '.join(pytest_args)
os.environ['PYTESTARGS'] = pytest_args
tox = ToxIni(self.repo, self.tox_ini)
if not envs:
envs = tox.envlist
# Prefer 'test' over 'cover' when there are pytest args as cover is likely to fail and distract from
# test results. And also remove style as user is focused on fixing a test, and style for the whole project
# isn't interesting yet.
if pytest_args:
if 'cover' in envs:
python = tox.get(tox.envsection('cover'), 'basepython')
version = ''.join(python.strip('python').split('.')) if python else '36'
envs[envs.index('cover')] = 'py' + version
if 'style' in envs:
envs.remove('style')
env_commands = {}
if self.install_only and not self.redevelop:
self.redevelop = 1
if self.show_dependencies:
if 'style' in envs:
envs.remove('style')
for env in envs:
self.show_installed_dependencies(tox, env, filter_name=self.show_dependencies)
elif self.install_editable:
if 'style' in envs:
envs.remove('style')
for env in envs:
if len(envs) > 1:
print(env + ':')
self.install_editable_dependencies(tox, env, editable_products=self.install_editable)
elif self.redevelop:
if self.tox_cmd:
cmd = self.tox_cmd
else:
cmd = ['tox', '-c', tox.tox_ini]
if envs:
cmd.extend(['-e', ','.join(envs)])
if self.redevelop > 1:
cmd.append('-r')
if self.install_only:
cmd.append('--notest')
output = run(cmd, cwd=self.repo, raises=not self.return_output, silent=self.silent, return_output=self.return_output)
if not output:
if self.return_output:
return False
else:
sys.exit(1)
for env in envs:
env_commands[env] = ' '.join(cmd)
# Touch envdir
envdir = tox.envdir(env)
if os.path.exists(envdir):
os.utime(envdir, None)
# Strip entry version
self._strip_version_from_entry_scripts(tox, env)
if self.return_output:
return output
else:
for env in envs:
envdir = tox.envdir(env)
def requirements_updated():
req_mtime = 0
requirements_files = ['requirements.txt', 'pinned.txt', 'tox.ini']
for req_file in requirements_files:
req_path = os.path.join(self.repo, req_file)
if os.path.exists(req_path):
req_mtime = max(req_mtime, os.stat(req_path).st_mtime)
return req_mtime > os.stat(envdir).st_mtime
if not os.path.exists(envdir) or requirements_updated():
env_commands.update(
self.commander.run('test', env_or_file=[env], repo=self.repo, redevelop=True, tox_cmd=self.tox_cmd,
tox_ini=self.tox_ini, tox_commands=self.tox_commands, match_test=self.match_test,
num_processes=self.num_processes, silent=self.silent,
debug=self.debug, extra_args=self.extra_args))
continue
commands = self.tox_commands.get(env) or tox.commands(env)
env_commands[env] = '\n'.join(commands)
for command in commands:
full_command = os.path.join(envdir, 'bin', command)
command_path = full_command.split()[0]
if os.path.exists(command_path):
if 'pytest' in full_command or 'py.test' in full_command:
if 'PYTESTARGS' in full_command:
full_command = full_command.replace('{env:PYTESTARGS:}', pytest_args)
else:
full_command += ' ' + pytest_args
activate = '. ' + os.path.join(envdir, 'bin', 'activate')
output = run(activate + '; ' + full_command, shell=True, cwd=self.repo, raises=False, silent=self.silent,
return_output=self.return_output)
if not output:
if self.return_output:
return False
else:
sys.exit(1)
if not self.silent and (len(envs) > 1 or env == 'style'):
click.secho(f'{env}: OK', fg='green')
if self.return_output:
return output
else:
log.error('%s does not exist', command_path)
if self.return_output:
return False
else:
sys.exit(1)
return env_commands
