def show_remaining(completed, all_args): completed_repos = set(product_name(r) for r, _, _ in completed) all_repos = set(product_name(r) for r, _, _ in all_args) remaining_repos = sorted(list(all_repos - completed_repos)) if len(remaining_repos): repo = remaining_repos.pop() more = '& %d more' % len(remaining_repos) if remaining_repos else '' return '%s %s' % (repo, more) else: return 'None'
def show_remaining(completed, all_args): completed_repos = set(product_name(r) for r, _, _ in completed) all_repos = set(product_name(r) for r, _, _ in all_args) remaining_repos = sorted(list(all_repos - completed_repos)) if len(remaining_repos): repo = remaining_repos.pop() more = '& %d more' % len( remaining_repos) if remaining_repos else '' return '%s %s' % (repo, more) else: return 'None'
def _update_repo(repo, raises=False, verbose=1): name = product_name(repo) try: if verbose == 1: click.echo('Updating ' + name) branch = current_branch(repo) parent = parent_branch(branch) if parent: checkout_branch(parent, repo) update_repo(repo, quiet=verbose != 2) if parent: if verbose == 2: click.echo('Rebasing ' + branch) checkout_branch(branch, repo_path=repo) update_branch(repo=repo, parent=parent) return True except Exception as e: if raises: raise else: log.error('%s: %s', name, e) return False
def _strip_version_from_entry_scripts(self, tox, env): """ Strip out version spec "==1.2.3" from entry scripts as they require re-develop when version is changed in develop mode. """ name = product_name(tox.path) script_bin = tox.bindir(env) if os.path.exists(script_bin): name_version_re = re.compile('%s==[0-9\.]+' % name) removed_from = [] for script in os.listdir(script_bin): script_path = os.path.join(script_bin, script) if os.path.isfile(script_path): try: with open(script_path) as fp: script = fp.read() except Exception: continue # Binary files if name_version_re.search(script): new_script = name_version_re.sub(name, script) with open(script_path, 'w') as fp: fp.write(new_script) removed_from.append(os.path.basename(script_path)) if removed_from: log.debug('Removed version spec from entry script(s): %s', ', '.join(removed_from))
def run(self): try: scm_repos = repos() in_repo = is_repo(os.getcwd()) optional = len(scm_repos) == 1 pager = ProductPager(optional=optional) for repo in scm_repos: stat_path = os.getcwd() if in_repo else repo output = stat_repo(stat_path, return_output=True, with_color=True) nothing_to_commit = ('nothing to commit' in output and 'Your branch is ahead of' not in output and 'Your branch is behind' not in output) branches = all_branches(repo, verbose=True) child_branches = [b for b in branches if '@' in b] if len(child_branches) >= 1 or len(scm_repos) == 1: show_branches = branches if len(scm_repos) == 1 else child_branches remotes = all_remotes() if len(scm_repos) == 1 else [] remotes = '\n# Remotes: {}'.format(' '.join(remotes)) if len(remotes) > 1 else '' if nothing_to_commit: output = '# Branches: {}{}'.format(' '.join(show_branches), remotes) nothing_to_commit = False elif len(show_branches) > 1: output = '# Branches: {}{}\n#\n{}'.format(' '.join(show_branches), remotes, output) if output and not nothing_to_commit: pager.write(product_name(repo), output) finally: pager.close_and_wait()
def run(self): if self.context: scm_repos = [os.getcwd()] else: scm_repos = repos() optional = len(scm_repos) == 1 pager = ProductPager(optional=optional) for repo in scm_repos: with log_exception(): cur_branch = current_branch(repo) branch = (parent_branch(cur_branch) or 'master') if self.parent else None color = not pager.pager or 'less' in pager.pager.args output = diff_repo(repo, branch=branch, context=self.context, return_output=True, name_only=self.name_only, color=color) if output: pager.write(product_name(repo), output, cur_branch) pager.close_and_wait()
def summarize(cls, tests, include_no_tests=True): """ Summarize the test results :param dict|str tests: Map of product name to test result, or the test result of the current prod. :param bool include_no_tests: Include "No tests" results when there are no tests found. :return: A tuple of (success, list(summaries)) where success is True if all tests pass and summaries is a list of passed/failed summary of each test or just str if 'tests' param is str. """ prod_name = product_name() if isinstance(tests, dict): product_tests = tests else: product_tests = {} product_tests[prod_name] = tests success = True summaries = [] def append_summary(summary, name=None): if len(product_tests) == 1: summaries.append(summary) else: summaries.append("%s: %s" % (name, summary)) for name in sorted(product_tests, key=lambda n: n == prod_name or n): if not product_tests[name]: success = False append_summary('Test failed / No output', name) elif product_tests[name] is True: append_summary('Test successful / No output', name) elif 'collected 0 items' in product_tests[ name] and 'error' not in product_tests[name]: append_summary('No tests') else: match = TEST_RE.search(product_tests[name]) if not match: # Fall back to build if there are no tests. match = BUILD_RE.search(product_tests[name]) if match: append_summary(match.group(0), name) else: append_summary('No test summary found in output', name) summary_lines = [ l for l in product_tests[name].replace('xfailed', '').split('\n') if l.startswith('===') and 'warnings summary' not in l ] if not len(summary_lines) == 2 or 'failed' in summary_lines[ -1] or 'error' in summary_lines[-1]: success = False return success, summaries if isinstance(tests, dict) else summaries[0]
def run(self): repo = repo_path() if repo: click.echo('Removing build/dist folders') silent_run("rm -rf build dist docs/_build */activate", cwd=repo, shell=True) click.echo('Removing *.pyc files') silent_run( "find . -type d \( -path '*/.tox' -o -path '*/mppy-*' \) -prune -o -name *.pyc -exec rm {} \;", cwd=repo, shell=True) if self.force: click.echo('Removing untracked/ignored files') silent_run('git clean -fdx') else: path = workspace_path() click.echo('Cleaning {}'.format(path)) if config.clean.remove_products_older_than_days or config.clean.remove_all_products_except: keep_time = 0 keep_products = [] if config.clean.remove_all_products_except: click.echo('Removing all products except: %s' % config.clean.remove_all_products_except) keep_products = expand_product_groups( config.clean.remove_all_products_except.split()) if config.clean.remove_products_older_than_days: click.echo('Removing products older than %s days' % config.clean.remove_products_older_than_days) keep_time = time( ) - config.clean.remove_products_older_than_days * 86400 removed_products = [] for repo in repos(path): name = product_name(repo) modified_time = os.stat(repo).st_mtime if keep_products and name not in keep_products or keep_time and modified_time < keep_time: status = stat_repo(repo, return_output=True) if (not status or 'nothing to commit' in status and ('working directory clean' in status or 'working tree clean' in status) and len(all_branches(repo)) <= 1): shutil.rmtree(repo) removed_products.append(name) else: click.echo( ' - Skipping "%s" as it has changes that may not be committed' % name) if removed_products: click.echo('Removed ' + ', '.join(removed_products))
def test_repo(repo, test_args, test_class): name = product_name(repo) branch = current_branch(repo) on_branch = '#' + branch if branch != 'master' and branch is not None else '' click.echo('Testing {} {}'.format(name, on_branch)) return name, test_class(repo=repo, **dict(test_args)).run()
def summarize(cls, tests, include_no_tests=True): """ Summarize the test results :param dict|str tests: Map of product name to test result, or the test result of the current prod. :param bool include_no_tests: Include "No tests" results when there are no tests found. :return: A tuple of (success, list(summaries)) where success is True if all tests pass and summaries is a list of passed/failed summary of each test or just str if 'tests' param is str. """ prod_name = product_name() if isinstance(tests, dict): product_tests = tests else: product_tests = {} product_tests[prod_name] = tests success = True summaries = [] def append_summary(summary, name=None): if len(product_tests) == 1: summaries.append(summary) else: summaries.append("%s: %s" % (name, summary)) for name in sorted(product_tests, key=lambda n: n == prod_name or n): if not product_tests[name]: success = False append_summary('Test failed / No output', name) elif product_tests[name] is True: append_summary('Test successful / No output', name) elif 'collected 0 items' in product_tests[name] and 'error' not in product_tests[name]: append_summary('No tests') else: match = TEST_RE.search(product_tests[name]) if not match: # Fall back to build if there are no tests. match = BUILD_RE.search(product_tests[name]) if match: append_summary(match.group(0), name) else: append_summary('No test summary found in output', name) summary_lines = [l for l in product_tests[name].replace('xfailed', '').split('\n') if l.startswith('===') and 'warnings summary' not in l] if not len(summary_lines) == 2 or 'failed' in summary_lines[-1] or 'error' in summary_lines[-1]: success = False return success, summaries if isinstance(tests, dict) else summaries[0]
def install_editable_dependencies(self, tox, env, editable_products): name = product_name(tox.path) editable_products = expand_product_groups(editable_products) dependencies_output = self.show_installed_dependencies(tox, env, return_output=True) if not dependencies_output: log.debug('%s is not installed or there is no dependencies - skipping editable mode changes', name) return try: product_dependencies_list = json.loads(dependencies_output) except Exception as e: log.debug('Failed to get installed dependencies - skipping editable mode changes: %s', e) return product_dependencies = {} for dep, _, path in product_dependencies_list: product_dependencies[dep] = path available_products = [os.path.basename(r) for r in product_repos()] libs = [d for d in editable_products if d in available_products and d in product_dependencies and tox.envdir(env) in product_dependencies[d]] already_editable = [d for d in editable_products if d in product_dependencies and tox.envdir(env) not in product_dependencies[d]] for lib in already_editable: click.echo('{} is already installed in editable mode.'.format(lib)) not_dependent = [d for d in editable_products if d not in product_dependencies] for lib in not_dependent: log.debug('%s is not currently installed (not a dependency) and will be ignored.', lib) not_available = [d for d in editable_products if d not in not_dependent and d not in available_products] for lib in not_available: click.echo('{} is a dependency but not checked out in workspace, and so can not be installed in editable mode.'.format(lib)) pip = tox.bindir(env, 'pip') for lib in libs: if not self.silent or self.debug: click.echo('{}: Installing {} in editable mode'.format(env, lib)) with log_exception('An error occurred when installing %s in editable mode' % lib): run([pip, 'uninstall', lib, '-y'], raises=False, silent=not self.debug) lib_path = product_path(lib) if os.path.exists(os.path.join(lib_path, lib, 'setup.py')): lib_path = os.path.join(lib_path, lib) run([pip, 'install', '--editable', lib_path], silent=not self.debug)
def run(self): if is_repo(): if len(self.target) == 1: if self.target[0] in all_branches(): checkout_branch(self.target[0]) click.echo('Switched to branch ' + self.target[0]) return else: possible_remotes = {upstream_remote()} if '/' in self.target[0]: possible_remotes.add(self.target[0].split('/')[0]) for pull_tags in [False, True]: if pull_tags: for remote in possible_remotes & set( all_remotes()): update_tags(remote) if '/' in self.target[0] and 'remotes/{}'.format( self.target[0]) in all_branches(remotes=True): checkout_branch(self.target[0]) click.echo('Switched to branch ' + self.target[0].split('/')[-1]) return if 'remotes/{}/{}'.format( upstream_remote(), self.target[0]) in all_branches(remotes=True): checkout_branch("{}/{}".format( upstream_remote(), self.target[0])) click.echo('Switched to branch ' + self.target[0]) return checkout_files(self.target) return product_urls = expand_product_groups(self.target) for product_url in product_urls: product_url = product_url.strip('/') product_path = product_checkout_path(product_url) if os.path.exists(product_path): click.echo('Updating ' + product_name(product_path)) else: click.echo('Checking out ' + product_url) checkout_product(product_url, product_path)
def run(self): repo = repo_path() if repo: click.echo('Removing build/dist folders') silent_run("rm -rf build dist docs/_build */activate", cwd=repo, shell=True) click.echo('Removing *.pyc files') silent_run("find . -type d \( -path '*/.tox' -o -path '*/mppy-*' \) -prune -o -name *.pyc -exec rm {} \;", cwd=repo, shell=True) if self.force: click.echo('Removing untracked/ignored files') silent_run('git clean -fdx') else: path = workspace_path() click.echo('Cleaning {}'.format(path)) if config.clean.remove_products_older_than_days or config.clean.remove_all_products_except: keep_time = 0 keep_products = [] if config.clean.remove_all_products_except: click.echo('Removing all products except: %s' % config.clean.remove_all_products_except) keep_products = expand_product_groups(config.clean.remove_all_products_except.split()) if config.clean.remove_products_older_than_days: click.echo('Removing products older than %s days' % config.clean.remove_products_older_than_days) keep_time = time() - config.clean.remove_products_older_than_days * 86400 removed_products = [] for repo in repos(path): name = product_name(repo) modified_time = os.stat(repo).st_mtime if keep_products and name not in keep_products or keep_time and modified_time < keep_time: status = stat_repo(repo, return_output=True) if (not status or 'nothing to commit' in status and ('working directory clean' in status or 'working tree clean' in status) and len(all_branches(repo)) <= 1): shutil.rmtree(repo) removed_products.append(name) else: click.echo(' - Skipping "%s" as it has changes that may not be committed' % name) if removed_products: click.echo('Removed ' + ', '.join(removed_products))
def run(self): if is_repo(): if len(self.target) == 1: if self.target[0] in all_branches(): checkout_branch(self.target[0]) click.echo('Switched to branch ' + self.target[0]) return else: possible_remotes = {upstream_remote()} if '/' in self.target[0]: possible_remotes.add(self.target[0].split('/')[0]) for pull_tags in [False, True]: if pull_tags: for remote in possible_remotes & set(all_remotes()): update_tags(remote) if '/' in self.target[0] and 'remotes/{}'.format(self.target[0]) in all_branches(remotes=True): checkout_branch(self.target[0]) click.echo('Switched to branch ' + self.target[0].split('/')[-1]) return if 'remotes/{}/{}'.format(upstream_remote(), self.target[0]) in all_branches(remotes=True): checkout_branch("{}/{}".format(upstream_remote(), self.target[0])) click.echo('Switched to branch ' + self.target[0]) return checkout_files(self.target) return product_urls = expand_product_groups(self.target) for product_url in product_urls: product_url = product_url.strip('/') product_path = product_checkout_path(product_url) if os.path.exists(product_path): click.echo('Updating ' + product_name(product_path)) else: click.echo('Checking out ' + product_url) checkout_product(product_url, product_path)
def run(self): if self.products: self.products = expand_product_groups(self.products) select_repos = [ repo for repo in repos() if not self.products or self.products and product_name(repo) in self.products ] if not select_repos: click.echo('No product found') elif len(select_repos) == 1: _update_repo(select_repos[0], raises=self.raises, verbose=0 if self.quiet else 2) else: if not all(parallel_call(_update_repo, select_repos).values()): sys.exit(1)
def run(self): try: scm_repos = repos() in_repo = is_repo(os.getcwd()) optional = len(scm_repos) == 1 pager = ProductPager(optional=optional) for repo in scm_repos: stat_path = os.getcwd() if in_repo else repo output = stat_repo(stat_path, return_output=True, with_color=True) nothing_to_commit = ('nothing to commit' in output and 'Your branch is ahead of' not in output and 'Your branch is behind' not in output) branches = all_branches(repo, verbose=True) child_branches = [b for b in branches if '@' in b] if len(child_branches) >= 1 or len(scm_repos) == 1: show_branches = branches if len( scm_repos) == 1 else child_branches remotes = all_remotes() if len(scm_repos) == 1 else [] remotes = '\n# Remotes: {}'.format( ' '.join(remotes)) if len(remotes) > 1 else '' if nothing_to_commit: output = '# Branches: {}{}'.format( ' '.join(show_branches), remotes) nothing_to_commit = False elif len(show_branches) > 1: output = '# Branches: {}{}\n#\n{}'.format( ' '.join(show_branches), remotes, output) if output and not nothing_to_commit: pager.write(product_name(repo), output) finally: pager.close_and_wait()
def run(self): if self.test_dependents: name = product_name() # Convert None to list for tuple([]) if not self.env_or_file: self.env_or_file = [] if not self.extra_args: self.extra_args = [] test_args = (('env_or_file', tuple(self.env_or_file)), ('return_output', True), ('num_processes', self.num_processes), ('silent', True), ('debug', self.debug), ('extra_args', tuple(self.extra_args))) test_repos = [repo_path()] test_repos.extend( r for r in repos(workspace_path()) if self.product_depends_on(r, name) and r not in test_repos) test_args = [(r, test_args, self.__class__) for r in test_repos] def test_done(result): name, output = result success, summary = self.summarize(output) if success: click.echo('{}: {}'.format(name, summary)) else: temp_output_file = os.path.join(tempfile.gettempdir(), 'test-%s.out' % name) with open(temp_output_file, 'w') as fp: fp.write(output) temp_output_file = 'See ' + temp_output_file log.error('%s: %s', name, '\n\t'.join([summary, temp_output_file])) def show_remaining(completed, all_args): completed_repos = set(product_name(r) for r, _, _ in completed) all_repos = set(product_name(r) for r, _, _ in all_args) remaining_repos = sorted(list(all_repos - completed_repos)) if len(remaining_repos): repo = remaining_repos.pop() more = '& %d more' % len( remaining_repos) if remaining_repos else '' return '%s %s' % (repo, more) else: return 'None' repo_results = parallel_call(test_repo, test_args, callback=test_done, show_progress=show_remaining, progress_title='Remaining') for result in list(repo_results.values()): if isinstance(result, tuple): _, result = result success, _ = self.summarize(result) if not (success or self.return_output): sys.exit(1) return dict(list(repo_results.values())) if not self.repo: self.repo = project_path() # Strip out venv bin path to python to avoid issues with it being removed when running tox if 'VIRTUAL_ENV' in os.environ: venv_bin = os.environ['VIRTUAL_ENV'] os.environ['PATH'] = os.pathsep.join([ p for p in os.environ['PATH'].split(os.pathsep) if os.path.exists(p) and not p.startswith(venv_bin) ]) envs = [] files = [] if self.env_or_file: for ef in self.env_or_file: if os.path.exists(ef): files.append(os.path.abspath(ef)) else: envs.append(ef) pytest_args = '' if self.match_test or self.num_processes is not None or files or self.extra_args: pytest_args = [] if self.match_test: pytest_args.append('-k ' + self.match_test) if self.num_processes is None: # Skip parallel for targeted test run / works better with pdb self.num_processes = 0 if self.num_processes is not None: pytest_args.append('-n ' + str(self.num_processes)) if self.extra_args: pytest_args.extend(self.extra_args) if files: pytest_args.extend(files) pytest_args = ' '.join(pytest_args) os.environ['PYTESTARGS'] = pytest_args tox = ToxIni(self.repo, self.tox_ini) if not envs: envs = tox.envlist # Prefer 'test' over 'cover' when there are pytest args as cover is likely to fail and distract from # test results. And also remove style as user is focused on fixing a test, and style for the whole project # isn't interesting yet. if pytest_args: if 'cover' in envs: python = tox.get(tox.envsection('cover'), 'basepython') version = ''.join( python.strip('python').split('.')) if python else '36' envs[envs.index('cover')] = 'py' + version if 'style' in envs: envs.remove('style') env_commands = {} if self.install_only and not self.redevelop: self.redevelop = 1 if self.show_dependencies: if 'style' in envs: envs.remove('style') for env in envs: self.show_installed_dependencies( tox, env, filter_name=self.show_dependencies) elif self.install_editable: if 'style' in envs: envs.remove('style') for env in envs: if len(envs) > 1: print(env + ':') self.install_editable_dependencies( tox, env, editable_products=self.install_editable) elif self.redevelop: if self.tox_cmd: cmd = self.tox_cmd else: cmd = ['tox', '-c', tox.tox_ini] if envs: cmd.extend(['-e', ','.join(envs)]) if self.redevelop > 1: cmd.append('-r') if self.install_only: cmd.append('--notest') output = run(cmd, cwd=self.repo, raises=not self.return_output, silent=self.silent, return_output=self.return_output) if not output: if self.return_output: return False else: sys.exit(1) for env in envs: env_commands[env] = ' '.join(cmd) # Touch envdir envdir = tox.envdir(env) if os.path.exists(envdir): os.utime(envdir, None) # Strip entry version self._strip_version_from_entry_scripts(tox, env) if self.return_output: return output else: for env in envs: envdir = tox.envdir(env) def requirements_updated(): req_mtime = 0 requirements_files = [ 'requirements.txt', 'pinned.txt', 'tox.ini' ] for req_file in requirements_files: req_path = os.path.join(self.repo, req_file) if os.path.exists(req_path): req_mtime = max(req_mtime, os.stat(req_path).st_mtime) return req_mtime > os.stat(envdir).st_mtime if not os.path.exists(envdir) or requirements_updated(): env_commands.update( self.commander.run('test', env_or_file=[env], repo=self.repo, redevelop=True, tox_cmd=self.tox_cmd, tox_ini=self.tox_ini, tox_commands=self.tox_commands, match_test=self.match_test, num_processes=self.num_processes, silent=self.silent, debug=self.debug, extra_args=self.extra_args)) continue commands = self.tox_commands.get(env) or tox.commands(env) env_commands[env] = '\n'.join(commands) for command in commands: full_command = os.path.join(envdir, 'bin', command) command_path = full_command.split()[0] if os.path.exists(command_path): if 'pytest' in full_command or 'py.test' in full_command: if 'PYTESTARGS' in full_command: full_command = full_command.replace( '{env:PYTESTARGS:}', pytest_args) else: full_command += ' ' + pytest_args activate = '. ' + os.path.join(envdir, 'bin', 'activate') output = run(activate + '; ' + full_command, shell=True, cwd=self.repo, raises=False, silent=self.silent, return_output=self.return_output) if not output: if self.return_output: return False else: sys.exit(1) if not self.silent and (len(envs) > 1 or env == 'style'): click.secho(f'{env}: OK', fg='green') if self.return_output: return output else: log.error('%s does not exist', command_path) if self.return_output: return False else: sys.exit(1) return env_commands
def show_installed_dependencies(self, tox, env, return_output=False, filter_name=None): script_template = """ import json import os import sys try: from pip._internal.utils.misc import get_installed_distributions # pip < 10 except Exception: from pip import get_installed_distributions # Required params to run this script package = '%s' json_output = %s env = '%s' filter_name = '%s' cwd = os.getcwd() workspace_dir = os.path.dirname(cwd) try: libs = [(p.key, p.version, p.location) for p in get_installed_distributions()] except Exception as e: print(e) sys.exit(1) output = [] if not json_output: print(env + ':') def strip_cwd(dir): if dir.startswith(cwd + '/'): dir = dir[len(cwd):].lstrip('/') elif dir.startswith(workspace_dir): dir = os.path.join('..', dir[len(workspace_dir):].lstrip('/')) return dir for lib, version, location in sorted(libs): if filter_name and filter_name not in lib: continue if json_output: output.append((lib, version, location)) else: output.append(' %%-25s %%-10s %%s' %% (lib, version, strip_cwd(location))) if json_output: print(json.dumps(output)) else: print('\\n'.join(output)) """ name = product_name(tox.path) filter_name = isinstance(filter_name, str) and filter_name or '' script = script_template % (name, return_output, env, filter_name) python = tox.bindir(env, 'python') if not os.path.exists(python): log.error( 'Test environment %s is not installed. Please run without -d / --show-dependencies to install it first.', env) sys.exit(1) return run([python, '-c', script], return_output=return_output, raises=False)
def install_editable_dependencies(self, tox, env, editable_products): name = product_name(tox.path) editable_products = expand_product_groups(editable_products) dependencies_output = self.show_installed_dependencies( tox, env, return_output=True) if not dependencies_output: log.debug( '%s is not installed or there is no dependencies - skipping editable mode changes', name) return try: product_dependencies_list = json.loads(dependencies_output) except Exception as e: log.debug( 'Failed to get installed dependencies - skipping editable mode changes: %s', e) return product_dependencies = {} for dep, _, path in product_dependencies_list: product_dependencies[dep] = path available_products = [os.path.basename(r) for r in product_repos()] libs = [ d for d in editable_products if d in available_products and d in product_dependencies and tox.envdir(env) in product_dependencies[d] ] already_editable = [ d for d in editable_products if d in product_dependencies and tox.envdir(env) not in product_dependencies[d] ] for lib in already_editable: click.echo('{} is already installed in editable mode.'.format(lib)) not_dependent = [ d for d in editable_products if d not in product_dependencies ] for lib in not_dependent: log.debug( '%s is not currently installed (not a dependency) and will be ignored.', lib) not_available = [ d for d in editable_products if d not in not_dependent and d not in available_products ] for lib in not_available: click.echo( '{} is a dependency but not checked out in workspace, and so can not be installed in editable mode.' .format(lib)) pip = tox.bindir(env, 'pip') for lib in libs: if not self.silent or self.debug: click.echo('{}: Installing {} in editable mode'.format( env, lib)) with log_exception( 'An error occurred when installing %s in editable mode' % lib): run([pip, 'uninstall', lib, '-y'], raises=False, silent=not self.debug) lib_path = product_path(lib) if os.path.exists(os.path.join(lib_path, lib, 'setup.py')): lib_path = os.path.join(lib_path, lib) run([pip, 'install', '--editable', lib_path], silent=not self.debug)
def setup_product(self): project_path = os.getcwd() name = product_name(project_path) sanitized_name = re.sub('[^A-Za-z]', '_', name) placeholder_info = '- please update <PLACEHOLDER> with appropriate value' tox_ini = TOX_INI_TMPL.replace('{name}', name) tox_ini_file = os.path.join(project_path, TOX_INI_FILE) self._create_or_update_file(tox_ini_file, tox_ini) readme_files = glob(os.path.join(project_path, 'README*')) if readme_files: readme_file = readme_files[0] else: readme_file = os.path.join(project_path, 'README.rst') with open(readme_file, 'w') as fp: fp.write(README_TMPL.format(name=name)) click.echo('Created {} {}'.format(self._relative_path(readme_file), placeholder_info)) coveragerc_file = os.path.join(project_path, '.coveragerc') self._create_or_update_file(coveragerc_file, COVERAGERC_TMPL) gitignore_file = os.path.join(project_path, '.gitignore') resp = requests.get( 'https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore' ) gitignore_content = resp.text.replace('htmlcov', 'htmlcov/\ntextcov') self._create_or_update_file(gitignore_file, gitignore_content) setup_py_file = os.path.join(project_path, 'setup.py') if os.path.exists(setup_py_file): setup_content = open(setup_py_file).read() if 'wheel' not in setup_content: setup_content = setup_content.replace( "setup_requires=['setuptools-git'],", "setup_requires=['setuptools-git', 'wheel'],") self._create_or_update_file(setup_py_file, setup_content) else: requirements_file = os.path.join(project_path, 'requirements.txt') if not os.path.exists(requirements_file): with open(requirements_file, 'w') as fp: pass click.echo('Created ' + self._relative_path(requirements_file)) readme_name = os.path.basename(readme_file) requirements_name = os.path.basename(requirements_file) with open(setup_py_file, 'w') as fp: fp.write(SETUP_PY_TMPL % (name, readme_name, requirements_name)) click.echo('Created {} {}'.format( self._relative_path(setup_py_file), placeholder_info)) package_dir = os.path.join(project_path, sanitized_name) if not os.path.exists(package_dir): os.makedirs(package_dir) init_file = os.path.join(package_dir, '__init__.py') open(init_file, 'w').close() click.echo('Created ' + self._relative_path(init_file)) test_dir = os.path.join(project_path, 'tests') if not os.path.exists(test_dir) and not os.path.exists( os.path.join(project_path, 'test')): os.makedirs(test_dir) test_file = os.path.join(test_dir, 'test_%s.py' % sanitized_name) with open(test_file, 'w') as fp: fp.write( 'def test_{}():\n """ Test is code\'s best friend. ^_^ """' .format(sanitized_name)) click.echo('Created ' + self._relative_path(test_file))
def run(self): if self.products: self.products = expand_product_groups(self.products) select_repos = [repo for repo in repos() if not self.products or self.products and product_name(repo) in self.products] if not select_repos: click.echo('No product found') elif len(select_repos) == 1: _update_repo(select_repos[0], raises=self.raises, verbose=0 if self.quiet else 2) else: if not all(parallel_call(_update_repo, select_repos).values()): sys.exit(1)
def show_installed_dependencies(self, tox, env, return_output=False, filter_name=None): script_template = """ import json import os import sys try: from pip._internal.utils.misc import get_installed_distributions # pip < 10 except Exception: from pip import get_installed_distributions # Required params to run this script package = '%s' json_output = %s env = '%s' filter_name = '%s' cwd = os.getcwd() workspace_dir = os.path.dirname(cwd) try: libs = [(p.key, p.version, p.location) for p in get_installed_distributions()] except Exception as e: print(e) sys.exit(1) output = [] if not json_output: print(env + ':') def strip_cwd(dir): if dir.startswith(cwd + '/'): dir = dir[len(cwd):].lstrip('/') elif dir.startswith(workspace_dir): dir = os.path.join('..', dir[len(workspace_dir):].lstrip('/')) return dir for lib, version, location in sorted(libs): if filter_name and filter_name not in lib: continue if json_output: output.append((lib, version, location)) else: output.append(' %%-25s %%-10s %%s' %% (lib, version, strip_cwd(location))) if json_output: print(json.dumps(output)) else: print('\\n'.join(output)) """ name = product_name(tox.path) filter_name = isinstance(filter_name, str) and filter_name or '' script = script_template % (name, return_output, env, filter_name) python = tox.bindir(env, 'python') if not os.path.exists(python): log.error('Test environment %s is not installed. Please run without -d / --show-dependencies to install it first.', env) sys.exit(1) return run([python, '-c', script], return_output=return_output, raises=False)
def setup_product(self): project_path = os.getcwd() name = product_name(project_path) sanitized_name = re.sub('[^A-Za-z]', '_', name) placeholder_info = '- please update <PLACEHOLDER> with appropriate value' tox_ini = TOX_INI_TMPL.replace('{name}', name) tox_ini_file = os.path.join(project_path, TOX_INI_FILE) self._create_or_update_file(tox_ini_file, tox_ini) readme_files = glob(os.path.join(project_path, 'README*')) if readme_files: readme_file = readme_files[0] else: readme_file = os.path.join(project_path, 'README.rst') with open(readme_file, 'w') as fp: fp.write(README_TMPL.format(name=name)) click.echo('Created {} {}'.format(self._relative_path(readme_file), placeholder_info)) coveragerc_file = os.path.join(project_path, '.coveragerc') self._create_or_update_file(coveragerc_file, COVERAGERC_TMPL) gitignore_file = os.path.join(project_path, '.gitignore') resp = requests.get('https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore') gitignore_content = resp.text.replace('htmlcov', 'htmlcov/\ntextcov') self._create_or_update_file(gitignore_file, gitignore_content) setup_py_file = os.path.join(project_path, 'setup.py') if os.path.exists(setup_py_file): setup_content = open(setup_py_file).read() if 'wheel' not in setup_content: setup_content = setup_content.replace("setup_requires=['setuptools-git'],", "setup_requires=['setuptools-git', 'wheel'],") self._create_or_update_file(setup_py_file, setup_content) else: requirements_file = os.path.join(project_path, 'requirements.txt') if not os.path.exists(requirements_file): with open(requirements_file, 'w') as fp: pass click.echo('Created ' + self._relative_path(requirements_file)) readme_name = os.path.basename(readme_file) requirements_name = os.path.basename(requirements_file) with open(setup_py_file, 'w') as fp: fp.write(SETUP_PY_TMPL % (name, readme_name, requirements_name)) click.echo('Created {} {}'.format(self._relative_path(setup_py_file), placeholder_info)) package_dir = os.path.join(project_path, sanitized_name) if not os.path.exists(package_dir): os.makedirs(package_dir) init_file = os.path.join(package_dir, '__init__.py') open(init_file, 'w').close() click.echo('Created ' + self._relative_path(init_file)) test_dir = os.path.join(project_path, 'tests') if not os.path.exists(test_dir) and not os.path.exists(os.path.join(project_path, 'test')): os.makedirs(test_dir) test_file = os.path.join(test_dir, 'test_%s.py' % sanitized_name) with open(test_file, 'w') as fp: fp.write('def test_{}():\n """ Test is code\'s best friend. ^_^ """'.format(sanitized_name)) click.echo('Created ' + self._relative_path(test_file))
def run(self): if self.test_dependents: name = product_name() # Convert None to list for tuple([]) if not self.env_or_file: self.env_or_file = [] if not self.extra_args: self.extra_args = [] test_args = ( ('env_or_file', tuple(self.env_or_file)), ('return_output', True), ('num_processes', self.num_processes), ('silent', True), ('debug', self.debug), ('extra_args', tuple(self.extra_args)) ) test_repos = [repo_path()] test_repos.extend(r for r in repos(workspace_path()) if self.product_depends_on(r, name) and r not in test_repos) test_args = [(r, test_args, self.__class__) for r in test_repos] def test_done(result): name, output = result success, summary = self.summarize(output) if success: click.echo('{}: {}'.format(name, summary)) else: temp_output_file = os.path.join(tempfile.gettempdir(), 'test-%s.out' % name) with open(temp_output_file, 'w') as fp: fp.write(output) temp_output_file = 'See ' + temp_output_file log.error('%s: %s', name, '\n\t'.join([summary, temp_output_file])) def show_remaining(completed, all_args): completed_repos = set(product_name(r) for r, _, _ in completed) all_repos = set(product_name(r) for r, _, _ in all_args) remaining_repos = sorted(list(all_repos - completed_repos)) if len(remaining_repos): repo = remaining_repos.pop() more = '& %d more' % len(remaining_repos) if remaining_repos else '' return '%s %s' % (repo, more) else: return 'None' repo_results = parallel_call(test_repo, test_args, callback=test_done, show_progress=show_remaining, progress_title='Remaining') for result in list(repo_results.values()): if isinstance(result, tuple): _, result = result success, _ = self.summarize(result) if not (success or self.return_output): sys.exit(1) return dict(list(repo_results.values())) if not self.repo: self.repo = project_path() # Strip out venv bin path to python to avoid issues with it being removed when running tox if 'VIRTUAL_ENV' in os.environ: venv_bin = os.environ['VIRTUAL_ENV'] os.environ['PATH'] = os.pathsep.join([p for p in os.environ['PATH'].split(os.pathsep) if os.path.exists(p) and not p.startswith(venv_bin)]) envs = [] files = [] if self.env_or_file: for ef in self.env_or_file: if os.path.exists(ef): files.append(os.path.abspath(ef)) else: envs.append(ef) pytest_args = '' if self.match_test or self.num_processes is not None or files or self.extra_args: pytest_args = [] if self.match_test: pytest_args.append('-k ' + self.match_test) if self.num_processes is None: # Skip parallel for targeted test run / works better with pdb self.num_processes = 0 if self.num_processes is not None: pytest_args.append('-n ' + str(self.num_processes)) if self.extra_args: pytest_args.extend(self.extra_args) if files: pytest_args.extend(files) pytest_args = ' '.join(pytest_args) os.environ['PYTESTARGS'] = pytest_args tox = ToxIni(self.repo, self.tox_ini) if not envs: envs = tox.envlist # Prefer 'test' over 'cover' when there are pytest args as cover is likely to fail and distract from # test results. And also remove style as user is focused on fixing a test, and style for the whole project # isn't interesting yet. if pytest_args: if 'cover' in envs: python = tox.get(tox.envsection('cover'), 'basepython') version = ''.join(python.strip('python').split('.')) if python else '36' envs[envs.index('cover')] = 'py' + version if 'style' in envs: envs.remove('style') env_commands = {} if self.install_only and not self.redevelop: self.redevelop = 1 if self.show_dependencies: if 'style' in envs: envs.remove('style') for env in envs: self.show_installed_dependencies(tox, env, filter_name=self.show_dependencies) elif self.install_editable: if 'style' in envs: envs.remove('style') for env in envs: if len(envs) > 1: print(env + ':') self.install_editable_dependencies(tox, env, editable_products=self.install_editable) elif self.redevelop: if self.tox_cmd: cmd = self.tox_cmd else: cmd = ['tox', '-c', tox.tox_ini] if envs: cmd.extend(['-e', ','.join(envs)]) if self.redevelop > 1: cmd.append('-r') if self.install_only: cmd.append('--notest') output = run(cmd, cwd=self.repo, raises=not self.return_output, silent=self.silent, return_output=self.return_output) if not output: if self.return_output: return False else: sys.exit(1) for env in envs: env_commands[env] = ' '.join(cmd) # Touch envdir envdir = tox.envdir(env) if os.path.exists(envdir): os.utime(envdir, None) # Strip entry version self._strip_version_from_entry_scripts(tox, env) if self.return_output: return output else: for env in envs: envdir = tox.envdir(env) def requirements_updated(): req_mtime = 0 requirements_files = ['requirements.txt', 'pinned.txt', 'tox.ini'] for req_file in requirements_files: req_path = os.path.join(self.repo, req_file) if os.path.exists(req_path): req_mtime = max(req_mtime, os.stat(req_path).st_mtime) return req_mtime > os.stat(envdir).st_mtime if not os.path.exists(envdir) or requirements_updated(): env_commands.update( self.commander.run('test', env_or_file=[env], repo=self.repo, redevelop=True, tox_cmd=self.tox_cmd, tox_ini=self.tox_ini, tox_commands=self.tox_commands, match_test=self.match_test, num_processes=self.num_processes, silent=self.silent, debug=self.debug, extra_args=self.extra_args)) continue commands = self.tox_commands.get(env) or tox.commands(env) env_commands[env] = '\n'.join(commands) for command in commands: full_command = os.path.join(envdir, 'bin', command) command_path = full_command.split()[0] if os.path.exists(command_path): if 'pytest' in full_command or 'py.test' in full_command: if 'PYTESTARGS' in full_command: full_command = full_command.replace('{env:PYTESTARGS:}', pytest_args) else: full_command += ' ' + pytest_args activate = '. ' + os.path.join(envdir, 'bin', 'activate') output = run(activate + '; ' + full_command, shell=True, cwd=self.repo, raises=False, silent=self.silent, return_output=self.return_output) if not output: if self.return_output: return False else: sys.exit(1) if not self.silent and (len(envs) > 1 or env == 'style'): click.secho(f'{env}: OK', fg='green') if self.return_output: return output else: log.error('%s does not exist', command_path) if self.return_output: return False else: sys.exit(1) return env_commands