def complete_remote_shell(prefix, parsed_args, **_): """ :type prefix: unicode :type parsed_args: any :rtype: list[str] """ del parsed_args images = read_lines_without_comments('test/runner/completion/remote.txt', remove_blank_lines=True) # 2008 doesn't support SSH so we do not add to the list of valid images images.extend(["windows/%s" % i for i in read_lines_without_comments('test/runner/completion/windows.txt', remove_blank_lines=True) if i != '2008']) return [i for i in images if i.startswith(prefix)]
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ if args.python_version in UNSUPPORTED_PYTHON_VERSIONS: display.warning('Skipping rstcheck on unsupported Python version %s.' % args.python_version) return SanitySkipped(self.name) ignore_file = os.path.join(ANSIBLE_ROOT, 'test/sanity/rstcheck/ignore-substitutions.txt') ignore_substitutions = sorted(set(read_lines_without_comments(ignore_file, remove_blank_lines=True))) settings = self.load_settings(args, None) paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] in ('.rst',)) paths = settings.filter_skipped_paths(paths) if not paths: return SanitySkipped(self.name) cmd = [ args.python_executable, '-m', 'rstcheck', '--report', 'warning', '--ignore-substitutions', ','.join(ignore_substitutions), ] + paths try: stdout, stderr = run_command(args, cmd, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stdout: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) if args.explain: return SanitySuccess(self.name) pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+): \((?P<level>INFO|WARNING|ERROR|SEVERE)/[0-4]\) (?P<message>.*)$' results = parse_to_list_of_dict(pattern, stderr) results = [SanityMessage( message=r['message'], path=r['path'], line=int(r['line']), column=0, level=r['level'], ) for r in results] settings.process_errors(results, paths) if results: return SanityFailure(self.name, messages=results) return SanitySuccess(self.name)
def complete_network_platform(prefix, parsed_args, **_): """ :type prefix: unicode :type parsed_args: any :rtype: list[str] """ images = read_lines_without_comments('test/runner/completion/network.txt', remove_blank_lines=True) return [i for i in images if i.startswith(prefix) and (not parsed_args.platform or i not in parsed_args.platform)]
def collect_code_smell_tests(): """ :rtype: tuple[SanityFunc] """ skip_file = os.path.join(ANSIBLE_ROOT, 'test/sanity/code-smell/skip.txt') ansible_only_file = os.path.join(ANSIBLE_ROOT, 'test/sanity/code-smell/ansible-only.txt') skip_tests = read_lines_without_comments(skip_file, remove_blank_lines=True, optional=True) if not data_context().content.is_ansible: skip_tests += read_lines_without_comments(ansible_only_file, remove_blank_lines=True) paths = glob.glob(os.path.join(ANSIBLE_ROOT, 'test/sanity/code-smell/*.py')) paths = sorted(p for p in paths if os.access(p, os.X_OK) and os.path.isfile(p) and os.path.basename(p) not in skip_tests) tests = tuple(SanityCodeSmellTest(p) for p in paths) return tests
def complete_remote(prefix, parsed_args, **_): """ :type prefix: unicode :type parsed_args: any :rtype: list[str] """ del parsed_args images = read_lines_without_comments('test/runner/completion/remote.txt', remove_blank_lines=True) return [i for i in images if i.startswith(prefix)]
def test(self, args, targets, python_version): """ :type args: SanityConfig :type targets: SanityTargets :type python_version: str :rtype: TestResult """ ignore_file = os.path.join(SANITY_ROOT, 'rstcheck', 'ignore-substitutions.txt') ignore_substitutions = sorted(set(read_lines_without_comments(ignore_file, remove_blank_lines=True))) settings = self.load_processor(args) paths = [target.path for target in targets.include] cmd = [ find_python(python_version), '-m', 'rstcheck', '--report', 'warning', '--ignore-substitutions', ','.join(ignore_substitutions), ] + paths try: stdout, stderr = run_command(args, cmd, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stdout: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) if args.explain: return SanitySuccess(self.name) pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+): \((?P<level>INFO|WARNING|ERROR|SEVERE)/[0-4]\) (?P<message>.*)$' results = parse_to_list_of_dict(pattern, stderr) results = [SanityMessage( message=r['message'], path=r['path'], line=int(r['line']), column=0, level=r['level'], ) for r in results] settings.process_errors(results, paths) if results: return SanityFailure(self.name, messages=results) return SanitySuccess(self.name)
def collect_code_smell_tests(): """ :rtype: tuple[SanityCodeSmellTest] """ skip_file = 'test/sanity/code-smell/skip.txt' skip_tests = read_lines_without_comments(skip_file, remove_blank_lines=True) paths = glob.glob('test/sanity/code-smell/*') paths = sorted(p for p in paths if os.access(p, os.X_OK) and os.path.isfile(p) and os.path.basename(p) not in skip_tests) tests = tuple(SanityCodeSmellTest(p) for p in paths) return tests
def complete_network_platform(prefix, parsed_args, **_): """ :type prefix: unicode :type parsed_args: any :rtype: list[str] """ images = read_lines_without_comments(os.path.join(ANSIBLE_TEST_DATA_ROOT, 'completion', 'network.txt'), remove_blank_lines=True) return [ i for i in images if i.startswith(prefix) and ( not parsed_args.platform or i not in parsed_args.platform) ]
def complete_remote_shell(prefix, parsed_args, **_): """ :type prefix: unicode :type parsed_args: any :rtype: list[str] """ del parsed_args images = sorted(get_remote_completion().keys()) # 2008 doesn't support SSH so we do not add to the list of valid images windows_completion_path = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'completion', 'windows.txt') images.extend([ "windows/%s" % i for i in read_lines_without_comments(windows_completion_path, remove_blank_lines=True) if i != '2008' ]) return [i for i in images if i.startswith(prefix)]
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ if args.python_version in UNSUPPORTED_PYTHON_VERSIONS: display.warning( 'Skipping validate-modules on unsupported Python version %s.' % args.python_version) return SanitySkipped(self.name) skip_paths = read_lines_without_comments(VALIDATE_SKIP_PATH, optional=True) skip_paths_set = set(skip_paths) env = ansible_environment(args, color=False) paths = sorted([ i.path for i in targets.include if i.module and i.path not in skip_paths_set ]) if not paths: return SanitySkipped(self.name) cmd = [ args.python_executable, os.path.join(INSTALL_ROOT, 'test/sanity/validate-modules/validate-modules'), '--format', 'json', '--arg-spec', ] + paths invalid_ignores = [] ignore_entries = read_lines_without_comments(VALIDATE_IGNORE_PATH, optional=True) ignore = collections.defaultdict( dict) # type: t.Dict[str, t.Dict[str, int]] line = 0 for ignore_entry in ignore_entries: line += 1 if not ignore_entry: continue if ' ' not in ignore_entry: invalid_ignores.append((line, 'Invalid syntax')) continue path, code = ignore_entry.split(' ', 1) ignore[path][code] = line if args.base_branch: cmd.extend([ '--base-branch', args.base_branch, ]) else: display.warning( 'Cannot perform module comparison against the base branch. Base branch not detected when running locally.' ) try: stdout, stderr = run_command(args, cmd, env=env, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr or status not in (0, 3): raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) if args.explain: return SanitySuccess(self.name) messages = json.loads(stdout) errors = [] for filename in messages: output = messages[filename] for item in output['errors']: errors.append( SanityMessage( path=filename, line=int(item['line']) if 'line' in item else 0, column=int(item['column']) if 'column' in item else 0, level='error', code='E%s' % item['code'], message=item['msg'], )) filtered = [] for error in errors: if error.code in ignore[error.path]: ignore[error.path][ error. code] = 0 # error ignored, clear line number of ignore entry to track usage else: filtered.append(error) # error not ignored errors = filtered for invalid_ignore in invalid_ignores: errors.append( SanityMessage( code='A201', message=invalid_ignore[1], path=VALIDATE_IGNORE_PATH, line=invalid_ignore[0], column=1, confidence=calculate_confidence(VALIDATE_IGNORE_PATH, line, args.metadata) if args.metadata.changes else None, )) line = 0 for path in skip_paths: line += 1 if not path: continue if not os.path.exists(path): # Keep files out of the list which no longer exist in the repo. errors.append( SanityMessage( code='A101', message='Remove "%s" since it does not exist' % path, path=VALIDATE_SKIP_PATH, line=line, column=1, confidence=calculate_best_confidence( ((VALIDATE_SKIP_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) for path in sorted(ignore.keys()): if os.path.exists(path): continue for line in sorted(ignore[path].values()): # Keep files out of the list which no longer exist in the repo. errors.append( SanityMessage( code='A101', message='Remove "%s" since it does not exist' % path, path=VALIDATE_IGNORE_PATH, line=line, column=1, confidence=calculate_best_confidence( ((VALIDATE_IGNORE_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) for path in paths: if path not in ignore: continue for code in ignore[path]: line = ignore[path][code] if not line: continue errors.append( SanityMessage( code='A102', message='Remove since "%s" passes "%s" test' % (path, code), path=VALIDATE_IGNORE_PATH, line=line, column=1, confidence=calculate_best_confidence( ((VALIDATE_IGNORE_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) if errors: return SanityFailure(self.name, messages=errors) return SanitySuccess(self.name)
def __init__(self, path, modules, prefixes): """ :type path: str :type modules: frozenset[str] :type prefixes: dict[str, str] """ super(IntegrationTarget, self).__init__() self.name = os.path.basename(path) self.path = path # script_path and type contents = sorted(os.listdir(path)) runme_files = tuple(c for c in contents if os.path.splitext(c)[0] == 'runme') test_files = tuple(c for c in contents if os.path.splitext(c)[0] == 'test') self.script_path = None if runme_files: self.type = 'script' self.script_path = os.path.join(path, runme_files[0]) elif test_files: self.type = 'special' elif os.path.isdir(os.path.join(path, 'tasks')) or os.path.isdir(os.path.join(path, 'defaults')): self.type = 'role' else: self.type = 'unknown' # static_aliases try: aliases_path = os.path.join(path, 'aliases') static_aliases = tuple(read_lines_without_comments(aliases_path, remove_blank_lines=True)) except IOError as ex: if ex.errno != errno.ENOENT: raise static_aliases = tuple() # modules if self.name in modules: module_name = self.name elif self.name.startswith('win_') and self.name[4:] in modules: module_name = self.name[4:] else: module_name = None self.modules = tuple(sorted(a for a in static_aliases + tuple([module_name]) if a in modules)) # groups groups = [self.type] groups += [a for a in static_aliases if a not in modules] groups += ['module/%s' % m for m in self.modules] if not self.modules: groups.append('non_module') if 'destructive' not in groups: groups.append('non_destructive') if '_' in self.name: prefix = self.name[:self.name.find('_')] else: prefix = None if prefix in prefixes: group = prefixes[prefix] if group != prefix: group = '%s/%s' % (group, prefix) groups.append(group) if self.name.startswith('win_'): groups.append('windows') if self.name.startswith('connection_'): groups.append('connection') if self.name.startswith('setup_') or self.name.startswith('prepare_'): groups.append('hidden') if self.type not in ('script', 'role'): groups.append('hidden') for group in itertools.islice(groups, 0, len(groups)): if '/' in group: parts = group.split('/') for i in range(1, len(parts)): groups.append('/'.join(parts[:i])) if not any(g in self.non_posix for g in groups): groups.append('posix') # aliases aliases = [self.name] + \ ['%s/' % g for g in groups] + \ ['%s/%s' % (g, self.name) for g in groups if g not in self.categories] if 'hidden/' in aliases: aliases = ['hidden/'] + ['hidden/%s' % a for a in aliases if not a.startswith('hidden/')] self.aliases = tuple(sorted(set(aliases))) # configuration self.setup_once = tuple(sorted(set(g.split('/')[2] for g in groups if g.startswith('setup/once/')))) self.setup_always = tuple(sorted(set(g.split('/')[2] for g in groups if g.startswith('setup/always/'))))
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ skip_paths = read_lines_without_comments(PSLINT_SKIP_PATH) invalid_ignores = [] ignore_entries = read_lines_without_comments(PSLINT_IGNORE_PATH) ignore = collections.defaultdict(dict) line = 0 for ignore_entry in ignore_entries: line += 1 if not ignore_entry: continue if ' ' not in ignore_entry: invalid_ignores.append((line, 'Invalid syntax')) continue path, code = ignore_entry.split(' ', 1) if not os.path.exists(path): invalid_ignores.append( (line, 'Remove "%s" since it does not exist' % path)) continue ignore[path][code] = line paths = sorted( i.path for i in targets.include if os.path.splitext(i.path)[1] in ( '.ps1', '.psm1', '.psd1') and i.path not in skip_paths) if not paths: return SanitySkipped(self.name) if not find_executable('pwsh', required='warning'): return SanitySkipped(self.name) # Make sure requirements are installed before running sanity checks cmds = [['test/runner/requirements/sanity.ps1'], ['test/sanity/pslint/pslint.ps1'] + paths] for cmd in cmds: try: stdout, stderr = run_command(args, cmd, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) if args.explain: return SanitySuccess(self.name) severity = [ 'Information', 'Warning', 'Error', 'ParseError', ] cwd = os.getcwd() + '/' # replace unicode smart quotes and ellipsis with ascii versions stdout = re.sub(u'[\u2018\u2019]', "'", stdout) stdout = re.sub(u'[\u201c\u201d]', '"', stdout) stdout = re.sub(u'[\u2026]', '...', stdout) messages = json.loads(stdout) errors = [ SanityMessage( code=m['RuleName'], message=m['Message'], path=m['ScriptPath'].replace(cwd, ''), line=m['Line'] or 0, column=m['Column'] or 0, level=severity[m['Severity']], ) for m in messages ] line = 0 filtered = [] for error in errors: if error.code in ignore[error.path]: ignore[error.path][ error. code] = None # error ignored, clear line number of ignore entry to track usage else: filtered.append(error) # error not ignored errors = filtered for invalid_ignore in invalid_ignores: errors.append( SanityMessage( code='A201', message=invalid_ignore[1], path=PSLINT_IGNORE_PATH, line=invalid_ignore[0], column=1, confidence=calculate_confidence(PSLINT_IGNORE_PATH, line, args.metadata) if args.metadata.changes else None, )) for path in skip_paths: line += 1 if not path: continue if not os.path.exists(path): # Keep files out of the list which no longer exist in the repo. errors.append( SanityMessage( code='A101', message='Remove "%s" since it does not exist' % path, path=PSLINT_SKIP_PATH, line=line, column=1, confidence=calculate_best_confidence( ((PSLINT_SKIP_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) for path in paths: if path not in ignore: continue for code in ignore[path]: line = ignore[path][code] if not line: continue errors.append( SanityMessage( code='A102', message='Remove since "%s" passes "%s" test' % (path, code), path=PSLINT_IGNORE_PATH, line=line, column=1, confidence=calculate_best_confidence( ((PSLINT_IGNORE_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) if errors: return SanityFailure(self.name, messages=errors) return SanitySuccess(self.name)
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ skip_paths = read_lines_without_comments(PEP8_SKIP_PATH) legacy_paths = read_lines_without_comments(PEP8_LEGACY_PATH) legacy_ignore_file = 'test/sanity/pep8/legacy-ignore.txt' legacy_ignore = set(read_lines_without_comments(legacy_ignore_file, remove_blank_lines=True)) current_ignore_file = 'test/sanity/pep8/current-ignore.txt' current_ignore = sorted(read_lines_without_comments(current_ignore_file, remove_blank_lines=True)) skip_paths_set = set(skip_paths) legacy_paths_set = set(legacy_paths) paths = sorted(i.path for i in targets.include if (os.path.splitext(i.path)[1] == '.py' or i.path.startswith('bin/')) and i.path not in skip_paths_set) cmd = [ args.python_executable, '-m', 'pycodestyle', '--max-line-length', '160', '--config', '/dev/null', '--ignore', ','.join(sorted(current_ignore)), ] + paths if paths: try: stdout, stderr = run_command(args, cmd, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) else: stdout = None if args.explain: return SanitySuccess(self.name) if stdout: pattern = '^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<code>[WE][0-9]{3}) (?P<message>.*)$' results = [re.search(pattern, line).groupdict() for line in stdout.splitlines()] else: results = [] results = [SanityMessage( message=r['message'], path=r['path'], line=int(r['line']), column=int(r['column']), level='warning' if r['code'].startswith('W') else 'error', code=r['code'], ) for r in results] failed_result_paths = set([result.path for result in results]) used_paths = set(paths) errors = [] summary = {} line = 0 for path in legacy_paths: line += 1 if not path: continue if not os.path.exists(path): # Keep files out of the list which no longer exist in the repo. errors.append(SanityMessage( code='A101', message='Remove "%s" since it does not exist' % path, path=PEP8_LEGACY_PATH, line=line, column=1, confidence=calculate_best_confidence(((PEP8_LEGACY_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) if path in used_paths and path not in failed_result_paths: # Keep files out of the list which no longer require the relaxed rule set. errors.append(SanityMessage( code='A201', message='Remove "%s" since it passes the current rule set' % path, path=PEP8_LEGACY_PATH, line=line, column=1, confidence=calculate_best_confidence(((PEP8_LEGACY_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) line = 0 for path in skip_paths: line += 1 if not path: continue if not os.path.exists(path): # Keep files out of the list which no longer exist in the repo. errors.append(SanityMessage( code='A101', message='Remove "%s" since it does not exist' % path, path=PEP8_SKIP_PATH, line=line, column=1, confidence=calculate_best_confidence(((PEP8_SKIP_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) for result in results: if result.path in legacy_paths_set and result.code in legacy_ignore: # Files on the legacy list are permitted to have errors on the legacy ignore list. # However, we want to report on their existence to track progress towards eliminating these exceptions. display.info('PEP 8: %s (legacy)' % result, verbosity=3) key = '%s %s' % (result.code, re.sub('[0-9]+', 'NNN', result.message)) if key not in summary: summary[key] = 0 summary[key] += 1 else: # Files not on the legacy list and errors not on the legacy ignore list are PEP 8 policy errors. errors.append(result) if summary: lines = [] count = 0 for key in sorted(summary): count += summary[key] lines.append('PEP 8: %5d %s' % (summary[key], key)) display.info('PEP 8: There were %d different legacy issues found (%d total):' % (len(summary), count), verbosity=1) display.info('PEP 8: Count Code Message', verbosity=1) for line in lines: display.info(line, verbosity=1) if errors: return SanityFailure(self.name, messages=errors) return SanitySuccess(self.name)
def test(self, args, targets, python_version): """ :type args: SanityConfig :type targets: SanityTargets :type python_version: str :rtype: TestResult """ #skip_file = 'test/sanity/import/skip.txt' skip_file = os.path.join( os.path.dirname(ansible_test.__file__), 'lib/sanity/import/skip.txt' ) skip_paths = read_lines_without_comments(skip_file, remove_blank_lines=True) skip_paths_set = set(skip_paths) paths = sorted( i.path for i in targets.include if os.path.splitext(i.path)[1] == '.py' and (i.path.startswith('lib/ansible/modules/') or i.path.startswith('lib/ansible/module_utils/')) and i.path not in skip_paths_set ) if not paths: return SanitySkipped(self.name, python_version=python_version) env = ansible_environment(args, color=False) # create a clean virtual environment to minimize the available imports beyond the python standard library virtual_environment_path = os.path.abspath('test/runner/.tox/minimal-py%s' % python_version.replace('.', '')) virtual_environment_bin = os.path.join(virtual_environment_path, 'bin') remove_tree(virtual_environment_path) python = find_python(python_version) cmd = [python, '-m', 'virtualenv', virtual_environment_path, '--python', python, '--no-setuptools', '--no-wheel'] if not args.coverage: cmd.append('--no-pip') run_command(args, cmd, capture=True) # add the importer to our virtual environment so it can be accessed through the coverage injector importer_path = os.path.join(virtual_environment_bin, 'importer.py') if not args.explain: os.symlink(os.path.abspath('test/sanity/import/importer.py'), importer_path) # create a minimal python library python_path = os.path.abspath('test/runner/.tox/import/lib') ansible_path = os.path.join(python_path, 'ansible') ansible_init = os.path.join(ansible_path, '__init__.py') ansible_link = os.path.join(ansible_path, 'module_utils') if not args.explain: make_dirs(ansible_path) with open(ansible_init, 'w'): pass if not os.path.exists(ansible_link): os.symlink('../../../../../../lib/ansible/module_utils', ansible_link) # activate the virtual environment env['PATH'] = '%s:%s' % (virtual_environment_bin, env['PATH']) env['PYTHONPATH'] = python_path # make sure coverage is available in the virtual environment if needed if args.coverage: run_command(args, generate_pip_install(['pip'], 'sanity.import', packages=['setuptools']), env=env) run_command(args, generate_pip_install(['pip'], 'sanity.import', packages=['coverage']), env=env) run_command(args, ['pip', 'uninstall', '--disable-pip-version-check', '-y', 'setuptools'], env=env) run_command(args, ['pip', 'uninstall', '--disable-pip-version-check', '-y', 'pip'], env=env) cmd = ['importer.py'] data = '\n'.join(paths) display.info(data, verbosity=4) results = [] virtualenv_python = os.path.join(virtual_environment_bin, 'python') try: stdout, stderr = intercept_command(args, cmd, self.name, env, capture=True, data=data, python_version=python_version, virtualenv=virtualenv_python) if stdout or stderr: raise SubprocessError(cmd, stdout=stdout, stderr=stderr) except SubprocessError as ex: if ex.status != 10 or ex.stderr or not ex.stdout: raise pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<message>.*)$' results = parse_to_list_of_dict(pattern, ex.stdout) results = [SanityMessage( message=r['message'], path=r['path'], line=int(r['line']), column=int(r['column']), ) for r in results] results = [result for result in results if result.path not in skip_paths_set] if results: return SanityFailure(self.name, messages=results, python_version=python_version) return SanitySuccess(self.name, python_version=python_version)
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ if args.python_version in UNSUPPORTED_PYTHON_VERSIONS: display.warning( 'Skipping pylint on unsupported Python version %s.' % args.python_version) return SanitySkipped(self.name) skip_paths = read_lines_without_comments(PYLINT_SKIP_PATH) invalid_ignores = [] supported_versions = set(SUPPORTED_PYTHON_VERSIONS) - set( UNSUPPORTED_PYTHON_VERSIONS) supported_versions = set([v.split('.')[0] for v in supported_versions ]) | supported_versions ignore_entries = read_lines_without_comments(PYLINT_IGNORE_PATH) ignore = collections.defaultdict(dict) line = 0 for ignore_entry in ignore_entries: line += 1 if not ignore_entry: continue if ' ' not in ignore_entry: invalid_ignores.append((line, 'Invalid syntax')) continue path, code = ignore_entry.split(' ', 1) if not os.path.exists(path): invalid_ignores.append( (line, 'Remove "%s" since it does not exist' % path)) continue if ' ' in code: code, version = code.split(' ', 1) if version not in supported_versions: invalid_ignores.append( (line, 'Invalid version: %s' % version)) continue if version not in (args.python_version, args.python_version.split('.')[0]): continue # ignore version specific entries for other versions ignore[path][code] = line skip_paths_set = set(skip_paths) paths = sorted( i.path for i in targets.include if (os.path.splitext(i.path)[1] == '.py' or i.path.startswith('bin/')) and i.path not in skip_paths_set) module_paths = [ p.split(os.path.sep) for p in paths if p.startswith('lib/ansible/modules/') ] module_dirs = sorted(set([p[3] for p in module_paths if len(p) > 4])) large_module_group_threshold = 500 large_module_groups = [ key for key, value in itertools.groupby( module_paths, lambda p: p[3] if len(p) > 4 else '') if len(list(value)) > large_module_group_threshold ] large_module_group_paths = [ p.split(os.path.sep) for p in paths if any( p.startswith('lib/ansible/modules/%s/' % g) for g in large_module_groups) ] large_module_group_dirs = sorted( set([ os.path.sep.join(p[3:5]) for p in large_module_group_paths if len(p) > 5 ])) contexts = [] remaining_paths = set(paths) def add_context(available_paths, context_name, context_filter): """ :type available_paths: set[str] :type context_name: str :type context_filter: (str) -> bool """ filtered_paths = set(p for p in available_paths if context_filter(p)) contexts.append((context_name, sorted(filtered_paths))) available_paths -= filtered_paths def filter_path(path_filter=None): """ :type path_filter: str :rtype: (str) -> bool """ def context_filter(path_to_filter): """ :type path_to_filter: str :rtype: bool """ return path_to_filter.startswith(path_filter) return context_filter add_context(remaining_paths, 'ansible-test', filter_path('test/runner/')) add_context(remaining_paths, 'units', filter_path('test/units/')) add_context(remaining_paths, 'test', filter_path('test/')) add_context(remaining_paths, 'hacking', filter_path('hacking/')) for large_module_group_dir in large_module_group_dirs: add_context( remaining_paths, 'modules/%s' % large_module_group_dir, filter_path('lib/ansible/modules/%s/' % large_module_group_dir)) for module_dir in module_dirs: add_context(remaining_paths, 'modules/%s' % module_dir, filter_path('lib/ansible/modules/%s/' % module_dir)) add_context(remaining_paths, 'modules', filter_path('lib/ansible/modules/')) add_context(remaining_paths, 'module_utils', filter_path('lib/ansible/module_utils/')) add_context(remaining_paths, 'ansible', lambda p: True) messages = [] context_times = [] test_start = datetime.datetime.utcnow() for context, context_paths in sorted(contexts): if not context_paths: continue context_start = datetime.datetime.utcnow() messages += self.pylint(args, context, context_paths) context_end = datetime.datetime.utcnow() context_times.append( '%s: %d (%s)' % (context, len(context_paths), context_end - context_start)) test_end = datetime.datetime.utcnow() for context_time in context_times: display.info(context_time, verbosity=4) display.info('total: %d (%s)' % (len(paths), test_end - test_start), verbosity=4) errors = [ SanityMessage( message=m['message'].replace('\n', ' '), path=m['path'], line=int(m['line']), column=int(m['column']), level=m['type'], code=m['symbol'], ) for m in messages ] if args.explain: return SanitySuccess(self.name) line = 0 filtered = [] for error in errors: if error.code in ignore[error.path]: ignore[error.path][ error. code] = None # error ignored, clear line number of ignore entry to track usage else: filtered.append(error) # error not ignored errors = filtered for invalid_ignore in invalid_ignores: errors.append( SanityMessage( code='A201', message=invalid_ignore[1], path=PYLINT_IGNORE_PATH, line=invalid_ignore[0], column=1, confidence=calculate_confidence(PYLINT_IGNORE_PATH, line, args.metadata) if args.metadata.changes else None, )) for path in skip_paths: line += 1 if not path: continue if not os.path.exists(path): # Keep files out of the list which no longer exist in the repo. errors.append( SanityMessage( code='A101', message='Remove "%s" since it does not exist' % path, path=PYLINT_SKIP_PATH, line=line, column=1, confidence=calculate_best_confidence( ((PYLINT_SKIP_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) for path in paths: if path not in ignore: continue for code in ignore[path]: line = ignore[path][code] if not line: continue errors.append( SanityMessage( code='A102', message='Remove since "%s" passes "%s" pylint test' % (path, code), path=PYLINT_IGNORE_PATH, line=line, column=1, confidence=calculate_best_confidence( ((PYLINT_IGNORE_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) if errors: return SanityFailure(self.name, messages=errors) return SanitySuccess(self.name)
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ skip_file = 'test/sanity/shellcheck/skip.txt' skip_paths = set( read_lines_without_comments(skip_file, remove_blank_lines=True)) exclude_file = 'test/sanity/shellcheck/exclude.txt' exclude = set( read_lines_without_comments(exclude_file, remove_blank_lines=True)) paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] == '.sh' and i.path not in skip_paths) if not paths: return SanitySkipped(self.name) cmd = [ 'shellcheck', '-e', ','.join(sorted(exclude)), '--format', 'checkstyle', ] + paths try: stdout, stderr = run_command(args, cmd, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr or status > 1: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) if args.explain: return SanitySuccess(self.name) # json output is missing file paths in older versions of shellcheck, so we'll use xml instead root = fromstring(stdout) # type: Element results = [] for item in root: # type: Element for entry in item: # type: Element results.append( SanityMessage( message=entry.attrib['message'], path=item.attrib['name'], line=int(entry.attrib['line']), column=int(entry.attrib['column']), level=entry.attrib['severity'], code=entry.attrib['source'].replace('ShellCheck.', ''), )) if results: return SanityFailure(self.name, messages=results) return SanitySuccess(self.name)
def __init__(self, args): # type: (SanityConfig) -> None if data_context().content.collection: ansible_version = '%s.%s' % tuple( get_ansible_version(args).split('.')[:2]) ansible_label = 'Ansible %s' % ansible_version file_name = 'ignore-%s.txt' % ansible_version else: ansible_label = 'Ansible' file_name = 'ignore.txt' self.args = args self.relative_path = os.path.join('test/sanity', file_name) self.path = os.path.join(data_context().content.root, self.relative_path) self.ignores = collections.defaultdict(lambda: collections.defaultdict( dict)) # type: t.Dict[str, t.Dict[str, t.Dict[str, int]]] self.skips = collections.defaultdict(lambda: collections.defaultdict( int)) # type: t.Dict[str, t.Dict[str, int]] self.parse_errors = [] # type: t.List[t.Tuple[int, int, str]] self.file_not_found_errors = [] # type: t.List[t.Tuple[int, str]] lines = read_lines_without_comments(self.path, optional=True) targets = SanityTargets.get_targets() paths = set(target.path for target in targets) tests_by_name = {} # type: t.Dict[str, SanityTest] versioned_test_names = set() # type: t.Set[str] unversioned_test_names = {} # type: t.Dict[str, str] directories = paths_to_dirs(list(paths)) paths_by_test = {} # type: t.Dict[str, t.Set[str]] display.info('Read %d sanity test ignore line(s) for %s from: %s' % (len(lines), ansible_label, self.relative_path), verbosity=1) for test in sanity_get_tests(): test_targets = list(targets) if test.include_directories: test_targets += tuple( TestTarget(path, None, None, '') for path in paths_to_dirs( [target.path for target in test_targets])) paths_by_test[test.name] = set( target.path for target in test.filter_targets(test_targets)) if isinstance(test, SanityMultipleVersion): versioned_test_names.add(test.name) tests_by_name.update( dict(('%s-%s' % (test.name, python_version), test) for python_version in test.supported_python_versions)) else: unversioned_test_names.update( dict(('%s-%s' % (test.name, python_version), test.name) for python_version in SUPPORTED_PYTHON_VERSIONS)) tests_by_name[test.name] = test for line_no, line in enumerate(lines, start=1): if not line: self.parse_errors.append( (line_no, 1, "Line cannot be empty or contain only a comment")) continue parts = line.split(' ') path = parts[0] codes = parts[1:] if not path: self.parse_errors.append( (line_no, 1, "Line cannot start with a space")) continue if path.endswith(os.path.sep): if path not in directories: self.file_not_found_errors.append((line_no, path)) continue else: if path not in paths: self.file_not_found_errors.append((line_no, path)) continue if not codes: self.parse_errors.append( (line_no, len(path), "Error code required after path")) continue code = codes[0] if not code: self.parse_errors.append( (line_no, len(path) + 1, "Error code after path cannot be empty")) continue if len(codes) > 1: self.parse_errors.append((line_no, len(path) + len(code) + 2, "Error code cannot contain spaces")) continue parts = code.split('!') code = parts[0] commands = parts[1:] parts = code.split(':') test_name = parts[0] error_codes = parts[1:] test = tests_by_name.get(test_name) if not test: unversioned_name = unversioned_test_names.get(test_name) if unversioned_name: self.parse_errors.append(( line_no, len(path) + len(unversioned_name) + 2, "Sanity test '%s' cannot use a Python version like '%s'" % (unversioned_name, test_name))) elif test_name in versioned_test_names: self.parse_errors.append(( line_no, len(path) + len(test_name) + 1, "Sanity test '%s' requires a Python version like '%s-%s'" % (test_name, test_name, args.python_version))) else: self.parse_errors.append( (line_no, len(path) + 2, "Sanity test '%s' does not exist" % test_name)) continue if path.endswith(os.path.sep) and not test.include_directories: self.parse_errors.append( (line_no, 1, "Sanity test '%s' does not support directory paths" % test_name)) continue if path not in paths_by_test[test.name] and not test.no_targets: self.parse_errors.append( (line_no, 1, "Sanity test '%s' does not test path '%s'" % (test_name, path))) continue if commands and error_codes: self.parse_errors.append( (line_no, len(path) + len(test_name) + 2, "Error code cannot contain both '!' and ':' characters")) continue if commands: command = commands[0] if len(commands) > 1: self.parse_errors.append( (line_no, len(path) + len(test_name) + len(command) + 3, "Error code cannot contain multiple '!' characters")) continue if command == 'skip': if not test.can_skip: self.parse_errors.append( (line_no, len(path) + len(test_name) + 2, "Sanity test '%s' cannot be skipped" % test_name)) continue existing_line_no = self.skips.get(test_name, {}).get(path) if existing_line_no: self.parse_errors.append(( line_no, 1, "Duplicate '%s' skip for path '%s' first found on line %d" % (test_name, path, existing_line_no))) continue self.skips[test_name][path] = line_no continue self.parse_errors.append( (line_no, len(path) + len(test_name) + 2, "Command '!%s' not recognized" % command)) continue if not test.can_ignore: self.parse_errors.append( (line_no, len(path) + 1, "Sanity test '%s' cannot be ignored" % test_name)) continue if test.error_code: if not error_codes: self.parse_errors.append( (line_no, len(path) + len(test_name) + 1, "Sanity test '%s' requires an error code" % test_name)) continue error_code = error_codes[0] if len(error_codes) > 1: self.parse_errors.append( (line_no, len(path) + len(test_name) + len(error_code) + 3, "Error code cannot contain multiple ':' characters")) continue else: if error_codes: self.parse_errors.append( (line_no, len(path) + len(test_name) + 2, "Sanity test '%s' does not support error codes" % test_name)) continue error_code = self.NO_CODE existing = self.ignores.get(test_name, {}).get(path, {}).get(error_code) if existing: if test.error_code: self.parse_errors.append(( line_no, 1, "Duplicate '%s' ignore for error code '%s' for path '%s' first found on line %d" % (test_name, error_code, path, existing))) else: self.parse_errors.append(( line_no, 1, "Duplicate '%s' ignore for path '%s' first found on line %d" % (test_name, path, existing))) continue self.ignores[test_name][path][error_code] = line_no
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ exclude_file = os.path.join(ANSIBLE_ROOT, 'test/sanity/shellcheck/exclude.txt') exclude = set( read_lines_without_comments(exclude_file, remove_blank_lines=True, optional=True)) settings = self.load_processor(args) paths = [target.path for target in targets.include] if not find_executable('shellcheck', required='warning'): return SanitySkipped(self.name) cmd = [ 'shellcheck', '-e', ','.join(sorted(exclude)), '--format', 'checkstyle', ] + paths try: stdout, stderr = run_command(args, cmd, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr or status > 1: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) if args.explain: return SanitySuccess(self.name) # json output is missing file paths in older versions of shellcheck, so we'll use xml instead root = fromstring(stdout) # type: Element results = [] for item in root: # type: Element for entry in item: # type: Element results.append( SanityMessage( message=entry.attrib['message'], path=item.attrib['name'], line=int(entry.attrib['line']), column=int(entry.attrib['column']), level=entry.attrib['severity'], code=entry.attrib['source'].replace('ShellCheck.', ''), )) results = settings.process_errors(results, paths) if results: return SanityFailure(self.name, messages=results) return SanitySuccess(self.name)
def test(self, args, targets, python_version): """ :type args: SanityConfig :type targets: SanityTargets :type python_version: str :rtype: TestResult """ skip_file = 'test/sanity/compile/python%s-skip.txt' % python_version if os.path.exists(skip_file): skip_paths = read_lines_without_comments(skip_file) else: skip_paths = [] paths = sorted( i.path for i in targets.include if (os.path.splitext(i.path)[1] == '.py' or i.path.startswith('bin/')) and i.path not in skip_paths) if not paths: return SanitySkipped(self.name, python_version=python_version) cmd = [ find_python(python_version), os.path.join(ANSIBLE_ROOT, 'test/sanity/compile/compile.py') ] data = '\n'.join(paths) display.info(data, verbosity=4) try: stdout, stderr = run_command(args, cmd, data=data, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) if args.explain: return SanitySuccess(self.name, python_version=python_version) pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<message>.*)$' results = parse_to_list_of_dict(pattern, stdout) results = [ SanityMessage( message=r['message'], path=r['path'].replace('./', ''), line=int(r['line']), column=int(r['column']), ) for r in results ] line = 0 for path in skip_paths: line += 1 if not path: continue if not os.path.exists(path): # Keep files out of the list which no longer exist in the repo. results.append( SanityMessage( code='A101', message='Remove "%s" since it does not exist' % path, path=skip_file, line=line, column=1, confidence=calculate_best_confidence( ((skip_file, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) if results: return SanityFailure(self.name, messages=results, python_version=python_version) return SanitySuccess(self.name, python_version=python_version)
def test(self, args, targets, python_version): """ :type args: SanityConfig :type targets: SanityTargets :type python_version: str :rtype: TestResult """ skip_file = 'test/sanity/ansible-doc/skip.txt' skip_modules = set( read_lines_without_comments(skip_file, remove_blank_lines=True)) plugin_type_blacklist = set([ # not supported by ansible-doc 'action', 'cliconf', 'filter', 'httpapi', 'netconf', 'terminal', 'test', ]) modules = sorted( set(m for i in targets.include_external for m in i.modules) - set(m for i in targets.exclude_external for m in i.modules) - skip_modules) plugins = [ os.path.splitext(i.path)[0].split('/')[-2:] + [i.path] for i in targets.include if os.path.splitext(i.path)[1] == '.py' and os.path.basename(i.path) != '__init__.py' and re.search(r'^lib/ansible/plugins/[^/]+/', i.path) and i.path != 'lib/ansible/plugins/cache/base.py' ] doc_targets = collections.defaultdict(list) target_paths = collections.defaultdict(dict) for module in modules: doc_targets['module'].append(module) for plugin_type, plugin_name, plugin_path in plugins: if plugin_type in plugin_type_blacklist: continue doc_targets[plugin_type].append(plugin_name) target_paths[plugin_type][plugin_name] = plugin_path if not doc_targets: return SanitySkipped(self.name, python_version=python_version) target_paths['module'] = dict( (t.module, t.path) for t in targets.targets if t.module) env = ansible_environment(args, color=False) error_messages = [] for doc_type in sorted(doc_targets): cmd = ['ansible-doc', '-t', doc_type] + sorted( doc_targets[doc_type]) try: stdout, stderr = intercept_command( args, cmd, target_name='ansible-doc', env=env, capture=True, python_version=python_version) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr: errors = stderr.strip().splitlines() messages = [self.parse_error(e, target_paths) for e in errors] if messages and all(messages): error_messages += messages continue if status: summary = u'%s' % SubprocessError( cmd=cmd, status=status, stderr=stderr) return SanityFailure(self.name, summary=summary, python_version=python_version) if stdout: display.info(stdout.strip(), verbosity=3) if stderr: summary = u'Output on stderr from ansible-doc is considered an error.\n\n%s' % SubprocessError( cmd, stderr=stderr) return SanityFailure(self.name, summary=summary, python_version=python_version) if error_messages: return SanityFailure(self.name, messages=error_messages, python_version=python_version) return SanitySuccess(self.name, python_version=python_version)
def __init__(self, path, modules, prefixes): """ :type path: str :type modules: frozenset[str] :type prefixes: dict[str, str] """ super(IntegrationTarget, self).__init__() self.name = os.path.basename(path) self.path = path # script_path and type contents = [os.path.basename(p) for p in data_context().content.get_files(path)] runme_files = tuple(c for c in contents if os.path.splitext(c)[0] == 'runme') test_files = tuple(c for c in contents if os.path.splitext(c)[0] == 'test') self.script_path = None if runme_files: self.type = 'script' self.script_path = os.path.join(path, runme_files[0]) elif test_files: self.type = 'special' elif os.path.isdir(os.path.join(path, 'tasks')) or os.path.isdir(os.path.join(path, 'defaults')): self.type = 'role' else: self.type = 'role' # ansible will consider these empty roles, so ansible-test should as well # static_aliases try: aliases_path = os.path.join(path, 'aliases') static_aliases = tuple(read_lines_without_comments(aliases_path, remove_blank_lines=True)) except IOError as ex: if ex.errno != errno.ENOENT: raise static_aliases = tuple() # modules if self.name in modules: module_name = self.name elif self.name.startswith('win_') and self.name[4:] in modules: module_name = self.name[4:] else: module_name = None self.modules = tuple(sorted(a for a in static_aliases + tuple([module_name]) if a in modules)) # groups groups = [self.type] groups += [a for a in static_aliases if a not in modules] groups += ['module/%s' % m for m in self.modules] if not self.modules: groups.append('non_module') if 'destructive' not in groups: groups.append('non_destructive') if '_' in self.name: prefix = self.name[:self.name.find('_')] else: prefix = None if prefix in prefixes: group = prefixes[prefix] if group != prefix: group = '%s/%s' % (group, prefix) groups.append(group) if self.name.startswith('win_'): groups.append('windows') if self.name.startswith('connection_'): groups.append('connection') if self.name.startswith('setup_') or self.name.startswith('prepare_'): groups.append('hidden') if self.type not in ('script', 'role'): groups.append('hidden') # Collect file paths before group expansion to avoid including the directories. # Ignore references to test targets, as those must be defined using `needs/target/*` or other target references. self.needs_file = tuple(sorted(set('/'.join(g.split('/')[2:]) for g in groups if g.startswith('needs/file/') and not g.startswith('needs/file/test/integration/targets/')))) for group in itertools.islice(groups, 0, len(groups)): if '/' in group: parts = group.split('/') for i in range(1, len(parts)): groups.append('/'.join(parts[:i])) if not any(g in self.non_posix for g in groups): groups.append('posix') # aliases aliases = [self.name] + \ ['%s/' % g for g in groups] + \ ['%s/%s' % (g, self.name) for g in groups if g not in self.categories] if 'hidden/' in aliases: aliases = ['hidden/'] + ['hidden/%s' % a for a in aliases if not a.startswith('hidden/')] self.aliases = tuple(sorted(set(aliases))) # configuration self.setup_once = tuple(sorted(set(g.split('/')[2] for g in groups if g.startswith('setup/once/')))) self.setup_always = tuple(sorted(set(g.split('/')[2] for g in groups if g.startswith('setup/always/')))) self.needs_target = tuple(sorted(set(g.split('/')[2] for g in groups if g.startswith('needs/target/'))))
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ current_ignore_file = os.path.join( ANSIBLE_ROOT, 'test/sanity/pep8/current-ignore.txt') current_ignore = sorted( read_lines_without_comments(current_ignore_file, remove_blank_lines=True)) settings = self.load_settings(args, 'A100') paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] == '.py' or i.path.startswith('bin/')) paths = settings.filter_skipped_paths(paths) cmd = [ args.python_executable, '-m', 'pycodestyle', '--max-line-length', '160', '--config', '/dev/null', '--ignore', ','.join(sorted(current_ignore)), ] + paths if paths: try: stdout, stderr = run_command(args, cmd, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) else: stdout = None if args.explain: return SanitySuccess(self.name) if stdout: pattern = '^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<code>[WE][0-9]{3}) (?P<message>.*)$' results = parse_to_list_of_dict(pattern, stdout) else: results = [] results = [ SanityMessage( message=r['message'], path=r['path'], line=int(r['line']), column=int(r['column']), level='warning' if r['code'].startswith('W') else 'error', code=r['code'], ) for r in results ] errors = settings.process_errors(results, paths) if errors: return SanityFailure(self.name, messages=errors) return SanitySuccess(self.name)
def __init__( self, args, # type: SanityConfig name, # type: str mode, # type: str code, # type: t.Optional[str] python_version, # type: t.Optional[str] ): # type: (...) -> None """ :param mode: must be either "ignore" or "skip" :param code: a code for ansible-test to use for internal errors, using a style that matches codes used by the test, or None if codes are not used """ if mode == 'ignore': self.parse_codes = bool(code) elif mode == 'skip': self.parse_codes = False else: raise Exception('Unsupported mode: %s' % mode) if name == 'compile': filename = 'python%s-%s' % (python_version, mode) else: filename = '%s-%s' % (mode, python_version) if python_version else mode self.args = args self.code = code self.relative_path = 'test/sanity/%s/%s.txt' % (name, filename) self.path = os.path.join(data_context().content.root, self.relative_path) self.entries = collections.defaultdict( dict) # type: t.Dict[str, t.Dict[str, int]] self.parse_errors = [] # type: t.List[t.Tuple[int, int, str]] self.file_not_found_errors = [] # type: t.List[t.Tuple[int, str]] self.used_line_numbers = set() # type: t.Set[int] lines = read_lines_without_comments(self.path, optional=True) paths = set(data_context().content.all_files()) for line_no, line in enumerate(lines, start=1): if not line: continue if line.startswith(' '): self.parse_errors.append( (line_no, 1, 'Line cannot start with a space')) continue if line.endswith(' '): self.parse_errors.append( (line_no, len(line), 'Line cannot end with a space')) continue parts = line.split(' ') path = parts[0] if path not in paths: self.file_not_found_errors.append((line_no, path)) continue if self.parse_codes: if len(parts) < 2: self.parse_errors.append( (line_no, len(line), 'Code required after path')) continue code = parts[1] if not code: self.parse_errors.append( (line_no, len(path) + 1, 'Code after path cannot be empty')) continue if len(parts) > 2: self.parse_errors.append( (line_no, len(path) + len(code) + 2, 'Code cannot contain spaces')) continue existing = self.entries.get(path, {}).get(code) if existing: self.parse_errors.append(( line_no, 1, "Duplicate code '%s' for path '%s' first found on line %d" % (code, path, existing))) continue else: if len(parts) > 1: self.parse_errors.append( (line_no, len(path) + 1, 'Path cannot contain spaces')) continue code = self.NO_CODE existing = self.entries.get(path) if existing: self.parse_errors.append( (line_no, 1, "Duplicate path '%s' first found on line %d" % (path, existing[code]))) continue self.entries[path][code] = line_no