def disable_pendo(self): """Disable Pendo tracking.""" display.info('Disable Pendo tracking') config = TowerConfig.parse(self.config_path) # tower-cli does not recognize TOWER_ environment variables cmd = [ 'tower-cli', 'setting', 'modify', 'PENDO_TRACKING_STATE', 'off', '-h', config.host, '-u', config.username, '-p', config.password ] attempts = 60 while True: attempts -= 1 try: run_command(self.args, cmd, capture=True) return except SubprocessError as ex: if not attempts: raise ApplicationError( 'Timed out trying to disable Pendo tracking:\n%s' % ex) time.sleep(5)
def command_coverage_report(args): """ :type args: CoverageReportConfig """ output_files = command_coverage_combine(args) for output_file in output_files: if args.group_by or args.stub: display.info( '>>> Coverage Group: %s' % ' '.join(os.path.basename(output_file).split('=')[1:])) options = [] if args.show_missing: options.append('--show-missing') if args.include: options.extend(['--include', args.include]) if args.omit: options.extend(['--omit', args.omit]) env = common_environment() env.update(dict(COVERAGE_FILE=output_file)) run_command(args, env=env, cmd=['coverage', 'report'] + options)
def __init__(self, args): """ :type args: EnvironmentConfig """ cache_dir = 'test/cache' self.key = os.path.join(cache_dir, self.KEY_NAME) self.pub = os.path.join(cache_dir, self.PUB_NAME) if not os.path.isfile(self.key) or not os.path.isfile(self.pub): base_dir = os.path.expanduser('~/.ansible/test/') key = os.path.join(base_dir, self.KEY_NAME) pub = os.path.join(base_dir, self.PUB_NAME) if not args.explain: make_dirs(base_dir) if not os.path.isfile(key) or not os.path.isfile(pub): run_command(args, [ 'ssh-keygen', '-m', 'PEM', '-q', '-t', 'rsa', '-N', '', '-f', key ]) if not args.explain: shutil.copy2(key, self.key) shutil.copy2(pub, self.pub) if args.explain: self.pub_contents = None else: with open(self.pub, 'r') as pub_fd: self.pub_contents = pub_fd.read().strip()
def setup_cli(self): """Install the correct Tower CLI for the version of Tower being tested.""" tower_cli_version = self._get_cloud_config('tower_cli_version') display.info('Installing Tower CLI version: %s' % tower_cli_version) cmd = self.args.pip_command + ['install', '--disable-pip-version-check', 'ansible-tower-cli==%s' % tower_cli_version] run_command(self.args, cmd)
def command_coverage_xml(args): """ :type args: CoverageConfig """ output_files = command_coverage_combine(args) for output_file in output_files: xml_name = 'test/results/reports/%s.xml' % os.path.basename(output_file) env = common_environment() env.update(dict(COVERAGE_FILE=output_file)) run_command(args, env=env, cmd=['coverage', 'xml', '-i', '-o', xml_name])
def scp(self, src, dst): """ :type src: str :type dst: str """ for dummy in range(1, 10): try: run_command(self.core_ci.args, ['scp'] + self.ssh_args + ['-P', '22', '-q', '-r', src, dst]) return except SubprocessError: time.sleep(10) raise ApplicationError('Failed transfer: %s -> %s' % (src, dst))
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ if args.python_version in UNSUPPORTED_PYTHON_VERSIONS: display.warning('Skipping rstcheck on unsupported Python version %s.' % args.python_version) return SanitySkipped(self.name) ignore_file = os.path.join(ANSIBLE_ROOT, 'test/sanity/rstcheck/ignore-substitutions.txt') ignore_substitutions = sorted(set(read_lines_without_comments(ignore_file, remove_blank_lines=True))) settings = self.load_settings(args, None) paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] in ('.rst',)) paths = settings.filter_skipped_paths(paths) if not paths: return SanitySkipped(self.name) cmd = [ args.python_executable, '-m', 'rstcheck', '--report', 'warning', '--ignore-substitutions', ','.join(ignore_substitutions), ] + paths try: stdout, stderr = run_command(args, cmd, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stdout: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) if args.explain: return SanitySuccess(self.name) pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+): \((?P<level>INFO|WARNING|ERROR|SEVERE)/[0-4]\) (?P<message>.*)$' results = parse_to_list_of_dict(pattern, stderr) results = [SanityMessage( message=r['message'], path=r['path'], line=int(r['line']), column=0, level=r['level'], ) for r in results] settings.process_errors(results, paths) if results: return SanityFailure(self.name, messages=results) return SanitySuccess(self.name)
def check_pyyaml(args, version): """ :type args: EnvironmentConfig :type version: str """ if version in CHECK_YAML_VERSIONS: return python = find_python(version) stdout, _dummy = run_command( args, [python, os.path.join(ANSIBLE_ROOT, 'test/runner/yamlcheck.py')], capture=True) if args.explain: return CHECK_YAML_VERSIONS[version] = result = json.loads(stdout) yaml = result['yaml'] cloader = result['cloader'] if not yaml: display.warning('PyYAML is not installed for interpreter: %s' % python) elif not cloader: display.warning( 'PyYAML will be slow due to installation without libyaml support for interpreter: %s' % python)
def pylint(args, context, paths, plugin_dir, plugin_names): # type: (SanityConfig, str, t.List[str], str, t.List[str]) -> t.List[t.Dict[str, str]] """Run pylint using the config specified by the context on the specified paths.""" rcfile = os.path.join(ANSIBLE_ROOT, 'test/sanity/pylint/config/%s' % context.split('/')[0]) if not os.path.exists(rcfile): rcfile = os.path.join(ANSIBLE_ROOT, 'test/sanity/pylint/config/default') parser = ConfigParser() parser.read(rcfile) if parser.has_section('ansible-test'): config = dict(parser.items('ansible-test')) else: config = dict() disable_plugins = set(i.strip() for i in config.get('disable-plugins', '').split(',') if i) load_plugins = set(plugin_names) - disable_plugins cmd = [ args.python_executable, '-m', 'pylint', '--jobs', '0', '--reports', 'n', '--max-line-length', '160', '--rcfile', rcfile, '--output-format', 'json', '--load-plugins', ','.join(load_plugins), ] + paths append_python_path = [plugin_dir] if data_context().content.collection: append_python_path.append(data_context().content.collection.root) env = ansible_environment(args) env['PYTHONPATH'] += os.path.pathsep + os.path.pathsep.join(append_python_path) if paths: display.info('Checking %d file(s) in context "%s" with config: %s' % (len(paths), context, rcfile), verbosity=1) try: stdout, stderr = run_command(args, cmd, env=env, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr or status >= 32: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) else: stdout = None if not args.explain and stdout: messages = json.loads(stdout) else: messages = [] return messages
def test(self, args, targets, python_version): """ :type args: SanityConfig :type targets: SanityTargets :type python_version: str :rtype: TestResult """ settings = self.load_processor(args, python_version) paths = [target.path for target in targets.include] cmd = [ find_python(python_version), os.path.join(SANITY_ROOT, 'compile', 'compile.py') ] data = '\n'.join(paths) display.info(data, verbosity=4) try: stdout, stderr = run_command(args, cmd, data=data, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) if args.explain: return SanitySuccess(self.name, python_version=python_version) pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<message>.*)$' results = parse_to_list_of_dict(pattern, stdout) results = [ SanityMessage( message=r['message'], path=r['path'].replace('./', ''), line=int(r['line']), column=int(r['column']), ) for r in results ] results = settings.process_errors(results, paths) if results: return SanityFailure(self.name, messages=results, python_version=python_version) return SanitySuccess(self.name, python_version=python_version)
def ssh(self, command, options=None): """ :type command: str | list[str] :type options: list[str] | None """ if not options: options = [] if isinstance(command, list): command = ' '.join(cmd_quote(c) for c in command) run_command(self.core_ci.args, ['ssh', '-tt', '-q'] + self.ssh_args + options + [ '-p', str(self.core_ci.connection.port), '%s@%s' % (self.core_ci.connection.username, self.core_ci.connection.hostname) ] + self.become + [cmd_quote(command)])
def __init__(self, args): """ :type args: EnvironmentConfig """ cache_dir = os.path.join(data_context().content.root, 'test/cache') self.key = os.path.join(cache_dir, self.KEY_NAME) self.pub = os.path.join(cache_dir, self.PUB_NAME) key_dst = os.path.relpath(self.key, data_context().content.root) pub_dst = os.path.relpath(self.pub, data_context().content.root) if not os.path.isfile(self.key) or not os.path.isfile(self.pub): base_dir = os.path.expanduser('~/.ansible/test/') key = os.path.join(base_dir, self.KEY_NAME) pub = os.path.join(base_dir, self.PUB_NAME) if not args.explain: make_dirs(base_dir) if not os.path.isfile(key) or not os.path.isfile(pub): run_command(args, [ 'ssh-keygen', '-m', 'PEM', '-q', '-t', 'rsa', '-N', '', '-f', key ]) self.key = key self.pub = pub def ssh_key_callback( files): # type: (t.List[t.Tuple[str, str]]) -> None """Add the SSH keys to the payload file list.""" files.append((key, key_dst)) files.append((pub, pub_dst)) data_context().register_payload_callback(ssh_key_callback) if args.explain: self.pub_contents = None else: with open(self.pub, 'r') as pub_fd: self.pub_contents = pub_fd.read().strip()
def ssh(self, command, options=None, force_pty=True): """ :type command: str | list[str] :type options: list[str] | None :type force_pty: bool """ if not options: options = [] if force_pty: options.append('-tt') if isinstance(command, list): command = ' '.join(cmd_quote(c) for c in command) run_command(self.core_ci.args, ['ssh', '-q'] + self.ssh_args + options + [ '-p', '22', '%s@%s' % (self.core_ci.connection.username, self.core_ci.connection.hostname) ] + [command])
def test(self, args, targets, python_version): """ :type args: SanityConfig :type targets: SanityTargets :type python_version: str :rtype: TestResult """ ignore_file = os.path.join(SANITY_ROOT, 'rstcheck', 'ignore-substitutions.txt') ignore_substitutions = sorted(set(read_lines_without_comments(ignore_file, remove_blank_lines=True))) settings = self.load_processor(args) paths = [target.path for target in targets.include] cmd = [ find_python(python_version), '-m', 'rstcheck', '--report', 'warning', '--ignore-substitutions', ','.join(ignore_substitutions), ] + paths try: stdout, stderr = run_command(args, cmd, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stdout: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) if args.explain: return SanitySuccess(self.name) pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+): \((?P<level>INFO|WARNING|ERROR|SEVERE)/[0-4]\) (?P<message>.*)$' results = parse_to_list_of_dict(pattern, stderr) results = [SanityMessage( message=r['message'], path=r['path'], line=int(r['line']), column=0, level=r['level'], ) for r in results] settings.process_errors(results, paths) if results: return SanityFailure(self.name, messages=results) return SanitySuccess(self.name)
def docker_command(args, cmd, capture=False, stdin=None, stdout=None, always=False): """ :type args: CommonConfig :type cmd: list[str] :type capture: bool :type stdin: file | None :type stdout: file | None :type always: bool :rtype: str | None, str | None """ env = docker_environment() return run_command(args, ['docker'] + cmd, env=env, capture=capture, stdin=stdin, stdout=stdout, always=always)
def test_paths(args, paths, python): """ :type args: SanityConfig :type paths: list[str] :type python: str :rtype: list[SanityMessage] """ cmd = [ python, os.path.join(SANITY_ROOT, 'yamllint', 'yamllinter.py'), ] data = '\n'.join(paths) display.info(data, verbosity=4) try: stdout, stderr = run_command(args, cmd, data=data, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) if args.explain: return [] results = json.loads(stdout)['messages'] results = [ SanityMessage( code=r['code'], message=r['message'], path=r['path'], line=int(r['line']), column=int(r['column']), level=r['level'], ) for r in results ] return results
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ current_ignore_file = os.path.join( ANSIBLE_ROOT, 'test/sanity/pep8/current-ignore.txt') current_ignore = sorted( read_lines_without_comments(current_ignore_file, remove_blank_lines=True)) settings = self.load_settings(args, 'A100') paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] == '.py' or i.path.startswith('bin/')) paths = settings.filter_skipped_paths(paths) cmd = [ args.python_executable, '-m', 'pycodestyle', '--max-line-length', '160', '--config', '/dev/null', '--ignore', ','.join(sorted(current_ignore)), ] + paths if paths: try: stdout, stderr = run_command(args, cmd, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) else: stdout = None if args.explain: return SanitySuccess(self.name) if stdout: pattern = '^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<code>[WE][0-9]{3}) (?P<message>.*)$' results = parse_to_list_of_dict(pattern, stdout) else: results = [] results = [ SanityMessage( message=r['message'], path=r['path'], line=int(r['line']), column=int(r['column']), level='warning' if r['code'].startswith('W') else 'error', code=r['code'], ) for r in results ] errors = settings.process_errors(results, paths) if errors: return SanityFailure(self.name, messages=errors) return SanitySuccess(self.name)
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ if args.python_version in UNSUPPORTED_PYTHON_VERSIONS: display.warning( 'Skipping validate-modules on unsupported Python version %s.' % args.python_version) return SanitySkipped(self.name) skip_paths = read_lines_without_comments(VALIDATE_SKIP_PATH, optional=True) skip_paths_set = set(skip_paths) env = ansible_environment(args, color=False) paths = sorted([ i.path for i in targets.include if i.module and i.path not in skip_paths_set ]) if not paths: return SanitySkipped(self.name) cmd = [ args.python_executable, os.path.join(INSTALL_ROOT, 'test/sanity/validate-modules/validate-modules'), '--format', 'json', '--arg-spec', ] + paths invalid_ignores = [] ignore_entries = read_lines_without_comments(VALIDATE_IGNORE_PATH, optional=True) ignore = collections.defaultdict( dict) # type: t.Dict[str, t.Dict[str, int]] line = 0 for ignore_entry in ignore_entries: line += 1 if not ignore_entry: continue if ' ' not in ignore_entry: invalid_ignores.append((line, 'Invalid syntax')) continue path, code = ignore_entry.split(' ', 1) ignore[path][code] = line if args.base_branch: cmd.extend([ '--base-branch', args.base_branch, ]) else: display.warning( 'Cannot perform module comparison against the base branch. Base branch not detected when running locally.' ) try: stdout, stderr = run_command(args, cmd, env=env, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr or status not in (0, 3): raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) if args.explain: return SanitySuccess(self.name) messages = json.loads(stdout) errors = [] for filename in messages: output = messages[filename] for item in output['errors']: errors.append( SanityMessage( path=filename, line=int(item['line']) if 'line' in item else 0, column=int(item['column']) if 'column' in item else 0, level='error', code='E%s' % item['code'], message=item['msg'], )) filtered = [] for error in errors: if error.code in ignore[error.path]: ignore[error.path][ error. code] = 0 # error ignored, clear line number of ignore entry to track usage else: filtered.append(error) # error not ignored errors = filtered for invalid_ignore in invalid_ignores: errors.append( SanityMessage( code='A201', message=invalid_ignore[1], path=VALIDATE_IGNORE_PATH, line=invalid_ignore[0], column=1, confidence=calculate_confidence(VALIDATE_IGNORE_PATH, line, args.metadata) if args.metadata.changes else None, )) line = 0 for path in skip_paths: line += 1 if not path: continue if not os.path.exists(path): # Keep files out of the list which no longer exist in the repo. errors.append( SanityMessage( code='A101', message='Remove "%s" since it does not exist' % path, path=VALIDATE_SKIP_PATH, line=line, column=1, confidence=calculate_best_confidence( ((VALIDATE_SKIP_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) for path in sorted(ignore.keys()): if os.path.exists(path): continue for line in sorted(ignore[path].values()): # Keep files out of the list which no longer exist in the repo. errors.append( SanityMessage( code='A101', message='Remove "%s" since it does not exist' % path, path=VALIDATE_IGNORE_PATH, line=line, column=1, confidence=calculate_best_confidence( ((VALIDATE_IGNORE_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) for path in paths: if path not in ignore: continue for code in ignore[path]: line = ignore[path][code] if not line: continue errors.append( SanityMessage( code='A102', message='Remove since "%s" passes "%s" test' % (path, code), path=VALIDATE_IGNORE_PATH, line=line, column=1, confidence=calculate_best_confidence( ((VALIDATE_IGNORE_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) if errors: return SanityFailure(self.name, messages=errors) return SanitySuccess(self.name)
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ if args.python_version in self.UNSUPPORTED_PYTHON_VERSIONS: display.warning('Skipping %s on unsupported Python version %s.' % (self.name, args.python_version)) return SanitySkipped(self.name) if self.path.endswith('.py'): cmd = [args.python_executable, self.path] else: cmd = [self.path] env = ansible_environment(args, color=False) pattern = None data = None if self.config: output = self.config.get('output') extensions = self.config.get('extensions') prefixes = self.config.get('prefixes') files = self.config.get('files') always = self.config.get('always') text = self.config.get('text') ignore_changes = self.config.get('ignore_changes') if output == 'path-line-column-message': pattern = '^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<message>.*)$' elif output == 'path-message': pattern = '^(?P<path>[^:]*): (?P<message>.*)$' else: pattern = ApplicationError('Unsupported output type: %s' % output) if ignore_changes: paths = sorted(i.path for i in targets.targets) always = False else: paths = sorted(i.path for i in targets.include) if always: paths = [] # short-term work-around for paths being str instead of unicode on python 2.x if sys.version_info[0] == 2: paths = [p.decode('utf-8') for p in paths] if text is not None: if text: paths = [p for p in paths if not is_binary_file(p)] else: paths = [p for p in paths if is_binary_file(p)] if extensions: paths = [ p for p in paths if os.path.splitext(p)[1] in extensions or ( p.startswith('bin/') and '.py' in extensions) ] if prefixes: paths = [ p for p in paths if any( p.startswith(pre) for pre in prefixes) ] if files: paths = [p for p in paths if os.path.basename(p) in files] if not paths and not always: return SanitySkipped(self.name) data = '\n'.join(paths) if data: display.info(data, verbosity=4) try: stdout, stderr = run_command(args, cmd, data=data, env=env, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stdout and not stderr: if pattern: matches = parse_to_list_of_dict(pattern, stdout) messages = [ SanityMessage( message=m['message'], path=m['path'], line=int(m.get('line', 0)), column=int(m.get('column', 0)), ) for m in matches ] return SanityFailure(self.name, messages=messages) if stderr or status: summary = u'%s' % SubprocessError( cmd=cmd, status=status, stderr=stderr, stdout=stdout) return SanityFailure(self.name, summary=summary) return SanitySuccess(self.name)
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ skip_paths = read_lines_without_comments(PSLINT_SKIP_PATH, optional=True) invalid_ignores = [] ignore_entries = read_lines_without_comments(PSLINT_IGNORE_PATH, optional=True) ignore = collections.defaultdict(dict) # type: t.Dict[str, t.Dict[str, int]] line = 0 for ignore_entry in ignore_entries: line += 1 if not ignore_entry: continue if ' ' not in ignore_entry: invalid_ignores.append((line, 'Invalid syntax')) continue path, code = ignore_entry.split(' ', 1) if not os.path.exists(path): invalid_ignores.append((line, 'Remove "%s" since it does not exist' % path)) continue ignore[path][code] = line paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] in ('.ps1', '.psm1', '.psd1') and i.path not in skip_paths) if not paths: return SanitySkipped(self.name) if not find_executable('pwsh', required='warning'): return SanitySkipped(self.name) # Make sure requirements are installed before running sanity checks cmds = [ [os.path.join(INSTALL_ROOT, 'test/runner/requirements/sanity.ps1')], [os.path.join(INSTALL_ROOT, 'test/sanity/pslint/pslint.ps1')] + paths ] stdout = '' for cmd in cmds: try: stdout, stderr = run_command(args, cmd, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) if args.explain: return SanitySuccess(self.name) severity = [ 'Information', 'Warning', 'Error', 'ParseError', ] cwd = os.getcwd() + '/' # replace unicode smart quotes and ellipsis with ascii versions stdout = re.sub(u'[\u2018\u2019]', "'", stdout) stdout = re.sub(u'[\u201c\u201d]', '"', stdout) stdout = re.sub(u'[\u2026]', '...', stdout) messages = json.loads(stdout) errors = [SanityMessage( code=m['RuleName'], message=m['Message'], path=m['ScriptPath'].replace(cwd, ''), line=m['Line'] or 0, column=m['Column'] or 0, level=severity[m['Severity']], ) for m in messages] line = 0 filtered = [] for error in errors: if error.code in ignore[error.path]: ignore[error.path][error.code] = 0 # error ignored, clear line number of ignore entry to track usage else: filtered.append(error) # error not ignored errors = filtered for invalid_ignore in invalid_ignores: errors.append(SanityMessage( code='A201', message=invalid_ignore[1], path=PSLINT_IGNORE_PATH, line=invalid_ignore[0], column=1, confidence=calculate_confidence(PSLINT_IGNORE_PATH, line, args.metadata) if args.metadata.changes else None, )) for path in skip_paths: line += 1 if not path: continue if not os.path.exists(path): # Keep files out of the list which no longer exist in the repo. errors.append(SanityMessage( code='A101', message='Remove "%s" since it does not exist' % path, path=PSLINT_SKIP_PATH, line=line, column=1, confidence=calculate_best_confidence(((PSLINT_SKIP_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) for path in paths: if path not in ignore: continue for code in ignore[path]: line = ignore[path][code] if not line: continue errors.append(SanityMessage( code='A102', message='Remove since "%s" passes "%s" test' % (path, code), path=PSLINT_IGNORE_PATH, line=line, column=1, confidence=calculate_best_confidence(((PSLINT_IGNORE_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) if errors: return SanityFailure(self.name, messages=errors) return SanitySuccess(self.name)
def test(self, args, targets, python_version): """ :type args: SanityConfig :type targets: SanityTargets :type python_version: str :rtype: TestResult """ if data_context().content.is_ansible: ignore_codes = () else: ignore_codes = (( 'E502', # only ansible content requires __init__.py for module subdirectories )) env = ansible_environment(args, color=False) settings = self.load_processor(args) paths = [target.path for target in targets.include] cmd = [ find_python(python_version), os.path.join(SANITY_ROOT, 'validate-modules', 'validate-modules'), '--format', 'json', '--arg-spec', ] + paths if args.base_branch: cmd.extend([ '--base-branch', args.base_branch, ]) else: display.warning( 'Cannot perform module comparison against the base branch. Base branch not detected when running locally.' ) try: stdout, stderr = run_command(args, cmd, env=env, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr or status not in (0, 3): raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) if args.explain: return SanitySuccess(self.name) messages = json.loads(stdout) errors = [] for filename in messages: output = messages[filename] for item in output['errors']: errors.append( SanityMessage( path=filename, line=int(item['line']) if 'line' in item else 0, column=int(item['column']) if 'column' in item else 0, level='error', code='E%s' % item['code'], message=item['msg'], )) errors = [error for error in errors if error.code not in ignore_codes] errors = settings.process_errors(errors, paths) if errors: return SanityFailure(self.name, messages=errors) return SanitySuccess(self.name)
def test(self, args, targets, python_version): """ :type args: SanityConfig :type targets: SanityTargets :type python_version: str :rtype: TestResult """ settings = self.load_processor(args, python_version) paths = [target.path for target in targets.include] env = ansible_environment(args, color=False) # create a clean virtual environment to minimize the available imports beyond the python standard library virtual_environment_path = os.path.abspath( 'test/runner/.tox/minimal-py%s' % python_version.replace('.', '')) virtual_environment_bin = os.path.join(virtual_environment_path, 'bin') remove_tree(virtual_environment_path) python = find_python(python_version) cmd = [ python, '-m', 'virtualenv', virtual_environment_path, '--python', python, '--no-setuptools', '--no-wheel' ] if not args.coverage: cmd.append('--no-pip') run_command(args, cmd, capture=True) # add the importer to our virtual environment so it can be accessed through the coverage injector importer_path = os.path.join(virtual_environment_bin, 'importer.py') if not args.explain: os.symlink( os.path.abspath( os.path.join(ANSIBLE_ROOT, 'test/sanity/import/importer.py')), importer_path) # create a minimal python library python_path = os.path.abspath('test/runner/.tox/import/lib') ansible_path = os.path.join(python_path, 'ansible') ansible_init = os.path.join(ansible_path, '__init__.py') ansible_link = os.path.join(ansible_path, 'module_utils') if not args.explain: remove_tree(ansible_path) make_dirs(ansible_path) with open(ansible_init, 'w'): pass os.symlink(os.path.join(ANSIBLE_ROOT, 'lib/ansible/module_utils'), ansible_link) if data_context().content.collection: # inject just enough Ansible code for the collections loader to work on all supported Python versions # the __init__.py files are needed only for Python 2.x # the empty modules directory is required for the collection loader to generate the synthetic packages list make_dirs(os.path.join(ansible_path, 'utils')) with open(os.path.join(ansible_path, 'utils/__init__.py'), 'w'): pass os.symlink( os.path.join(ANSIBLE_ROOT, 'lib/ansible/utils/collection_loader.py'), os.path.join(ansible_path, 'utils/collection_loader.py')) os.symlink( os.path.join(ANSIBLE_ROOT, 'lib/ansible/utils/singleton.py'), os.path.join(ansible_path, 'utils/singleton.py')) make_dirs(os.path.join(ansible_path, 'modules')) with open(os.path.join(ansible_path, 'modules/__init__.py'), 'w'): pass # activate the virtual environment env['PATH'] = '%s:%s' % (virtual_environment_bin, env['PATH']) env['PYTHONPATH'] = python_path # make sure coverage is available in the virtual environment if needed if args.coverage: run_command(args, generate_pip_install(['pip'], 'sanity.import', packages=['setuptools']), env=env) run_command(args, generate_pip_install(['pip'], 'sanity.import', packages=['coverage']), env=env) run_command(args, [ 'pip', 'uninstall', '--disable-pip-version-check', '-y', 'setuptools' ], env=env) run_command(args, [ 'pip', 'uninstall', '--disable-pip-version-check', '-y', 'pip' ], env=env) cmd = ['importer.py'] data = '\n'.join(paths) display.info(data, verbosity=4) results = [] virtualenv_python = os.path.join(virtual_environment_bin, 'python') try: with coverage_context(args): stdout, stderr = intercept_command( args, cmd, self.name, env, capture=True, data=data, python_version=python_version, virtualenv=virtualenv_python) if stdout or stderr: raise SubprocessError(cmd, stdout=stdout, stderr=stderr) except SubprocessError as ex: if ex.status != 10 or ex.stderr or not ex.stdout: raise pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<message>.*)$' results = parse_to_list_of_dict(pattern, ex.stdout) results = [ SanityMessage( message=r['message'], path=r['path'], line=int(r['line']), column=int(r['column']), ) for r in results ] results = settings.process_errors(results, paths) if results: return SanityFailure(self.name, messages=results, python_version=python_version) return SanitySuccess(self.name, python_version=python_version)
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ settings = self.load_processor(args) paths = [target.path for target in targets.include] if not find_executable('pwsh', required='warning'): return SanitySkipped(self.name) cmds = [] if args.requirements: cmds.append([ os.path.join(ANSIBLE_ROOT, 'test/runner/requirements/sanity.ps1') ]) cmds.append( [os.path.join(ANSIBLE_ROOT, 'test/sanity/pslint/pslint.ps1')] + paths) stdout = '' for cmd in cmds: try: stdout, stderr = run_command(args, cmd, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) if args.explain: return SanitySuccess(self.name) severity = [ 'Information', 'Warning', 'Error', 'ParseError', ] cwd = data_context().content.root + '/' # replace unicode smart quotes and ellipsis with ascii versions stdout = re.sub(u'[\u2018\u2019]', "'", stdout) stdout = re.sub(u'[\u201c\u201d]', '"', stdout) stdout = re.sub(u'[\u2026]', '...', stdout) messages = json.loads(stdout) errors = [ SanityMessage( code=m['RuleName'], message=m['Message'], path=m['ScriptPath'].replace(cwd, ''), line=m['Line'] or 0, column=m['Column'] or 0, level=severity[m['Severity']], ) for m in messages ] errors = settings.process_errors(errors, paths) if errors: return SanityFailure(self.name, messages=errors) return SanitySuccess(self.name)
def test(self, args, targets, python_version): """ :type args: SanityConfig :type targets: SanityTargets :type python_version: str :rtype: TestResult """ skip_file = 'test/sanity/compile/python%s-skip.txt' % python_version if os.path.exists(skip_file): skip_paths = read_lines_without_comments(skip_file) else: skip_paths = [] paths = sorted( i.path for i in targets.include if (os.path.splitext(i.path)[1] == '.py' or i.path.startswith('bin/')) and i.path not in skip_paths) if not paths: return SanitySkipped(self.name, python_version=python_version) cmd = [ find_python(python_version), os.path.join(ANSIBLE_ROOT, 'test/sanity/compile/compile.py') ] data = '\n'.join(paths) display.info(data, verbosity=4) try: stdout, stderr = run_command(args, cmd, data=data, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) if args.explain: return SanitySuccess(self.name, python_version=python_version) pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<message>.*)$' results = parse_to_list_of_dict(pattern, stdout) results = [ SanityMessage( message=r['message'], path=r['path'].replace('./', ''), line=int(r['line']), column=int(r['column']), ) for r in results ] line = 0 for path in skip_paths: line += 1 if not path: continue if not os.path.exists(path): # Keep files out of the list which no longer exist in the repo. results.append( SanityMessage( code='A101', message='Remove "%s" since it does not exist' % path, path=skip_file, line=line, column=1, confidence=calculate_best_confidence( ((skip_file, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) if results: return SanityFailure(self.name, messages=results, python_version=python_version) return SanitySuccess(self.name, python_version=python_version)
def delegate_docker(args, exclude, require, integration_targets): """ :type args: EnvironmentConfig :type exclude: list[str] :type require: list[str] :type integration_targets: tuple[IntegrationTarget] """ test_image = args.docker privileged = args.docker_privileged if isinstance(args, ShellConfig): use_httptester = args.httptester else: use_httptester = args.httptester and any('needs/httptester/' in target.aliases for target in integration_targets) if use_httptester: docker_pull(args, args.httptester) docker_pull(args, test_image) httptester_id = None test_id = None options = { '--docker': 1, '--docker-privileged': 0, '--docker-util': 1, } python_interpreter = get_python_interpreter(args, get_docker_completion(), args.docker_raw) install_root = '/root/ansible' if data_context().content.collection: content_root = os.path.join(install_root, data_context().content.collection.directory) else: content_root = install_root cmd = generate_command(args, python_interpreter, install_root, content_root, options, exclude, require) if isinstance(args, TestConfig): if args.coverage and not args.coverage_label: image_label = args.docker_raw image_label = re.sub('[^a-zA-Z0-9]+', '-', image_label) cmd += ['--coverage-label', 'docker-%s' % image_label] if isinstance(args, IntegrationConfig): if not args.allow_destructive: cmd.append('--allow-destructive') cmd_options = [] if isinstance(args, ShellConfig) or (isinstance(args, IntegrationConfig) and args.debug_strategy): cmd_options.append('-it') with tempfile.NamedTemporaryFile(prefix='ansible-source-', suffix='.tgz') as local_source_fd: try: create_payload(args, local_source_fd.name) if use_httptester: httptester_id = run_httptester(args) else: httptester_id = None test_options = [ '--detach', '--volume', '/sys/fs/cgroup:/sys/fs/cgroup:ro', '--privileged=%s' % str(privileged).lower(), ] if args.docker_memory: test_options.extend([ '--memory=%d' % args.docker_memory, '--memory-swap=%d' % args.docker_memory, ]) docker_socket = '/var/run/docker.sock' if args.docker_seccomp != 'default': test_options += ['--security-opt', 'seccomp=%s' % args.docker_seccomp] if os.path.exists(docker_socket): test_options += ['--volume', '%s:%s' % (docker_socket, docker_socket)] if httptester_id: test_options += ['--env', 'HTTPTESTER=1'] for host in HTTPTESTER_HOSTS: test_options += ['--link', '%s:%s' % (httptester_id, host)] if isinstance(args, IntegrationConfig): cloud_platforms = get_cloud_providers(args) for cloud_platform in cloud_platforms: test_options += cloud_platform.get_docker_run_options() test_id = docker_run(args, test_image, options=test_options)[0] if args.explain: test_id = 'test_id' else: test_id = test_id.strip() # write temporary files to /root since /tmp isn't ready immediately on container start docker_put(args, test_id, os.path.join(ANSIBLE_ROOT, 'test/runner/setup/docker.sh'), '/root/docker.sh') docker_exec(args, test_id, ['/bin/bash', '/root/docker.sh']) docker_put(args, test_id, local_source_fd.name, '/root/ansible.tgz') docker_exec(args, test_id, ['mkdir', '/root/ansible']) docker_exec(args, test_id, ['tar', 'oxzf', '/root/ansible.tgz', '-C', '/root/ansible']) # docker images are only expected to have a single python version available if isinstance(args, UnitsConfig) and not args.python: cmd += ['--python', 'default'] # run unit tests unprivileged to prevent stray writes to the source tree # also disconnect from the network once requirements have been installed if isinstance(args, UnitsConfig): writable_dirs = [ os.path.join(install_root, '.pytest_cache'), ] if content_root != install_root: writable_dirs.append(os.path.join(content_root, 'test/results/junit')) writable_dirs.append(os.path.join(content_root, 'test/results/coverage')) docker_exec(args, test_id, ['mkdir', '-p'] + writable_dirs) docker_exec(args, test_id, ['chmod', '777'] + writable_dirs) if content_root == install_root: docker_exec(args, test_id, ['find', os.path.join(content_root, 'test/results/'), '-type', 'd', '-exec', 'chmod', '777', '{}', '+']) docker_exec(args, test_id, ['chmod', '755', '/root']) docker_exec(args, test_id, ['chmod', '644', os.path.join(content_root, args.metadata_path)]) docker_exec(args, test_id, ['useradd', 'pytest', '--create-home']) docker_exec(args, test_id, cmd + ['--requirements-mode', 'only'], options=cmd_options) networks = get_docker_networks(args, test_id) for network in networks: docker_network_disconnect(args, test_id, network) cmd += ['--requirements-mode', 'skip'] cmd_options += ['--user', 'pytest'] try: docker_exec(args, test_id, cmd, options=cmd_options) finally: with tempfile.NamedTemporaryFile(prefix='ansible-result-', suffix='.tgz') as local_result_fd: docker_exec(args, test_id, ['tar', 'czf', '/root/results.tgz', '-C', os.path.join(content_root, 'test'), 'results']) docker_get(args, test_id, '/root/results.tgz', local_result_fd.name) run_command(args, ['tar', 'oxzf', local_result_fd.name, '-C', 'test']) finally: if httptester_id: docker_rm(args, httptester_id) if test_id: docker_rm(args, test_id)
def delegate_tox(args, exclude, require, integration_targets): """ :type args: EnvironmentConfig :type exclude: list[str] :type require: list[str] :type integration_targets: tuple[IntegrationTarget] """ if args.python: versions = (args.python_version,) if args.python_version not in SUPPORTED_PYTHON_VERSIONS: raise ApplicationError('tox does not support Python version %s' % args.python_version) else: versions = SUPPORTED_PYTHON_VERSIONS if args.httptester: needs_httptester = sorted(target.name for target in integration_targets if 'needs/httptester/' in target.aliases) if needs_httptester: display.warning('Use --docker or --remote to enable httptester for tests marked "needs/httptester": %s' % ', '.join(needs_httptester)) options = { '--tox': args.tox_args, '--tox-sitepackages': 0, } for version in versions: tox = ['tox', '-c', 'test/runner/tox.ini', '-e', 'py' + version.replace('.', '')] if args.tox_sitepackages: tox.append('--sitepackages') tox.append('--') cmd = generate_command(args, None, ANSIBLE_ROOT, data_context().content.root, options, exclude, require) if not args.python: cmd += ['--python', version] # newer versions of tox do not support older python versions and will silently fall back to a different version # passing this option will allow the delegated ansible-test to verify it is running under the expected python version # tox 3.0.0 dropped official python 2.6 support: https://tox.readthedocs.io/en/latest/changelog.html#v3-0-0-2018-04-02 # tox 3.1.3 is the first version to support python 3.8 and later: https://tox.readthedocs.io/en/latest/changelog.html#v3-1-3-2018-08-03 # tox 3.1.3 appears to still work with python 2.6, making it a good version to use when supporting all python versions we use # virtualenv 16.0.0 dropped python 2.6 support: https://virtualenv.pypa.io/en/latest/changes/#v16-0-0-2018-05-16 cmd += ['--check-python', version] if isinstance(args, TestConfig): if args.coverage and not args.coverage_label: cmd += ['--coverage-label', 'tox-%s' % version] env = common_environment() # temporary solution to permit ansible-test delegated to tox to provision remote resources optional = ( 'SHIPPABLE', 'SHIPPABLE_BUILD_ID', 'SHIPPABLE_JOB_NUMBER', ) env.update(pass_vars(required=[], optional=optional)) run_command(args, tox + cmd, env=env)
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ skip_file = 'test/sanity/shellcheck/skip.txt' skip_paths = set( read_lines_without_comments(skip_file, remove_blank_lines=True, optional=True)) exclude_file = 'test/sanity/shellcheck/exclude.txt' exclude = set( read_lines_without_comments(exclude_file, remove_blank_lines=True, optional=True)) paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] == '.sh' and i.path not in skip_paths) if not paths: return SanitySkipped(self.name) cmd = [ 'shellcheck', '-e', ','.join(sorted(exclude)), '--format', 'checkstyle', ] + paths try: stdout, stderr = run_command(args, cmd, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr or status > 1: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) if args.explain: return SanitySuccess(self.name) # json output is missing file paths in older versions of shellcheck, so we'll use xml instead root = fromstring(stdout) # type: Element results = [] for item in root: # type: Element for entry in item: # type: Element results.append( SanityMessage( message=entry.attrib['message'], path=item.attrib['name'], line=int(entry.attrib['line']), column=int(entry.attrib['column']), level=entry.attrib['severity'], code=entry.attrib['source'].replace('ShellCheck.', ''), )) if results: return SanityFailure(self.name, messages=results) return SanitySuccess(self.name)
def test(self, args, targets, python_version): """ :type args: SanityConfig :type targets: SanityTargets :type python_version: str :rtype: TestResult """ cmd = [find_python(python_version), self.path] env = ansible_environment(args, color=False) pattern = None data = None settings = self.load_processor(args) paths = [target.path for target in targets.include] if self.config: if self.output == 'path-line-column-message': pattern = '^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<message>.*)$' elif self.output == 'path-message': pattern = '^(?P<path>[^:]*): (?P<message>.*)$' else: pattern = ApplicationError('Unsupported output type: %s' % self.output) if not self.no_targets: data = '\n'.join(paths) if data: display.info(data, verbosity=4) try: stdout, stderr = run_command(args, cmd, data=data, env=env, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stdout and not stderr: if pattern: matches = parse_to_list_of_dict(pattern, stdout) messages = [ SanityMessage( message=m['message'], path=m['path'], line=int(m.get('line', 0)), column=int(m.get('column', 0)), ) for m in matches ] messages = settings.process_errors(messages, paths) if not messages: return SanitySuccess(self.name) return SanityFailure(self.name, messages=messages) if stderr or status: summary = u'%s' % SubprocessError( cmd=cmd, status=status, stderr=stderr, stdout=stdout) return SanityFailure(self.name, summary=summary) messages = settings.process_errors([], paths) if messages: return SanityFailure(self.name, messages=messages) return SanitySuccess(self.name)
def test(self, args, targets): """ :type args: SanityConfig :type targets: SanityTargets :rtype: TestResult """ skip_paths = read_lines_without_comments(PEP8_SKIP_PATH, optional=True) legacy_paths = read_lines_without_comments(PEP8_LEGACY_PATH, optional=True) legacy_ignore_file = os.path.join( INSTALL_ROOT, 'test/sanity/pep8/legacy-ignore.txt') legacy_ignore = set( read_lines_without_comments(legacy_ignore_file, remove_blank_lines=True)) current_ignore_file = os.path.join( INSTALL_ROOT, 'test/sanity/pep8/current-ignore.txt') current_ignore = sorted( read_lines_without_comments(current_ignore_file, remove_blank_lines=True)) skip_paths_set = set(skip_paths) legacy_paths_set = set(legacy_paths) paths = sorted( i.path for i in targets.include if (os.path.splitext(i.path)[1] == '.py' or i.path.startswith('bin/')) and i.path not in skip_paths_set) cmd = [ args.python_executable, '-m', 'pycodestyle', '--max-line-length', '160', '--config', '/dev/null', '--ignore', ','.join(sorted(current_ignore)), ] + paths if paths: try: stdout, stderr = run_command(args, cmd, capture=True) status = 0 except SubprocessError as ex: stdout = ex.stdout stderr = ex.stderr status = ex.status if stderr: raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) else: stdout = None if args.explain: return SanitySuccess(self.name) if stdout: pattern = '^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<code>[WE][0-9]{3}) (?P<message>.*)$' results = parse_to_list_of_dict(pattern, stdout) else: results = [] results = [ SanityMessage( message=r['message'], path=r['path'], line=int(r['line']), column=int(r['column']), level='warning' if r['code'].startswith('W') else 'error', code=r['code'], ) for r in results ] failed_result_paths = set([result.path for result in results]) used_paths = set(paths) errors = [] summary = {} line = 0 for path in legacy_paths: line += 1 if not path: continue if not os.path.exists(path): # Keep files out of the list which no longer exist in the repo. errors.append( SanityMessage( code='A101', message='Remove "%s" since it does not exist' % path, path=PEP8_LEGACY_PATH, line=line, column=1, confidence=calculate_best_confidence( ((PEP8_LEGACY_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) if path in used_paths and path not in failed_result_paths: # Keep files out of the list which no longer require the relaxed rule set. errors.append( SanityMessage( code='A201', message= 'Remove "%s" since it passes the current rule set' % path, path=PEP8_LEGACY_PATH, line=line, column=1, confidence=calculate_best_confidence( ((PEP8_LEGACY_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) line = 0 for path in skip_paths: line += 1 if not path: continue if not os.path.exists(path): # Keep files out of the list which no longer exist in the repo. errors.append( SanityMessage( code='A101', message='Remove "%s" since it does not exist' % path, path=PEP8_SKIP_PATH, line=line, column=1, confidence=calculate_best_confidence( ((PEP8_SKIP_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, )) for result in results: if result.path in legacy_paths_set and result.code in legacy_ignore: # Files on the legacy list are permitted to have errors on the legacy ignore list. # However, we want to report on their existence to track progress towards eliminating these exceptions. display.info('PEP 8: %s (legacy)' % result, verbosity=3) key = '%s %s' % (result.code, re.sub('[0-9]+', 'NNN', result.message)) if key not in summary: summary[key] = 0 summary[key] += 1 else: # Files not on the legacy list and errors not on the legacy ignore list are PEP 8 policy errors. errors.append(result) if summary: lines = [] count = 0 for key in sorted(summary): count += summary[key] lines.append('PEP 8: %5d %s' % (summary[key], key)) display.info( 'PEP 8: There were %d different legacy issues found (%d total):' % (len(summary), count), verbosity=1) display.info('PEP 8: Count Code Message', verbosity=1) for line in lines: display.info(line, verbosity=1) if errors: return SanityFailure(self.name, messages=errors) return SanitySuccess(self.name)
def request(self, method, url, data=None, headers=None): """ :type method: str :type url: str :type data: str | None :type headers: dict[str, str] | None :rtype: HttpResponse """ cmd = ['curl', '-s', '-S', '-i', '-X', method] if self.insecure: cmd += ['--insecure'] if headers is None: headers = {} headers['Expect'] = '' # don't send expect continue header if self.username: if self.password: display.sensitive.add(self.password) cmd += ['-u', '%s:%s' % (self.username, self.password)] else: cmd += ['-u', self.username] for header in headers.keys(): cmd += ['-H', '%s: %s' % (header, headers[header])] if data is not None: cmd += ['-d', data] if self.proxy: cmd += ['-x', self.proxy] cmd += [url] attempts = 0 max_attempts = 3 sleep_seconds = 3 # curl error codes which are safe to retry (request never sent to server) retry_on_status = ( 6, # CURLE_COULDNT_RESOLVE_HOST ) stdout = '' while True: attempts += 1 try: stdout = run_command(self.args, cmd, capture=True, always=self.always, cmd_verbosity=2)[0] break except SubprocessError as ex: if ex.status in retry_on_status and attempts < max_attempts: display.warning(u'%s' % ex) time.sleep(sleep_seconds) continue raise if self.args.explain and not self.always: return HttpResponse(method, url, 200, '') header, body = stdout.split('\r\n\r\n', 1) response_headers = header.split('\r\n') first_line = response_headers[0] http_response = first_line.split(' ') status_code = int(http_response[1]) return HttpResponse(method, url, status_code, body)