def docker_run(args, image, options, cmd=None):
"""
:type args: EnvironmentConfig
:type image: str
:type options: list[str] | None
:type cmd: list[str] | None
:rtype: str | None, str | None
"""
if not options:
options = []
if not cmd:
cmd = []
for _ in range(1, 3):
try:
return docker_command(args, ['run'] + options + [image] + cmd,
capture=True)
except SubprocessError as ex:
display.error(ex)
display.warning(
'Failed to run docker image "%s". Waiting a few seconds before trying again.'
% image)
time.sleep(3)
raise ApplicationError('Failed to run docker image "%s".' % image)
def _start_try_endpoints(self, data, headers):
"""
:type data: dict[str, any]
:type headers: dict[str, str]
:rtype: HttpResponse
"""
threshold = 1
while threshold <= self.max_threshold:
for self.endpoint in self.endpoints:
try:
return self._start_at_threshold(data, headers, threshold)
except CoreHttpError as ex:
if ex.status == 503:
display.info('Service Unavailable: %s' % ex.remote_message, verbosity=1)
continue
display.error(ex.remote_message)
except HttpError as ex:
display.error(u'%s' % ex)
time.sleep(3)
threshold += 1
raise ApplicationError('Maximum threshold reached and all endpoints exhausted.')
def main():
"""Main program function."""
try:
args = parse_args()
display.verbosity = args.verbosity
display.color = args.color
try:
run_id = os.environ['SHIPPABLE_BUILD_ID']
except KeyError as ex:
raise MissingEnvironmentVariable(ex.args[0])
client = HttpClient(args)
response = client.get('https://api.shippable.com/jobs?runIds=%s' %
run_id)
jobs = response.json()
if len(jobs) == 1:
raise ApplicationError(
'Shippable run %s has only one job. Did you use the "Rebuild with SSH" option?'
% run_id)
except ApplicationWarning as ex:
display.warning(str(ex))
exit(0)
except ApplicationError as ex:
display.error(str(ex))
exit(1)
except KeyboardInterrupt:
exit(2)
except IOError as ex:
if ex.errno == errno.EPIPE:
exit(3)
raise
def main():
"""Main program function."""
try:
git_root = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..'))
os.chdir(git_root)
initialize_cloud_plugins()
sanity_init()
args = parse_args()
config = args.config(args)
display.verbosity = config.verbosity
display.truncate = config.truncate
display.color = config.color
display.info_stderr = (isinstance(config, SanityConfig) and config.lint) or (isinstance(config, IntegrationConfig) and config.list_targets)
check_startup()
try:
args.func(config)
except Delegate as ex:
delegate(config, ex.exclude, ex.require)
display.review_warnings()
except ApplicationWarning as ex:
display.warning(str(ex))
exit(0)
except ApplicationError as ex:
display.error(str(ex))
exit(1)
except KeyboardInterrupt:
exit(2)
except IOError as ex:
if ex.errno == errno.EPIPE:
exit(3)
raise
def main():
"""Main program function."""
try:
args = parse_args()
display.verbosity = args.verbosity
display.color = args.color
command = [args.command] + args.args
for attempt in range(0, args.tries):
if attempt > 0:
time.sleep(args.sleep)
try:
raw_command(command, env=os.environ)
return
except SubprocessError as ex:
display.error(ex)
except ApplicationWarning as ex:
display.warning(str(ex))
exit(0)
except ApplicationError as ex:
display.error(str(ex))
exit(1)
except KeyboardInterrupt:
exit(2)
except IOError as ex:
if ex.errno == errno.EPIPE:
exit(3)
raise
def validate(self, target_name, throw):
"""
:type target_name: str
:type throw: bool
:rtype: bool
"""
current = EnvironmentDescription(self.args)
original_json = str(self)
current_json = str(current)
if original_json == current_json:
return True
message = ('Test target "%s" has changed the test environment!\n'
'If these changes are necessary, they must be reverted before the test finishes.\n'
'>>> Original Environment\n'
'%s\n'
'>>> Current Environment\n'
'%s' % (target_name, original_json, current_json))
if throw:
raise ApplicationError(message)
display.error(message)
return False
def validate(self, target_name, throw):
"""
:type target_name: str
:type throw: bool
:rtype: bool
"""
current = EnvironmentDescription(self.args)
original_json = str(self)
current_json = str(current)
if original_json == current_json:
return True
message = ('Test target "%s" has changed the test environment!\n'
'If these changes are necessary, they must be reverted before the test finishes.\n'
'>>> Original Environment\n'
'%s\n'
'>>> Current Environment\n'
'%s' % (target_name, original_json, current_json))
if throw:
raise ApplicationError(message)
display.error(message)
return False
def _start_try_endpoints(self, data, headers):
"""
:type data: dict[str, any]
:type headers: dict[str, str]
:rtype: HttpResponse
"""
threshold = 1
while threshold <= self.max_threshold:
for self.endpoint in self.endpoints:
try:
return self._start_at_threshold(data, headers, threshold)
except CoreHttpError as ex:
if ex.status == 503:
display.info('Service Unavailable: %s' %
ex.remote_message,
verbosity=1)
continue
display.error(ex.remote_message)
except HttpError as ex:
display.error(u'%s' % ex)
time.sleep(3)
threshold += 1
raise ApplicationError(
'Maximum threshold reached and all endpoints exhausted.')
def command_sanity_ansible_doc(args, targets, python_version):
"""
:type args: SanityConfig
:type targets: SanityTargets
:type python_version: str
"""
with open('test/sanity/ansible-doc/skip.txt', 'r') as skip_fd:
skip_modules = set(skip_fd.read().splitlines())
modules = sorted(
set(m for i in targets.include_external
for m in i.modules) - set(m for i in targets.exclude_external
for m in i.modules) - skip_modules)
if not modules:
display.info('No tests applicable.', verbosity=1)
return
env = ansible_environment(args)
cmd = ['ansible-doc'] + modules
stdout, stderr = intercept_command(args,
cmd,
env=env,
capture=True,
python_version=python_version)
if stderr:
display.error(
'Output on stderr from ansible-doc is considered an error.')
raise SubprocessError(cmd, stderr=stderr)
if stdout:
display.info(stdout.strip(), verbosity=3)
def main():
"""Main program function."""
try:
git_root = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..'))
os.chdir(git_root)
args = parse_args()
config = args.config(args)
display.verbosity = config.verbosity
display.color = config.color
check_startup()
try:
args.func(config)
except Delegate as ex:
delegate(config, ex.exclude, ex.require)
display.review_warnings()
except ApplicationWarning as ex:
display.warning(str(ex))
exit(0)
except ApplicationError as ex:
display.error(str(ex))
exit(1)
except KeyboardInterrupt:
exit(2)
except IOError as ex:
if ex.errno == errno.EPIPE:
exit(3)
raise
def main():
"""Main program function."""
try:
git_root = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..'))
os.chdir(git_root)
initialize_cloud_plugins()
sanity_init()
args = parse_args()
config = args.config(args)
display.verbosity = config.verbosity
display.color = config.color
display.info_stderr = (isinstance(config, SanityConfig) and config.lint) or (isinstance(config, IntegrationConfig) and config.list_targets)
check_startup()
try:
args.func(config)
except Delegate as ex:
delegate(config, ex.exclude, ex.require)
display.review_warnings()
except ApplicationWarning as ex:
display.warning(str(ex))
exit(0)
except ApplicationError as ex:
display.error(str(ex))
exit(1)
except KeyboardInterrupt:
exit(2)
except IOError as ex:
if ex.errno == errno.EPIPE:
exit(3)
raise
def main():
"""Main program function."""
try:
args = parse_args()
display.verbosity = args.verbosity
display.color = args.color
try:
run_id = os.environ['SHIPPABLE_BUILD_ID']
except KeyError as ex:
raise MissingEnvironmentVariable(ex.args[0])
client = HttpClient(args)
response = client.get('https://api.shippable.com/jobs?runIds=%s' % run_id)
jobs = response.json()
if not isinstance(jobs, list):
raise ApplicationError(json.dumps(jobs, indent=4, sort_keys=True))
if len(jobs) == 1:
raise ApplicationError('Shippable run %s has only one job. Did you use the "Rebuild with SSH" option?' % run_id)
except ApplicationWarning as ex:
display.warning(str(ex))
exit(0)
except ApplicationError as ex:
display.error(str(ex))
exit(1)
except KeyboardInterrupt:
exit(2)
except IOError as ex:
if ex.errno == errno.EPIPE:
exit(3)
raise
def main():
"""Main program function."""
try:
git_root = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..'))
os.chdir(git_root)
args = parse_args()
config = args.config(args)
display.verbosity = config.verbosity
display.color = config.color
try:
args.func(config)
except Delegate as ex:
delegate(config, ex.exclude, ex.require)
display.review_warnings()
except ApplicationWarning as ex:
display.warning(str(ex))
exit(0)
except ApplicationError as ex:
display.error(str(ex))
exit(1)
except KeyboardInterrupt:
exit(2)
except IOError as ex:
if ex.errno == errno.EPIPE:
exit(3)
raise
def main():
"""Main program function."""
try:
git_root = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)), '..',
'..', '..'))
os.chdir(git_root)
initialize_cloud_plugins()
sanity_init()
args = parse_args()
config = args.config(args)
display.verbosity = config.verbosity
display.truncate = config.truncate
display.redact = config.redact
display.color = config.color
display.info_stderr = (isinstance(config, SanityConfig)
and config.lint) or (isinstance(
config, IntegrationConfig)
and config.list_targets)
check_startup()
# to achieve a consistent nofile ulimit, set to 16k here, this can affect performance in subprocess.Popen when
# being called with close_fds=True on Python (8x the time on some environments)
nofile_limit = 16 * 1024
current_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
new_limit = (nofile_limit, nofile_limit)
if current_limit > new_limit:
display.info('RLIMIT_NOFILE: %s -> %s' %
(current_limit, new_limit),
verbosity=2)
resource.setrlimit(resource.RLIMIT_NOFILE,
(nofile_limit, nofile_limit))
else:
display.info('RLIMIT_NOFILE: %s' % (current_limit, ), verbosity=2)
try:
args.func(config)
except Delegate as ex:
delegate(config, ex.exclude, ex.require, ex.integration_targets)
display.review_warnings()
except ApplicationWarning as ex:
display.warning(u'%s' % ex)
exit(0)
except ApplicationError as ex:
display.error(u'%s' % ex)
exit(1)
except KeyboardInterrupt:
exit(2)
except IOError as ex:
if ex.errno == errno.EPIPE:
exit(3)
raise
def _get_simulator_address(self):
networks = docker_network_inspect(self.args, 'bridge')
try:
bridge = [network for network in networks if network['Name'] == 'bridge'][0]
containers = bridge['Containers']
container = [containers[container] for container in containers if containers[container]['Name'] == self.DOCKER_SIMULATOR_NAME][0]
return re.sub(r'/[0-9]+$', '', container['IPv4Address'])
except:
display.error('Failed to process the following docker network inspect output:\n%s' %
json.dumps(networks, indent=4, sort_keys=True))
raise
def _get_container_address(self):
networks = docker_network_inspect(self.args, 'bridge')
try:
bridge = [network for network in networks if network['Name'] == 'bridge'][0]
containers = bridge['Containers']
container = [containers[container] for container in containers if containers[container]['Name'] == self.DOCKER_CONTAINER_NAME][0]
return re.sub(r'/[0-9]+$', '', container['IPv4Address'])
except Exception:
display.error('Failed to process the following docker network inspect output:\n%s' %
json.dumps(networks, indent=4, sort_keys=True))
raise
def main():
"""Main program function."""
try:
os.chdir(data_context().content.root)
initialize_cloud_plugins()
sanity_init()
args = parse_args()
config = args.config(args) # type: CommonConfig
display.verbosity = config.verbosity
display.truncate = config.truncate
display.redact = config.redact
display.color = config.color
display.info_stderr = (isinstance(config, SanityConfig)
and config.lint) or (isinstance(
config, IntegrationConfig)
and config.list_targets)
check_startup()
check_delegation_args(config)
configure_timeout(config)
display.info('RLIMIT_NOFILE: %s' % (CURRENT_RLIMIT_NOFILE, ),
verbosity=2)
display.info('MAXFD: %d' % MAXFD, verbosity=2)
try:
args.func(config)
delegate_args = None
except Delegate as ex:
# save delegation args for use once we exit the exception handler
delegate_args = (ex.exclude, ex.require, ex.integration_targets)
if delegate_args:
delegate(config, *delegate_args)
display.review_warnings()
except ApplicationWarning as ex:
display.warning(u'%s' % ex)
exit(0)
except ApplicationError as ex:
display.error(u'%s' % ex)
exit(1)
except KeyboardInterrupt:
exit(2)
except IOError as ex:
if ex.errno == errno.EPIPE:
exit(3)
raise
def main():
"""Main program function."""
try:
git_root = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)), '..',
'..', '..'))
#os.chdir(git_root)
initialize_cloud_plugins()
sanity_init()
args = parse_args()
config = args.config(args)
display.verbosity = config.verbosity
display.truncate = config.truncate
display.redact = config.redact
display.color = config.color
display.info_stderr = (isinstance(config, SanityConfig)
and config.lint) or (isinstance(
config, IntegrationConfig)
and config.list_targets)
check_startup()
check_delegation_args(config)
configure_timeout(config)
display.info('RLIMIT_NOFILE: %s' % (CURRENT_RLIMIT_NOFILE, ),
verbosity=2)
display.info('MAXFD: %d' % MAXFD, verbosity=2)
try:
args.func(config)
except Delegate as ex:
delegate(config, ex.exclude, ex.require, ex.integration_targets)
display.review_warnings()
except ApplicationWarning as ex:
display.warning(u'%s' % ex)
exit(0)
except ApplicationError as ex:
display.error(u'%s' % ex)
exit(1)
except KeyboardInterrupt:
exit(2)
except IOError as ex:
if ex.errno == errno.EPIPE:
exit(3)
raise
def command_compile(args):
"""
:type args: CompileConfig
"""
changes = get_changes_filter(args)
require = (args.require or []) + changes
include, exclude = walk_external_targets(walk_compile_targets(),
args.include, args.exclude,
require)
if not include:
raise AllTargetsSkipped()
if args.delegate:
raise Delegate(require=changes)
install_command_requirements(args)
total = 0
failed = []
for version in COMPILE_PYTHON_VERSIONS:
# run all versions unless version given, in which case run only that version
if args.python and version != args.python:
continue
display.info('Compile with Python %s' % version)
result = compile_version(args, version, include, exclude)
result.write(args)
total += 1
if isinstance(result, TestFailure):
failed.append('compile --python %s' % version)
if failed:
message = 'The %d compile test(s) listed below (out of %d) failed. See error output above for details.\n%s' % (
len(failed), total, '\n'.join(failed))
if args.failure_ok:
display.error(message)
else:
raise ApplicationError(message)
def docker_run(args, image, options):
"""
:type args: EnvironmentConfig
:type image: str
:type options: list[str] | None
:rtype: str | None, str | None
"""
if not options:
options = []
for _ in range(1, 3):
try:
return docker_command(args, ['run'] + options + [image], capture=True)
except SubprocessError as ex:
display.error(ex)
display.warning('Failed to run docker image "%s". Waiting a few seconds before trying again.' % image)
time.sleep(3)
raise ApplicationError('Failed to run docker image "%s".' % image)
def command_compile(args):
"""
:type args: CompileConfig
"""
changes = get_changes_filter(args)
require = (args.require or []) + changes
include, exclude = walk_external_targets(walk_compile_targets(), args.include, args.exclude, require)
if not include:
raise AllTargetsSkipped()
if args.delegate:
raise Delegate(require=changes)
install_command_requirements(args)
total = 0
failed = []
for version in COMPILE_PYTHON_VERSIONS:
# run all versions unless version given, in which case run only that version
if args.python and version != args.python:
continue
display.info('Compile with Python %s' % version)
result = compile_version(args, version, include, exclude)
result.write(args)
total += 1
if isinstance(result, TestFailure):
failed.append('compile --python %s' % version)
if failed:
message = 'The %d compile test(s) listed below (out of %d) failed. See error output above for details.\n%s' % (
len(failed), total, '\n'.join(failed))
if args.failure_ok:
display.error(message)
else:
raise ApplicationError(message)
def _get_credentials(self):
"""Wait for the CloudStack simulator to return credentials.
:rtype: dict[str, str]
"""
client = HttpClient(self.args, always=True)
endpoint = '%s/admin.json' % self.endpoint
for _ in range(1, 30):
display.info('Waiting for CloudStack credentials: %s' % endpoint, verbosity=1)
response = client.get(endpoint)
if response.status_code == 200:
try:
return response.json()
except HttpError as ex:
display.error(ex)
time.sleep(30)
raise ApplicationError('Timeout waiting for CloudStack credentials.')
def _get_credentials(self):
"""Wait for the CloudStack simulator to return credentials.
:rtype: dict[str, str]
"""
client = HttpClient(self.args, always=True)
endpoint = '%s/admin.json' % self.endpoint
for _ in range(1, 30):
display.info('Waiting for CloudStack credentials: %s' % endpoint, verbosity=1)
response = client.get(endpoint)
if response.status_code == 200:
try:
return response.json()
except HttpError as ex:
display.error(ex)
time.sleep(30)
raise ApplicationError('Timeout waiting for CloudStack credentials.')
def write_console(self):
"""Write results to console."""
if self.summary:
display.error(self.summary)
else:
display.error('Found %d %s issue(s) which need to be resolved:' %
(len(self.messages), self.test))
for message in self.messages:
display.error(message)
def write_console(self):
"""Write results to console."""
if self.summary:
display.error(self.summary)
else:
if self.python_version:
specifier = ' on python %s' % self.python_version
else:
specifier = ''
display.error('Found %d %s issue(s)%s which need to be resolved:' % (len(self.messages), self.test or self.command, specifier))
for message in self.messages:
display.error(message.format(show_confidence=True))
def write_console(self):
"""Write results to console."""
if self.summary:
display.error(self.summary)
else:
if self.python_version:
specifier = ' on python %s' % self.python_version
else:
specifier = ''
display.error('Found %d %s issue(s)%s which need to be resolved:' % (len(self.messages), self.test or self.command, specifier))
for message in self.messages:
display.error(message.format(show_confidence=True))
def command_sanity_pep8(args, targets):
"""
:type args: SanityConfig
:type targets: SanityTargets
"""
skip_path = 'test/sanity/pep8/skip.txt'
legacy_path = 'test/sanity/pep8/legacy-files.txt'
with open(skip_path, 'r') as skip_fd:
skip_paths = set(skip_fd.read().splitlines())
with open(legacy_path, 'r') as legacy_fd:
legacy_paths = set(legacy_fd.read().splitlines())
with open('test/sanity/pep8/legacy-ignore.txt', 'r') as ignore_fd:
legacy_ignore = set(ignore_fd.read().splitlines())
with open('test/sanity/pep8/current-ignore.txt', 'r') as ignore_fd:
current_ignore = sorted(ignore_fd.read().splitlines())
paths = sorted(
i.path for i in targets.include
if os.path.splitext(i.path)[1] == '.py' and i.path not in skip_paths)
if not paths:
display.info('No tests applicable.', verbosity=1)
return
cmd = [
'pep8',
'--max-line-length',
'160',
'--config',
'/dev/null',
'--ignore',
','.join(sorted(current_ignore)),
] + paths
try:
stdout, stderr = run_command(args, cmd, capture=True)
status = 0
except SubprocessError as ex:
stdout = ex.stdout
stderr = ex.stderr
status = ex.status
if stderr:
raise SubprocessError(cmd=cmd, status=status, stderr=stderr)
pattern = '^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<code>[A-Z0-9]{4}) (?P<message>.*)$'
results = [
re.search(pattern, line).groupdict() for line in stdout.splitlines()
]
for result in results:
for key in 'line', 'column':
result[key] = int(result[key])
failed_result_paths = set([result['path'] for result in results])
passed_legacy_paths = set([
path for path in paths
if path in legacy_paths and path not in failed_result_paths
])
errors = []
summary = {}
for path in sorted(passed_legacy_paths):
# Keep files out of the list which no longer require the relaxed rule set.
errors.append(
'PEP 8: %s: Passes current rule set. Remove from legacy list (%s).'
% (path, legacy_path))
for path in sorted(skip_paths):
if not os.path.exists(path):
# Keep files out of the list which no longer exist in the repo.
errors.append(
'PEP 8: %s: Does not exist. Remove from skip list (%s).' %
(path, skip_path))
for path in sorted(legacy_paths):
if not os.path.exists(path):
# Keep files out of the list which no longer exist in the repo.
errors.append(
'PEP 8: %s: Does not exist. Remove from legacy list (%s).' %
(path, legacy_path))
for result in results:
path = result['path']
line = result['line']
column = result['column']
code = result['code']
message = result['message']
msg = 'PEP 8: %s:%s:%s: %s %s' % (path, line, column, code, message)
if path in legacy_paths:
msg += ' (legacy)'
else:
msg += ' (current)'
if path in legacy_paths and code in legacy_ignore:
# Files on the legacy list are permitted to have errors on the legacy ignore list.
# However, we want to report on their existence to track progress towards eliminating these exceptions.
display.info(msg, verbosity=3)
key = '%s %s' % (code, re.sub('[0-9]+', 'NNN', message))
if key not in summary:
summary[key] = 0
summary[key] += 1
else:
# Files not on the legacy list and errors not on the legacy ignore list are PEP 8 policy errors.
errors.append(msg)
for error in errors:
display.error(error)
if summary:
lines = []
count = 0
for key in sorted(summary):
count += summary[key]
lines.append('PEP 8: %5d %s' % (summary[key], key))
display.info(
'PEP 8: There were %d different legacy issues found (%d total):' %
(len(summary), count),
verbosity=1)
display.info('PEP 8: Count Code Message', verbosity=1)
for line in lines:
display.info(line, verbosity=1)
if errors:
raise ApplicationError(
'PEP 8: There are %d issues which need to be resolved.' %
len(errors))
def command_coverage_combine(args):
"""Patch paths in coverage files and merge into a single file.
:type args: CoverageConfig
"""
coverage = initialize_coverage(args)
modules = dict((t.module, t.path) for t in list(walk_module_targets()))
coverage_files = [os.path.join(COVERAGE_DIR, f) for f in os.listdir(COVERAGE_DIR)
if f.startswith('coverage') and f != 'coverage']
arc_data = {}
ansible_path = os.path.abspath('lib/ansible/') + '/'
root_path = os.getcwd() + '/'
for coverage_file in coverage_files:
original = coverage.CoverageData()
if os.path.getsize(coverage_file) == 0:
display.warning('Empty coverage file: %s' % coverage_file)
continue
try:
original.read_file(coverage_file)
except Exception as ex: # pylint: disable=locally-disabled, broad-except
display.error(str(ex))
continue
for filename in original.measured_files():
arcs = original.arcs(filename)
if '/ansible_modlib.zip/ansible/' in filename:
new_name = re.sub('^.*/ansible_modlib.zip/ansible/', ansible_path, filename)
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
elif '/ansible_module_' in filename:
module = re.sub('^.*/ansible_module_(?P<module>.*).py$', '\\g<module>', filename)
new_name = os.path.abspath(modules[module])
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
elif filename.startswith('/root/ansible/'):
new_name = re.sub('^/.*?/ansible/', root_path, filename)
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
if filename not in arc_data:
arc_data[filename] = []
arc_data[filename] += arcs
updated = coverage.CoverageData()
for filename in arc_data:
if not os.path.isfile(filename):
display.warning('Invalid coverage path: %s' % filename)
continue
updated.add_arcs({filename: arc_data[filename]})
if not args.explain:
updated.write_file(COVERAGE_FILE)
def command_sanity(args):
"""
:type args: SanityConfig
"""
changes = get_changes_filter(args)
require = (args.require or []) + changes
targets = SanityTargets(args.include, args.exclude, require)
if not targets.include:
raise AllTargetsSkipped()
if args.delegate:
raise Delegate(require=changes)
install_command_requirements(args)
tests = sanity_get_tests()
if args.test:
tests = [t for t in tests if t.name in args.test]
if args.skip_test:
tests = [t for t in tests if t.name not in args.skip_test]
total = 0
failed = []
for test in tests:
if args.list_tests:
display.info(test.name)
continue
if test.intercept:
versions = SUPPORTED_PYTHON_VERSIONS
else:
versions = None,
for version in versions:
if args.python and version and version != args.python:
continue
display.info('Sanity check using %s%s' % (test.name, ' with Python %s' % version if version else ''))
options = ''
if test.script:
result = test.func(args, targets, test.script)
elif test.intercept:
result = test.func(args, targets, python_version=version)
options = ' --python %s' % version
else:
result = test.func(args, targets)
result.write(args)
total += 1
if isinstance(result, SanityFailure):
failed.append(result.test + options)
if failed:
message = 'The %d sanity test(s) listed below (out of %d) failed. See error output above for details.\n%s' % (
len(failed), total, '\n'.join(failed))
if args.failure_ok:
display.error(message)
else:
raise ApplicationError(message)
def command_integration_filtered(args, targets, all_targets):
"""
:type args: IntegrationConfig
:type targets: tuple[IntegrationTarget]
:type all_targets: tuple[IntegrationTarget]
"""
found = False
passed = []
failed = []
targets_iter = iter(targets)
all_targets_dict = dict((target.name, target) for target in all_targets)
setup_errors = []
setup_targets_executed = set()
for target in all_targets:
for setup_target in target.setup_once + target.setup_always:
if setup_target not in all_targets_dict:
setup_errors.append('Target "%s" contains invalid setup target: %s' % (target.name, setup_target))
if setup_errors:
raise ApplicationError('Found %d invalid setup aliases:\n%s' % (len(setup_errors), '\n'.join(setup_errors)))
test_dir = os.path.expanduser('~/ansible_testing')
if not args.explain and any('needs/ssh/' in target.aliases for target in targets):
max_tries = 20
display.info('SSH service required for tests. Checking to make sure we can connect.')
for i in range(1, max_tries + 1):
try:
run_command(args, ['ssh', '-o', 'BatchMode=yes', 'localhost', 'id'], capture=True)
display.info('SSH service responded.')
break
except SubprocessError:
if i == max_tries:
raise
seconds = 3
display.warning('SSH service not responding. Waiting %d second(s) before checking again.' % seconds)
time.sleep(seconds)
start_at_task = args.start_at_task
results = {}
for target in targets_iter:
if args.start_at and not found:
found = target.name == args.start_at
if not found:
continue
if args.list_targets:
print(target.name)
continue
tries = 2 if args.retry_on_error else 1
verbosity = args.verbosity
cloud_environment = get_cloud_environment(args, target)
original_environment = EnvironmentDescription(args)
display.info('>>> Environment Description\n%s' % original_environment, verbosity=3)
try:
while tries:
tries -= 1
try:
run_setup_targets(args, test_dir, target.setup_once, all_targets_dict, setup_targets_executed, False)
start_time = time.time()
run_setup_targets(args, test_dir, target.setup_always, all_targets_dict, setup_targets_executed, True)
if not args.explain:
# create a fresh test directory for each test target
remove_tree(test_dir)
make_dirs(test_dir)
if target.script_path:
command_integration_script(args, target)
else:
command_integration_role(args, target, start_at_task)
start_at_task = None
end_time = time.time()
results[target.name] = dict(
name=target.name,
type=target.type,
aliases=target.aliases,
modules=target.modules,
run_time_seconds=int(end_time - start_time),
setup_once=target.setup_once,
setup_always=target.setup_always,
coverage=args.coverage,
coverage_label=args.coverage_label,
python_version=args.python_version,
)
break
except SubprocessError:
if cloud_environment:
cloud_environment.on_failure(target, tries)
if not original_environment.validate(target.name, throw=False):
raise
if not tries:
raise
display.warning('Retrying test target "%s" with maximum verbosity.' % target.name)
display.verbosity = args.verbosity = 6
original_environment.validate(target.name, throw=True)
passed.append(target)
except Exception as ex:
failed.append(target)
if args.continue_on_error:
display.error(ex)
continue
display.notice('To resume at this test target, use the option: --start-at %s' % target.name)
next_target = next(targets_iter, None)
if next_target:
display.notice('To resume after this test target, use the option: --start-at %s' % next_target.name)
raise
finally:
display.verbosity = args.verbosity = verbosity
if not args.explain:
results_path = 'test/results/data/%s-%s.json' % (args.command, re.sub(r'[^0-9]', '-', str(datetime.datetime.utcnow().replace(microsecond=0))))
data = dict(
targets=results,
)
with open(results_path, 'w') as results_fd:
results_fd.write(json.dumps(data, sort_keys=True, indent=4))
if failed:
raise ApplicationError('The %d integration test(s) listed below (out of %d) failed. See error output above for details:\n%s' % (
len(failed), len(passed) + len(failed), '\n'.join(target.name for target in failed)))
def command_coverage_combine(args):
"""Patch paths in coverage files and merge into a single file.
:type args: CoverageConfig
:rtype: list[str]
"""
coverage = initialize_coverage(args)
modules = dict((t.module, t.path) for t in list(walk_module_targets())
if t.path.endswith('.py'))
coverage_files = [
os.path.join(COVERAGE_DIR, f) for f in os.listdir(COVERAGE_DIR)
if '=coverage.' in f
]
ansible_path = os.path.abspath('lib/ansible/') + '/'
root_path = data_context().content.root + '/'
counter = 0
groups = {}
if args.all or args.stub:
sources = sorted(
os.path.abspath(target.path) for target in walk_compile_targets())
else:
sources = []
if args.stub:
stub_group = []
stub_groups = [stub_group]
stub_line_limit = 500000
stub_line_count = 0
for source in sources:
with open(source, 'r') as source_fd:
source_line_count = len(source_fd.read().splitlines())
stub_group.append(source)
stub_line_count += source_line_count
if stub_line_count > stub_line_limit:
stub_line_count = 0
stub_group = []
stub_groups.append(stub_group)
for stub_index, stub_group in enumerate(stub_groups):
if not stub_group:
continue
groups['=stub-%02d' % (stub_index + 1)] = dict(
(source, set()) for source in stub_group)
if data_context().content.collection:
collection_search_re = re.compile(
r'/%s/' % data_context().content.collection.directory)
collection_sub_re = re.compile(
r'^.*?/%s/' % data_context().content.collection.directory)
else:
collection_search_re = None
collection_sub_re = None
for coverage_file in coverage_files:
counter += 1
display.info('[%4d/%4d] %s' %
(counter, len(coverage_files), coverage_file),
verbosity=2)
original = coverage.CoverageData()
group = get_coverage_group(args, coverage_file)
if group is None:
display.warning('Unexpected name for coverage file: %s' %
coverage_file)
continue
if os.path.getsize(coverage_file) == 0:
display.warning('Empty coverage file: %s' % coverage_file)
continue
try:
original.read_file(coverage_file)
except Exception as ex: # pylint: disable=locally-disabled, broad-except
display.error(u'%s' % ex)
continue
for filename in original.measured_files():
arcs = set(original.arcs(filename) or [])
if not arcs:
# This is most likely due to using an unsupported version of coverage.
display.warning('No arcs found for "%s" in coverage file: %s' %
(filename, coverage_file))
continue
if '/ansible_modlib.zip/ansible/' in filename:
# Rewrite the module_utils path from the remote host to match the controller. Ansible 2.6 and earlier.
new_name = re.sub('^.*/ansible_modlib.zip/ansible/',
ansible_path, filename)
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
elif collection_search_re and collection_search_re.search(
filename):
new_name = os.path.abspath(collection_sub_re.sub('', filename))
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
elif re.search(r'/ansible_[^/]+_payload\.zip/ansible/', filename):
# Rewrite the module_utils path from the remote host to match the controller. Ansible 2.7 and later.
new_name = re.sub(r'^.*/ansible_[^/]+_payload\.zip/ansible/',
ansible_path, filename)
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
elif '/ansible_module_' in filename:
# Rewrite the module path from the remote host to match the controller. Ansible 2.6 and earlier.
module_name = re.sub('^.*/ansible_module_(?P<module>.*).py$',
'\\g<module>', filename)
if module_name not in modules:
display.warning('Skipping coverage of unknown module: %s' %
module_name)
continue
new_name = os.path.abspath(modules[module_name])
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
elif re.search(
r'/ansible_[^/]+_payload(_[^/]+|\.zip)/__main__\.py$',
filename):
# Rewrite the module path from the remote host to match the controller. Ansible 2.7 and later.
# AnsiballZ versions using zipimporter will match the `.zip` portion of the regex.
# AnsiballZ versions not using zipimporter will match the `_[^/]+` portion of the regex.
module_name = re.sub(
r'^.*/ansible_(?P<module>[^/]+)_payload(_[^/]+|\.zip)/__main__\.py$',
'\\g<module>', filename).rstrip('_')
if module_name not in modules:
display.warning('Skipping coverage of unknown module: %s' %
module_name)
continue
new_name = os.path.abspath(modules[module_name])
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
elif re.search('^(/.*?)?/root/ansible/', filename):
# Rewrite the path of code running on a remote host or in a docker container as root.
new_name = re.sub('^(/.*?)?/root/ansible/', root_path,
filename)
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
elif '/.ansible/test/tmp/' in filename:
# Rewrite the path of code running from an integration test temporary directory.
new_name = re.sub(r'^.*/\.ansible/test/tmp/[^/]+/', root_path,
filename)
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
if group not in groups:
groups[group] = {}
arc_data = groups[group]
if filename not in arc_data:
arc_data[filename] = set()
arc_data[filename].update(arcs)
output_files = []
invalid_path_count = 0
invalid_path_chars = 0
for group in sorted(groups):
arc_data = groups[group]
updated = coverage.CoverageData()
for filename in arc_data:
if not os.path.isfile(filename):
invalid_path_count += 1
invalid_path_chars += len(filename)
if args.verbosity > 1:
display.warning('Invalid coverage path: %s' % filename)
continue
updated.add_arcs({filename: list(arc_data[filename])})
if args.all:
updated.add_arcs(dict((source, []) for source in sources))
if not args.explain:
output_file = COVERAGE_FILE + group
updated.write_file(output_file)
output_files.append(output_file)
if invalid_path_count > 0:
display.warning(
'Ignored %d characters from %d invalid coverage path(s).' %
(invalid_path_chars, invalid_path_count))
return sorted(output_files)
def command_coverage_combine(args):
"""Patch paths in coverage files and merge into a single file.
:type args: CoverageConfig
"""
coverage = initialize_coverage(args)
modules = dict((t.module, t.path) for t in list(walk_module_targets()))
coverage_files = [
os.path.join(COVERAGE_DIR, f) for f in os.listdir(COVERAGE_DIR)
if f.startswith('coverage') and f != 'coverage'
]
arc_data = {}
ansible_path = os.path.abspath('lib/ansible/') + '/'
root_path = os.getcwd() + '/'
counter = 0
for coverage_file in coverage_files:
counter += 1
display.info('[%4d/%4d] %s' %
(counter, len(coverage_files), coverage_file),
verbosity=2)
original = coverage.CoverageData()
if os.path.getsize(coverage_file) == 0:
display.warning('Empty coverage file: %s' % coverage_file)
continue
try:
original.read_file(coverage_file)
except Exception as ex: # pylint: disable=locally-disabled, broad-except
display.error(str(ex))
continue
for filename in original.measured_files():
arcs = original.arcs(filename)
if '/ansible_modlib.zip/ansible/' in filename:
new_name = re.sub('^.*/ansible_modlib.zip/ansible/',
ansible_path, filename)
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
elif '/ansible_module_' in filename:
module = re.sub('^.*/ansible_module_(?P<module>.*).py$',
'\\g<module>', filename)
new_name = os.path.abspath(modules[module])
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
elif filename.startswith('/root/ansible/'):
new_name = re.sub('^/.*?/ansible/', root_path, filename)
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
if filename not in arc_data:
arc_data[filename] = []
arc_data[filename] += arcs
updated = coverage.CoverageData()
for filename in arc_data:
if not os.path.isfile(filename):
display.warning('Invalid coverage path: %s' % filename)
continue
updated.add_arcs({filename: arc_data[filename]})
if not args.explain:
updated.write_file(COVERAGE_FILE)
def command_integration_filtered(args, targets):
"""
:type args: IntegrationConfig
:type targets: tuple[IntegrationTarget]
"""
found = False
passed = []
failed = []
targets_iter = iter(targets)
test_dir = os.path.expanduser('~/ansible_testing')
if not args.explain and any('needs/ssh/' in target.aliases
for target in targets):
max_tries = 20
display.info(
'SSH service required for tests. Checking to make sure we can connect.'
)
for i in range(1, max_tries + 1):
try:
run_command(args,
['ssh', '-o', 'BatchMode=yes', 'localhost', 'id'],
capture=True)
display.info('SSH service responded.')
break
except SubprocessError:
if i == max_tries:
raise
seconds = 3
display.warning(
'SSH service not responding. Waiting %d second(s) before checking again.'
% seconds)
time.sleep(seconds)
start_at_task = args.start_at_task
for target in targets_iter:
if args.start_at and not found:
found = target.name == args.start_at
if not found:
continue
if args.list_targets:
print(target.name)
continue
tries = 2 if args.retry_on_error else 1
verbosity = args.verbosity
cloud_environment = get_cloud_environment(args, target)
original_environment = EnvironmentDescription(args)
display.info('>>> Environment Description\n%s' % original_environment,
verbosity=3)
try:
while tries:
tries -= 1
if not args.explain:
# create a fresh test directory for each test target
remove_tree(test_dir)
make_dirs(test_dir)
try:
if target.script_path:
command_integration_script(args, target)
else:
command_integration_role(args, target, start_at_task)
start_at_task = None
break
except SubprocessError:
if cloud_environment:
cloud_environment.on_failure(target, tries)
if not original_environment.validate(target.name,
throw=False):
raise
if not tries:
raise
display.warning(
'Retrying test target "%s" with maximum verbosity.' %
target.name)
display.verbosity = args.verbosity = 6
original_environment.validate(target.name, throw=True)
passed.append(target)
except Exception as ex:
failed.append(target)
if args.continue_on_error:
display.error(ex)
continue
display.notice(
'To resume at this test target, use the option: --start-at %s'
% target.name)
next_target = next(targets_iter, None)
if next_target:
display.notice(
'To resume after this test target, use the option: --start-at %s'
% next_target.name)
raise
finally:
display.verbosity = args.verbosity = verbosity
if failed:
raise ApplicationError(
'The %d integration test(s) listed below (out of %d) failed. See error output above for details:\n%s'
% (len(failed), len(passed) + len(failed), '\n'.join(
target.name for target in failed)))
def command_coverage_combine(args):
"""Patch paths in coverage files and merge into a single file.
:type args: CoverageConfig
:rtype: list[str]
"""
coverage = initialize_coverage(args)
modules = dict((t.module, t.path) for t in list(walk_module_targets()))
coverage_files = [os.path.join(COVERAGE_DIR, f) for f in os.listdir(COVERAGE_DIR) if '=coverage.' in f]
ansible_path = os.path.abspath('lib/ansible/') + '/'
root_path = os.getcwd() + '/'
counter = 0
groups = {}
if args.all or args.stub:
sources = sorted(os.path.abspath(target.path) for target in walk_compile_targets())
else:
sources = []
if args.stub:
groups['=stub'] = dict((source, set()) for source in sources)
for coverage_file in coverage_files:
counter += 1
display.info('[%4d/%4d] %s' % (counter, len(coverage_files), coverage_file), verbosity=2)
original = coverage.CoverageData()
group = get_coverage_group(args, coverage_file)
if group is None:
display.warning('Unexpected name for coverage file: %s' % coverage_file)
continue
if os.path.getsize(coverage_file) == 0:
display.warning('Empty coverage file: %s' % coverage_file)
continue
try:
original.read_file(coverage_file)
except Exception as ex: # pylint: disable=locally-disabled, broad-except
display.error(str(ex))
continue
for filename in original.measured_files():
arcs = set(original.arcs(filename) or [])
if not arcs:
# This is most likely due to using an unsupported version of coverage.
display.warning('No arcs found for "%s" in coverage file: %s' % (filename, coverage_file))
continue
if '/ansible_modlib.zip/ansible/' in filename:
new_name = re.sub('^.*/ansible_modlib.zip/ansible/', ansible_path, filename)
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
elif '/ansible_module_' in filename:
module_name = re.sub('^.*/ansible_module_(?P<module>.*).py$', '\\g<module>', filename)
if module_name not in modules:
display.warning('Skipping coverage of unknown module: %s' % module_name)
continue
new_name = os.path.abspath(modules[module_name])
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
elif re.search('^(/.*?)?/root/ansible/', filename):
new_name = re.sub('^(/.*?)?/root/ansible/', root_path, filename)
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
if group not in groups:
groups[group] = {}
arc_data = groups[group]
if filename not in arc_data:
arc_data[filename] = set()
arc_data[filename].update(arcs)
output_files = []
for group in sorted(groups):
arc_data = groups[group]
updated = coverage.CoverageData()
for filename in arc_data:
if not os.path.isfile(filename):
display.warning('Invalid coverage path: %s' % filename)
continue
updated.add_arcs({filename: list(arc_data[filename])})
if args.all:
updated.add_arcs(dict((source, []) for source in sources))
if not args.explain:
output_file = COVERAGE_FILE + group
updated.write_file(output_file)
output_files.append(output_file)
return sorted(output_files)
def command_coverage_combine(args):
"""Patch paths in coverage files and merge into a single file.
:type args: CoverageConfig
:rtype: list[str]
"""
coverage = initialize_coverage(args)
modules = dict((t.module, t.path) for t in list(walk_module_targets()))
coverage_files = [os.path.join(COVERAGE_DIR, f) for f in os.listdir(COVERAGE_DIR) if '=coverage.' in f]
ansible_path = os.path.abspath('lib/ansible/') + '/'
root_path = os.getcwd() + '/'
counter = 0
groups = {}
if args.all or args.stub:
sources = sorted(os.path.abspath(target.path) for target in walk_compile_targets())
else:
sources = []
if args.stub:
groups['=stub'] = dict((source, set()) for source in sources)
for coverage_file in coverage_files:
counter += 1
display.info('[%4d/%4d] %s' % (counter, len(coverage_files), coverage_file), verbosity=2)
original = coverage.CoverageData()
group = get_coverage_group(args, coverage_file)
if group is None:
display.warning('Unexpected name for coverage file: %s' % coverage_file)
continue
if os.path.getsize(coverage_file) == 0:
display.warning('Empty coverage file: %s' % coverage_file)
continue
try:
original.read_file(coverage_file)
except Exception as ex: # pylint: disable=locally-disabled, broad-except
display.error(str(ex))
continue
for filename in original.measured_files():
arcs = set(original.arcs(filename) or [])
if not arcs:
# This is most likely due to using an unsupported version of coverage.
display.warning('No arcs found for "%s" in coverage file: %s' % (filename, coverage_file))
continue
if '/ansible_modlib.zip/ansible/' in filename:
new_name = re.sub('^.*/ansible_modlib.zip/ansible/', ansible_path, filename)
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
elif '/ansible_module_' in filename:
module = re.sub('^.*/ansible_module_(?P<module>.*).py$', '\\g<module>', filename)
if module not in modules:
display.warning('Skipping coverage of unknown module: %s' % module)
continue
new_name = os.path.abspath(modules[module])
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
elif re.search('^(/.*?)?/root/ansible/', filename):
new_name = re.sub('^(/.*?)?/root/ansible/', root_path, filename)
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
if group not in groups:
groups[group] = {}
arc_data = groups[group]
if filename not in arc_data:
arc_data[filename] = set()
arc_data[filename].update(arcs)
output_files = []
for group in sorted(groups):
arc_data = groups[group]
updated = coverage.CoverageData()
for filename in arc_data:
if not os.path.isfile(filename):
display.warning('Invalid coverage path: %s' % filename)
continue
updated.add_arcs({filename: list(arc_data[filename])})
if args.all:
updated.add_arcs(dict((source, []) for source in sources))
if not args.explain:
output_file = COVERAGE_FILE + group
updated.write_file(output_file)
output_files.append(output_file)
return sorted(output_files)
def command_sanity(args):
"""
:type args: SanityConfig
"""
changes = get_changes_filter(args)
require = (args.require or []) + changes
targets = SanityTargets(args.include, args.exclude, require)
if not targets.include:
raise AllTargetsSkipped()
if args.delegate:
raise Delegate(require=changes)
install_command_requirements(args)
tests = sanity_get_tests()
if args.test:
tests = [t for t in tests if t.name in args.test]
else:
disabled = [t.name for t in tests if not t.enabled and not args.allow_disabled]
tests = [t for t in tests if t.enabled or args.allow_disabled]
if disabled:
display.warning('Skipping tests disabled by default without --allow-disabled: %s' % ', '.join(sorted(disabled)))
if args.skip_test:
tests = [t for t in tests if t.name not in args.skip_test]
total = 0
failed = []
for test in tests:
if args.list_tests:
display.info(test.name)
continue
if isinstance(test, SanityMultipleVersion):
versions = SUPPORTED_PYTHON_VERSIONS
else:
versions = (None,)
for version in versions:
if args.python and version and version != args.python_version:
continue
display.info('Sanity check using %s%s' % (test.name, ' with Python %s' % version if version else ''))
options = ''
if isinstance(test, SanityCodeSmellTest):
result = test.test(args, targets)
elif isinstance(test, SanityMultipleVersion):
result = test.test(args, targets, python_version=version)
options = ' --python %s' % version
elif isinstance(test, SanitySingleVersion):
result = test.test(args, targets)
else:
raise Exception('Unsupported test type: %s' % type(test))
result.write(args)
total += 1
if isinstance(result, SanityFailure):
failed.append(result.test + options)
if failed:
message = 'The %d sanity test(s) listed below (out of %d) failed. See error output above for details.\n%s' % (
len(failed), total, '\n'.join(failed))
if args.failure_ok:
display.error(message)
else:
raise ApplicationError(message)
def command_integration_filtered(args, targets, all_targets):
"""
:type args: IntegrationConfig
:type targets: tuple[IntegrationTarget]
:type all_targets: tuple[IntegrationTarget]
"""
found = False
passed = []
failed = []
targets_iter = iter(targets)
all_targets_dict = dict((target.name, target) for target in all_targets)
setup_errors = []
setup_targets_executed = set()
for target in all_targets:
for setup_target in target.setup_once + target.setup_always:
if setup_target not in all_targets_dict:
setup_errors.append('Target "%s" contains invalid setup target: %s' % (target.name, setup_target))
if setup_errors:
raise ApplicationError('Found %d invalid setup aliases:\n%s' % (len(setup_errors), '\n'.join(setup_errors)))
test_dir = os.path.expanduser('~/ansible_testing')
if not args.explain and any('needs/ssh/' in target.aliases for target in targets):
max_tries = 20
display.info('SSH service required for tests. Checking to make sure we can connect.')
for i in range(1, max_tries + 1):
try:
run_command(args, ['ssh', '-o', 'BatchMode=yes', 'localhost', 'id'], capture=True)
display.info('SSH service responded.')
break
except SubprocessError:
if i == max_tries:
raise
seconds = 3
display.warning('SSH service not responding. Waiting %d second(s) before checking again.' % seconds)
time.sleep(seconds)
start_at_task = args.start_at_task
results = {}
for target in targets_iter:
if args.start_at and not found:
found = target.name == args.start_at
if not found:
continue
if args.list_targets:
print(target.name)
continue
tries = 2 if args.retry_on_error else 1
verbosity = args.verbosity
cloud_environment = get_cloud_environment(args, target)
original_environment = EnvironmentDescription(args)
display.info('>>> Environment Description\n%s' % original_environment, verbosity=3)
try:
while tries:
tries -= 1
try:
run_setup_targets(args, test_dir, target.setup_once, all_targets_dict, setup_targets_executed, False)
start_time = time.time()
run_setup_targets(args, test_dir, target.setup_always, all_targets_dict, setup_targets_executed, True)
if not args.explain:
# create a fresh test directory for each test target
remove_tree(test_dir)
make_dirs(test_dir)
if target.script_path:
command_integration_script(args, target)
else:
command_integration_role(args, target, start_at_task)
start_at_task = None
end_time = time.time()
results[target.name] = dict(
name=target.name,
type=target.type,
aliases=target.aliases,
modules=target.modules,
run_time_seconds=int(end_time - start_time),
setup_once=target.setup_once,
setup_always=target.setup_always,
coverage=args.coverage,
coverage_label=args.coverage_label,
python_version=args.python_version,
)
break
except SubprocessError:
if cloud_environment:
cloud_environment.on_failure(target, tries)
if not original_environment.validate(target.name, throw=False):
raise
if not tries:
raise
display.warning('Retrying test target "%s" with maximum verbosity.' % target.name)
display.verbosity = args.verbosity = 6
original_environment.validate(target.name, throw=True)
passed.append(target)
except Exception as ex:
failed.append(target)
if args.continue_on_error:
display.error(ex)
continue
display.notice('To resume at this test target, use the option: --start-at %s' % target.name)
next_target = next(targets_iter, None)
if next_target:
display.notice('To resume after this test target, use the option: --start-at %s' % next_target.name)
raise
finally:
display.verbosity = args.verbosity = verbosity
if not args.explain:
results_path = 'test/results/data/%s-%s.json' % (args.command, re.sub(r'[^0-9]', '-', str(datetime.datetime.utcnow().replace(microsecond=0))))
data = dict(
targets=results,
)
with open(results_path, 'w') as results_fd:
results_fd.write(json.dumps(data, sort_keys=True, indent=4))
if failed:
raise ApplicationError('The %d integration test(s) listed below (out of %d) failed. See error output above for details:\n%s' % (
len(failed), len(passed) + len(failed), '\n'.join(target.name for target in failed)))
def command_sanity(args):
"""
:type args: SanityConfig
"""
changes = get_changes_filter(args)
require = (args.require or []) + changes
targets = SanityTargets(args.include, args.exclude, require)
if not targets.include:
raise AllTargetsSkipped()
if args.delegate:
raise Delegate(require=changes)
install_command_requirements(args)
tests = sanity_get_tests()
if args.test:
tests = [t for t in tests if t.name in args.test]
else:
disabled = [
t.name for t in tests if not t.enabled and not args.allow_disabled
]
tests = [t for t in tests if t.enabled or args.allow_disabled]
if disabled:
display.warning(
'Skipping tests disabled by default without --allow-disabled: %s'
% ', '.join(sorted(disabled)))
if args.skip_test:
tests = [t for t in tests if t.name not in args.skip_test]
total = 0
failed = []
for test in tests:
if args.list_tests:
display.info(test.name)
continue
if isinstance(test, SanityMultipleVersion):
versions = SUPPORTED_PYTHON_VERSIONS
else:
versions = (None, )
for version in versions:
if args.python and version and version != args.python_version:
continue
display.info(
'Sanity check using %s%s' %
(test.name, ' with Python %s' % version if version else ''))
options = ''
if isinstance(test, SanityCodeSmellTest):
result = test.test(args, targets)
elif isinstance(test, SanityMultipleVersion):
result = test.test(args, targets, python_version=version)
options = ' --python %s' % version
elif isinstance(test, SanitySingleVersion):
result = test.test(args, targets)
else:
raise Exception('Unsupported test type: %s' % type(test))
result.write(args)
total += 1
if isinstance(result, SanityFailure):
failed.append(result.test + options)
if failed:
message = 'The %d sanity test(s) listed below (out of %d) failed. See error output above for details.\n%s' % (
len(failed), total, '\n'.join(failed))
if args.failure_ok:
display.error(message)
else:
raise ApplicationError(message)
def command_sanity(args):
"""
:type args: SanityConfig
"""
changes = get_changes_filter(args)
require = args.require + changes
targets = SanityTargets.create(args.include, args.exclude, require)
if not targets.include:
raise AllTargetsSkipped()
if args.delegate:
raise Delegate(require=changes, exclude=args.exclude)
install_command_requirements(args)
tests = sanity_get_tests()
if args.test:
tests = [target for target in tests if target.name in args.test]
else:
disabled = [
target.name for target in tests
if not target.enabled and not args.allow_disabled
]
tests = [
target for target in tests if target.enabled or args.allow_disabled
]
if disabled:
display.warning(
'Skipping tests disabled by default without --allow-disabled: %s'
% ', '.join(sorted(disabled)))
if args.skip_test:
tests = [
target for target in tests if target.name not in args.skip_test
]
total = 0
failed = []
for test in tests:
if args.list_tests:
display.info(test.name)
continue
available_versions = get_available_python_versions(
SUPPORTED_PYTHON_VERSIONS)
if args.python:
# specific version selected
versions = (args.python, )
elif isinstance(test, SanityMultipleVersion):
# try all supported versions for multi-version tests when a specific version has not been selected
versions = test.supported_python_versions
elif not test.supported_python_versions or args.python_version in test.supported_python_versions:
# the test works with any version or the version we're already running
versions = (args.python_version, )
else:
# available versions supported by the test
versions = tuple(
sorted(
set(available_versions)
& set(test.supported_python_versions)))
# use the lowest available version supported by the test or the current version as a fallback (which will be skipped)
versions = versions[:1] or (args.python_version, )
for version in versions:
if isinstance(test, SanityMultipleVersion):
skip_version = version
else:
skip_version = None
options = ''
if test.supported_python_versions and version not in test.supported_python_versions:
display.warning(
"Skipping sanity test '%s' on unsupported Python %s." %
(test.name, version))
result = SanitySkipped(test.name, skip_version)
elif not args.python and version not in available_versions:
display.warning(
"Skipping sanity test '%s' on Python %s due to missing interpreter."
% (test.name, version))
result = SanitySkipped(test.name, skip_version)
else:
check_pyyaml(args, version)
if test.supported_python_versions:
display.info("Running sanity test '%s' with Python %s" %
(test.name, version))
else:
display.info("Running sanity test '%s'" % test.name)
if isinstance(test, SanityCodeSmellTest):
settings = test.load_processor(args)
elif isinstance(test, SanityMultipleVersion):
settings = test.load_processor(args, version)
elif isinstance(test, SanitySingleVersion):
settings = test.load_processor(args)
elif isinstance(test, SanityVersionNeutral):
settings = test.load_processor(args)
else:
raise Exception('Unsupported test type: %s' % type(test))
if test.all_targets:
usable_targets = targets.targets
elif test.no_targets:
usable_targets = tuple()
else:
usable_targets = targets.include
if test.include_directories:
usable_targets += tuple(
TestTarget(path, None, None, '')
for path in paths_to_dirs(
[target.path for target in usable_targets]))
usable_targets = sorted(
test.filter_targets(list(usable_targets)))
usable_targets = settings.filter_skipped_targets(
usable_targets)
sanity_targets = SanityTargets(targets.targets,
tuple(usable_targets))
if usable_targets or test.no_targets:
if isinstance(test, SanityCodeSmellTest):
result = test.test(args, sanity_targets, version)
elif isinstance(test, SanityMultipleVersion):
result = test.test(args, sanity_targets, version)
options = ' --python %s' % version
elif isinstance(test, SanitySingleVersion):
result = test.test(args, sanity_targets, version)
elif isinstance(test, SanityVersionNeutral):
result = test.test(args, sanity_targets)
else:
raise Exception('Unsupported test type: %s' %
type(test))
else:
result = SanitySkipped(test.name, skip_version)
result.write(args)
total += 1
if isinstance(result, SanityFailure):
failed.append(result.test + options)
if failed:
message = 'The %d sanity test(s) listed below (out of %d) failed. See error output above for details.\n%s' % (
len(failed), total, '\n'.join(failed))
if args.failure_ok:
display.error(message)
else:
raise ApplicationError(message)
