def task_two_three(packages, koji_build, artifact):
'''Check whether given rpms depends on Python 2 and 3 at the same time'''
# libtaskotron is not available on Python 3, so we do it inside
# to make the above function testable anyway
from libtaskotron import check
outcome = 'PASSED'
bads = {}
for package in packages:
log.debug('Checking {}'.format(package.filename))
name, py_versions = check_two_three(package)
if name in WHITELIST:
log.warn('{} is excluded from this check'.format(name))
elif len(py_versions) == 0:
log.info('{} does not require Python, that\'s OK'.format(
package.filename))
elif len(py_versions) == 1:
py_version = next(iter(py_versions))
log.info('{} requires Python {} only, that\'s OK'.format(
package.filename, py_version))
else:
log.error('{} requires both Python 2 and 3, that\'s usually bad. '
'Python 2 dragged by {}. '
'Python 3 dragged by {}.'.format(package.filename,
py_versions[2],
py_versions[3]))
outcome = 'FAILED'
bads[package.filename] = py_versions
detail = check.CheckDetail(checkname='two_three',
item=koji_build,
report_type=check.ReportType.KOJI_BUILD,
outcome=outcome)
if bads:
detail.artifact = artifact
rpms = ''
for name, py_versions in bads.items():
rpms += ('{}\n'
' * Python 2 dependency: {}\n'
' * Python 3 dependecny: {}\n'.format(
name, py_versions[2], py_versions[3]))
write_to_artifact(artifact, MESSAGE.format(rpms), INFO_URL)
names = ', '.join(str(k) for k in bads.keys())
problems = 'Problematic RPMs:\n' + names
else:
problems = 'No problems found.'
summary = 'subcheck two_three {} for {}. {}'.format(
outcome, koji_build, problems)
log.info(summary)
return detail
def run(koji_build, workdir='.', artifactsdir='artifacts'):
'''The main method to run from Taskotron'''
workdir = os.path.abspath(workdir)
# find files to run on
files = sorted(os.listdir(workdir))
packages = []
srpm_packages = []
for file_ in files:
path = os.path.join(workdir, file_)
if file_.endswith('.rpm'):
try:
package = Package(path)
except PackageException as err:
log.error('{}: {}'.format(file_, err))
else:
if package.is_srpm:
srpm_packages.append(package)
else:
packages.append(package)
else:
log.debug('Ignoring non-rpm file: {}'.format(path))
if not packages:
log.warn('No binary rpm files found')
artifact = os.path.join(artifactsdir, 'output.log')
# put all the details form subtask in this list
details = []
details.append(task_two_three(packages, koji_build, artifact))
details.append(task_naming_scheme(packages, koji_build, artifact))
details.append(
task_requires_naming_scheme(srpm_packages + packages, koji_build,
artifact))
# finally, the main detail with overall results
outcome = 'PASSED'
for detail in details:
if detail.outcome == 'FAILED':
outcome = 'FAILED'
break
details.append(
check.CheckDetail(checkname='python-versions',
item=koji_build,
report_type=check.ReportType.KOJI_BUILD,
outcome=outcome))
if outcome == 'FAILED':
details[-1].artifact = artifact
summary = 'python-versions {} for {}.'.format(outcome, koji_build)
log.info(summary)
output = check.export_YAML(details)
return output
def massage_results(results, checkname, item, item_type, log_path):
for test in results['tests']:
outcome = status_lookup.get(test['status'], 'NEEDS_INSPECTION')
note = test['fail_reason']
if note == "None":
note = "No issues found"
output = [test['whiteboard']]
detail = check.CheckDetail(
item=item,
report_type=item_type,
outcome=outcome,
note=note,
output=output,
checkname='%s.%s' % (checkname, sanitize(test['id'])),
artifact=log_path,
)
# print some summary to console
log.info('%s %s for %s (%s)' % (
detail.checkname, detail.outcome, detail.item, detail.note))
yield detail
# Finally, create a wrapper-level CheckResult and tack it on the end.
if results['errors'] or results['failures']:
outcome = 'NEEDS_INSPECTION'
note = "%i issues found" % (results['errors'] + results['failures'])
else:
outcome = 'PASSED'
note = "%i checks passed" % results['pass']
yield check.CheckDetail(
item=item,
report_type=item_type,
outcome=outcome,
note=note,
checkname=checkname,
artifact=log_path,
)
def task_naming_scheme(packages, koji_build, artifact):
"""Check if the given packages are named according
to Python package naming guidelines.
"""
# libtaskotron is not available on Python 3, so we do it inside
# to make the above functions testable anyway
from libtaskotron import check
outcome = 'PASSED'
incorrect_names = set()
name_by_version = collections.defaultdict(set)
for package in packages:
for version in package.py_versions:
name_by_version[version].add(package.name)
for package in packages:
log.debug('Checking {}'.format(package.filename))
if 2 not in package.py_versions:
log.info('{} does not require Python 2, '
'skipping name check'.format(package.filename))
continue
misnamed = check_naming_policy(package, name_by_version)
if misnamed:
log.error(
'{} violates the new Python package'
' naming guidelines'.format(package.filename))
outcome = 'FAILED'
incorrect_names.add(package.nvr)
else:
log.info('{} is using a correct naming scheme'.format(
package.filename))
detail = check.CheckDetail(
checkname='naming_scheme',
item=koji_build,
report_type=check.ReportType.KOJI_BUILD,
outcome=outcome)
if incorrect_names:
detail.artifact = artifact
names = ', '.join(incorrect_names)
write_to_artifact(artifact, MESSAGE.format(names), INFO_URL)
problems = 'Problematic RPMs:\n' + names
else:
problems = 'No problems found.'
summary = 'subcheck naming_scheme {} for {}. {}'.format(
outcome, koji_build, problems)
log.info(summary)
return detail
def task_executables(packages, koji_build, artifact):
"""Check that if there are any executables in Python 2 packages,
there should be executables in Python 3 packages as well.
"""
# libtaskotron is not available on Python 3, so we do it inside
# to make the above functions testable anyway
from libtaskotron import check
outcome = 'PASSED'
message = ''
pkg_by_version = collections.defaultdict(list)
for package in packages:
for version in package.py_versions:
pkg_by_version[version].append(package)
py2_packages = pkg_by_version[2]
py3_packages = pkg_by_version[3]
if koji_build.startswith(WHITELIST):
log.warn('This package is excluded from executables check')
elif not py2_packages or not py3_packages:
log.info('The package is not built for both Python 2 and Python 3. '
'Skipping executables check')
elif len(py2_packages) > len(py3_packages):
log.info('The package is not fully ported to Python 3. '
'Skipping executables check')
elif have_binaries(py2_packages) and not have_binaries(py3_packages):
outcome = 'FAILED'
for package, bins in get_binaries(py2_packages).items():
log.error('{} contains executables which are missing in '
'the Python 3 version of this package'.format(package))
message += '\n{}:\n * {}'.format(package,
'\n * '.join(sorted(bins)))
detail = check.CheckDetail(checkname='python-versions.executables',
item=koji_build,
report_type=check.ReportType.KOJI_BUILD,
outcome=outcome)
if message:
detail.artifact = artifact
write_to_artifact(artifact, MESSAGE.format(message), INFO_URL)
log.info('python-versions.executables {} for {}'.format(
outcome, koji_build))
return detail
def task_requires_naming_scheme(packages, koji_build, artifact):
"""Check if the given packages use names with `python-` prefix
without a version in Requires.
"""
# libtaskotron is not available on Python 3, so we do it inside
# to make the above functions testable anyway
from libtaskotron import check
fedora_release = koji_build.split('fc')[-1]
repoquery = DNFQuery(fedora_release)
outcome = 'PASSED'
problem_rpms = set()
message_rpms = ''
for package in packages:
log.debug('Checking requires of {}'.format(package.filename))
requires = check_requires_naming_scheme(package, repoquery)
if requires:
outcome = 'FAILED'
problem_rpms.add(package.nvr)
message = '\n{} {}Requires:\n * {}\n'.format(
package.nvr,
'Build' if package.is_srpm else '',
'\n * '.join(sorted(requires)))
if message not in message_rpms:
message_rpms += message
detail = check.CheckDetail(
checkname='requires_naming_scheme',
item=koji_build,
report_type=check.ReportType.KOJI_BUILD,
outcome=outcome)
if problem_rpms:
detail.artifact = artifact
write_to_artifact(artifact, MESSAGE.format(message_rpms), INFO_URL)
problems = 'Problematic RPMs:\n' + ', '.join(problem_rpms)
else:
problems = 'No problems found.'
summary = 'subcheck requires_naming_scheme {} for {}. {}'.format(
outcome, koji_build, problems)
log.info(summary)
return detail
def task_python_usage(packages, koji_build, artifact):
"""Check if the packages depend on /usr/bin/python.
"""
# libtaskotron is not available on Python 3, so we do it inside
# to make the above functions testable anyway
from libtaskotron import check
outcome = 'PASSED'
problem_rpms = set()
for package in packages:
log.debug('Checking {}'.format(package.filename))
for name in package.require_names:
name = name.decode()
if name in PYTHON_COMMAND:
log.error('{} requires {}'.format(package.filename, name))
problem_rpms.add(package.filename)
outcome = 'FAILED'
detail = check.CheckDetail(checkname='python_usage',
item=koji_build,
report_type=check.ReportType.KOJI_BUILD,
outcome=outcome)
if problem_rpms:
detail.artifact = artifact
write_to_artifact(artifact,
MESSAGE.format('\n * '.join(problem_rpms)),
INFO_URL)
problems = 'Problematic RPMs:\n' + ', '.join(problem_rpms)
else:
problems = 'No problems found.'
summary = 'subcheck python_usage {} for {}. {}'.format(
outcome, koji_build, problems)
log.info(summary)
return detail
def task_unversioned_shebangs(packages, koji_build, artifact):
"""Check if some of the binaries contains '/usr/bin/python'
shebang or '/usr/bin/env python' shebang.
"""
# libtaskotron is not available on Python 3, so we do it inside
# to make the above functions testable anyway
from libtaskotron import check
outcome = 'PASSED'
problem_rpms = {}
shebang_message = ''
for package in packages:
log.debug('Checking shebangs of {}'.format(package.filename))
problem_rpms[package.nvr] = get_scripts_summary(package)
for package, pkg_summary in problem_rpms.items():
for shebang, scripts in pkg_summary.items():
outcome = 'FAILED'
shebang_message += \
'{}\n * Scripts containing `{}` shebang:\n {}'.format(
package, shebang, '\n '.join(sorted(scripts)))
detail = check.CheckDetail(
checkname='python-versions.unversioned_shebangs',
item=koji_build,
report_type=check.ReportType.KOJI_BUILD,
outcome=outcome)
if outcome == 'FAILED':
detail.artifact = artifact
write_to_artifact(artifact, MESSAGE.format(shebang_message), INFO_URL)
else:
shebang_message = 'No problems found.'
log.info('python-versions.unversioned_shebangs {} for {}. {}'.format(
outcome, koji_build, shebang_message))
return detail
def task_unversioned_shebangs(packages, logs, koji_build, artifact):
"""Check if some of the binaries contain '/usr/bin/python'
shebang or '/usr/bin/env python' shebang or whether those
shebangs were mangled during the build.
"""
# libtaskotron is not available on Python 3, so we do it inside
# to make the above functions testable anyway
from libtaskotron import check
outcome = 'PASSED'
message = ''
problems = ''
problems = check_packages(packages)
if problems:
outcome = 'FAILED'
message = MESSAGE.format(problems)
mangled_on_arches = check_logs(logs)
if mangled_on_arches:
outcome = 'FAILED'
message = MANGLED_MESSAGE.format(mangled_on_arches)
problems = 'Shebangs mangled on: {}'.format(mangled_on_arches)
detail = check.CheckDetail(checkname='unversioned_shebangs',
item=koji_build,
report_type=check.ReportType.KOJI_BUILD,
outcome=outcome)
if outcome == 'FAILED':
detail.artifact = artifact
write_to_artifact(artifact, message, INFO_URL)
else:
problems = 'No problems found.'
log.info('subcheck unversioned_shebangs {} for {}. {}'.format(
outcome, koji_build, problems))
return detail
def task_python_usage(logs, koji_build, artifact):
"""Parses the build.logs for /usr/bin/python invocation warning
"""
# libtaskotron is not available on Python 3, so we do it inside
# to make the above functions testable anyway
from libtaskotron import check
outcome = 'PASSED'
problem_arches = set()
for buildlog in logs: # not "log" because we use that name for logging
log.debug('Will parse {}'.format(buildlog))
if file_contains(buildlog, WARNING):
log.debug('{} contains our warning'.format(buildlog))
_, _, arch = buildlog.rpartition('.')
problem_arches.add(arch)
outcome = 'FAILED'
detail = check.CheckDetail(checkname='python_usage',
item=koji_build,
report_type=check.ReportType.KOJI_BUILD,
outcome=outcome)
if problem_arches:
detail.artifact = artifact
info = '{}: {}'.format(koji_build, ', '.join(sorted(problem_arches)))
write_to_artifact(artifact, MESSAGE.format(info), INFO_URL)
problems = 'Problematic architectures: ' + info
else:
problems = 'No problems found.'
summary = 'subcheck python_usage {} for {}. {}'.format(
outcome, koji_build, problems)
log.info(summary)
return detail
def task_py3_support(packages, koji_build, artifact):
"""Check that the package is packaged for Python 3,
if upstream is Python 3 ready.
Source of data: https://bugzilla.redhat.com/show_bug.cgi?id=1285816
"""
# libtaskotron is not available on Python 3, so we do it inside
# to make the above functions testable anyway
from libtaskotron import check
outcome = 'PASSED'
message = ''
srpm, packages = packages[0], packages[1:]
if not ported_to_py3(packages):
bugzilla_urls = get_py3_bugzillas_for(srpm.name)
if bugzilla_urls:
outcome = 'FAILED'
log.error('This software supports Python 3 upstream,'
' but is not packaged for Python 3 in Fedora')
message = ', '.join(bugzilla_urls)
else:
log.info('This software does not support Python 3'
' upstream, skipping Py3 support check')
detail = check.CheckDetail(checkname='py3_support',
item=koji_build,
report_type=check.ReportType.KOJI_BUILD,
outcome=outcome)
if message:
detail.artifact = artifact
write_to_artifact(artifact, MESSAGE.format(message), INFO_URL)
log.info('subcheck py3_support {} for {}'.format(outcome, koji_build))
return detail
def run(koji_build,
workdir='.',
artifactsdir='artifacts',
testcase='dist.python-versions'):
'''The main method to run from Taskotron'''
workdir = os.path.abspath(workdir)
results_path = os.path.join(artifactsdir, 'taskotron', 'results.yml')
artifact = os.path.join(artifactsdir, 'output.log')
# find files to run on
files = sorted(os.listdir(workdir))
logs = []
packages = []
srpm_packages = []
for file_ in files:
path = os.path.join(workdir, file_)
if file_.endswith('.rpm'):
try:
package = Package(path)
except PackageException as err:
log.error('{}: {}'.format(file_, err))
else:
if package.is_srpm:
srpm_packages.append(package)
else:
packages.append(package)
elif file_.startswith('build.log'): # it's build.log.{arch}
logs.append(path)
else:
log.debug('Ignoring non-rpm, non-build.log file: {}'.format(path))
if not packages:
log.warn('No binary rpm files found')
if not logs:
log.warn('No build.log found, that should not happen')
# put all the details form subtask in this list
details = []
details.append(task_two_three(packages, koji_build, artifact))
details.append(task_naming_scheme(packages, koji_build, artifact))
details.append(
task_requires_naming_scheme(srpm_packages + packages, koji_build,
artifact))
details.append(task_executables(packages, koji_build, artifact))
details.append(
task_unversioned_shebangs(packages, logs, koji_build, artifact))
details.append(
task_py3_support(srpm_packages + packages, koji_build, artifact))
details.append(task_python_usage(logs, koji_build, artifact))
# update testcase for all subtasks (use their existing testcase as a
# suffix)
for detail in details:
detail.checkname = '{}.{}'.format(testcase, detail.checkname)
# finally, the main detail with overall results
outcome = 'PASSED'
for detail in details:
if detail.outcome == 'FAILED':
outcome = 'FAILED'
break
overall_detail = check.CheckDetail(checkname=testcase,
item=koji_build,
report_type=check.ReportType.KOJI_BUILD,
outcome=outcome)
if outcome == 'FAILED':
overall_detail.artifact = artifact
details.append(overall_detail)
summary = 'python-versions {} for {}.'.format(outcome, koji_build)
log.info(summary)
# generate output reportable to ResultsDB
output = check.export_YAML(details)
with open(results_path, 'w') as results_file:
results_file.write(output)
return 0 if overall_detail.outcome in ['PASSED', 'INFO'] else 1
