def task_two_three(packages, koji_build, artifact): '''Check whether given rpms depends on Python 2 and 3 at the same time''' # libtaskotron is not available on Python 3, so we do it inside # to make the above function testable anyway from libtaskotron import check outcome = 'PASSED' bads = {} for package in packages: log.debug('Checking {}'.format(package.filename)) name, py_versions = check_two_three(package) if name in WHITELIST: log.warn('{} is excluded from this check'.format(name)) elif len(py_versions) == 0: log.info('{} does not require Python, that\'s OK'.format( package.filename)) elif len(py_versions) == 1: py_version = next(iter(py_versions)) log.info('{} requires Python {} only, that\'s OK'.format( package.filename, py_version)) else: log.error('{} requires both Python 2 and 3, that\'s usually bad. ' 'Python 2 dragged by {}. ' 'Python 3 dragged by {}.'.format(package.filename, py_versions[2], py_versions[3])) outcome = 'FAILED' bads[package.filename] = py_versions detail = check.CheckDetail(checkname='two_three', item=koji_build, report_type=check.ReportType.KOJI_BUILD, outcome=outcome) if bads: detail.artifact = artifact rpms = '' for name, py_versions in bads.items(): rpms += ('{}\n' ' * Python 2 dependency: {}\n' ' * Python 3 dependecny: {}\n'.format( name, py_versions[2], py_versions[3])) write_to_artifact(artifact, MESSAGE.format(rpms), INFO_URL) names = ', '.join(str(k) for k in bads.keys()) problems = 'Problematic RPMs:\n' + names else: problems = 'No problems found.' summary = 'subcheck two_three {} for {}. {}'.format( outcome, koji_build, problems) log.info(summary) return detail
def run(koji_build, workdir='.', artifactsdir='artifacts'): '''The main method to run from Taskotron''' workdir = os.path.abspath(workdir) # find files to run on files = sorted(os.listdir(workdir)) packages = [] srpm_packages = [] for file_ in files: path = os.path.join(workdir, file_) if file_.endswith('.rpm'): try: package = Package(path) except PackageException as err: log.error('{}: {}'.format(file_, err)) else: if package.is_srpm: srpm_packages.append(package) else: packages.append(package) else: log.debug('Ignoring non-rpm file: {}'.format(path)) if not packages: log.warn('No binary rpm files found') artifact = os.path.join(artifactsdir, 'output.log') # put all the details form subtask in this list details = [] details.append(task_two_three(packages, koji_build, artifact)) details.append(task_naming_scheme(packages, koji_build, artifact)) details.append( task_requires_naming_scheme(srpm_packages + packages, koji_build, artifact)) # finally, the main detail with overall results outcome = 'PASSED' for detail in details: if detail.outcome == 'FAILED': outcome = 'FAILED' break details.append( check.CheckDetail(checkname='python-versions', item=koji_build, report_type=check.ReportType.KOJI_BUILD, outcome=outcome)) if outcome == 'FAILED': details[-1].artifact = artifact summary = 'python-versions {} for {}.'.format(outcome, koji_build) log.info(summary) output = check.export_YAML(details) return output
def massage_results(results, checkname, item, item_type, log_path): for test in results['tests']: outcome = status_lookup.get(test['status'], 'NEEDS_INSPECTION') note = test['fail_reason'] if note == "None": note = "No issues found" output = [test['whiteboard']] detail = check.CheckDetail( item=item, report_type=item_type, outcome=outcome, note=note, output=output, checkname='%s.%s' % (checkname, sanitize(test['id'])), artifact=log_path, ) # print some summary to console log.info('%s %s for %s (%s)' % ( detail.checkname, detail.outcome, detail.item, detail.note)) yield detail # Finally, create a wrapper-level CheckResult and tack it on the end. if results['errors'] or results['failures']: outcome = 'NEEDS_INSPECTION' note = "%i issues found" % (results['errors'] + results['failures']) else: outcome = 'PASSED' note = "%i checks passed" % results['pass'] yield check.CheckDetail( item=item, report_type=item_type, outcome=outcome, note=note, checkname=checkname, artifact=log_path, )
def task_naming_scheme(packages, koji_build, artifact): """Check if the given packages are named according to Python package naming guidelines. """ # libtaskotron is not available on Python 3, so we do it inside # to make the above functions testable anyway from libtaskotron import check outcome = 'PASSED' incorrect_names = set() name_by_version = collections.defaultdict(set) for package in packages: for version in package.py_versions: name_by_version[version].add(package.name) for package in packages: log.debug('Checking {}'.format(package.filename)) if 2 not in package.py_versions: log.info('{} does not require Python 2, ' 'skipping name check'.format(package.filename)) continue misnamed = check_naming_policy(package, name_by_version) if misnamed: log.error( '{} violates the new Python package' ' naming guidelines'.format(package.filename)) outcome = 'FAILED' incorrect_names.add(package.nvr) else: log.info('{} is using a correct naming scheme'.format( package.filename)) detail = check.CheckDetail( checkname='naming_scheme', item=koji_build, report_type=check.ReportType.KOJI_BUILD, outcome=outcome) if incorrect_names: detail.artifact = artifact names = ', '.join(incorrect_names) write_to_artifact(artifact, MESSAGE.format(names), INFO_URL) problems = 'Problematic RPMs:\n' + names else: problems = 'No problems found.' summary = 'subcheck naming_scheme {} for {}. {}'.format( outcome, koji_build, problems) log.info(summary) return detail
def task_executables(packages, koji_build, artifact): """Check that if there are any executables in Python 2 packages, there should be executables in Python 3 packages as well. """ # libtaskotron is not available on Python 3, so we do it inside # to make the above functions testable anyway from libtaskotron import check outcome = 'PASSED' message = '' pkg_by_version = collections.defaultdict(list) for package in packages: for version in package.py_versions: pkg_by_version[version].append(package) py2_packages = pkg_by_version[2] py3_packages = pkg_by_version[3] if koji_build.startswith(WHITELIST): log.warn('This package is excluded from executables check') elif not py2_packages or not py3_packages: log.info('The package is not built for both Python 2 and Python 3. ' 'Skipping executables check') elif len(py2_packages) > len(py3_packages): log.info('The package is not fully ported to Python 3. ' 'Skipping executables check') elif have_binaries(py2_packages) and not have_binaries(py3_packages): outcome = 'FAILED' for package, bins in get_binaries(py2_packages).items(): log.error('{} contains executables which are missing in ' 'the Python 3 version of this package'.format(package)) message += '\n{}:\n * {}'.format(package, '\n * '.join(sorted(bins))) detail = check.CheckDetail(checkname='python-versions.executables', item=koji_build, report_type=check.ReportType.KOJI_BUILD, outcome=outcome) if message: detail.artifact = artifact write_to_artifact(artifact, MESSAGE.format(message), INFO_URL) log.info('python-versions.executables {} for {}'.format( outcome, koji_build)) return detail
def task_requires_naming_scheme(packages, koji_build, artifact): """Check if the given packages use names with `python-` prefix without a version in Requires. """ # libtaskotron is not available on Python 3, so we do it inside # to make the above functions testable anyway from libtaskotron import check fedora_release = koji_build.split('fc')[-1] repoquery = DNFQuery(fedora_release) outcome = 'PASSED' problem_rpms = set() message_rpms = '' for package in packages: log.debug('Checking requires of {}'.format(package.filename)) requires = check_requires_naming_scheme(package, repoquery) if requires: outcome = 'FAILED' problem_rpms.add(package.nvr) message = '\n{} {}Requires:\n * {}\n'.format( package.nvr, 'Build' if package.is_srpm else '', '\n * '.join(sorted(requires))) if message not in message_rpms: message_rpms += message detail = check.CheckDetail( checkname='requires_naming_scheme', item=koji_build, report_type=check.ReportType.KOJI_BUILD, outcome=outcome) if problem_rpms: detail.artifact = artifact write_to_artifact(artifact, MESSAGE.format(message_rpms), INFO_URL) problems = 'Problematic RPMs:\n' + ', '.join(problem_rpms) else: problems = 'No problems found.' summary = 'subcheck requires_naming_scheme {} for {}. {}'.format( outcome, koji_build, problems) log.info(summary) return detail
def task_python_usage(packages, koji_build, artifact): """Check if the packages depend on /usr/bin/python. """ # libtaskotron is not available on Python 3, so we do it inside # to make the above functions testable anyway from libtaskotron import check outcome = 'PASSED' problem_rpms = set() for package in packages: log.debug('Checking {}'.format(package.filename)) for name in package.require_names: name = name.decode() if name in PYTHON_COMMAND: log.error('{} requires {}'.format(package.filename, name)) problem_rpms.add(package.filename) outcome = 'FAILED' detail = check.CheckDetail(checkname='python_usage', item=koji_build, report_type=check.ReportType.KOJI_BUILD, outcome=outcome) if problem_rpms: detail.artifact = artifact write_to_artifact(artifact, MESSAGE.format('\n * '.join(problem_rpms)), INFO_URL) problems = 'Problematic RPMs:\n' + ', '.join(problem_rpms) else: problems = 'No problems found.' summary = 'subcheck python_usage {} for {}. {}'.format( outcome, koji_build, problems) log.info(summary) return detail
def task_unversioned_shebangs(packages, koji_build, artifact): """Check if some of the binaries contains '/usr/bin/python' shebang or '/usr/bin/env python' shebang. """ # libtaskotron is not available on Python 3, so we do it inside # to make the above functions testable anyway from libtaskotron import check outcome = 'PASSED' problem_rpms = {} shebang_message = '' for package in packages: log.debug('Checking shebangs of {}'.format(package.filename)) problem_rpms[package.nvr] = get_scripts_summary(package) for package, pkg_summary in problem_rpms.items(): for shebang, scripts in pkg_summary.items(): outcome = 'FAILED' shebang_message += \ '{}\n * Scripts containing `{}` shebang:\n {}'.format( package, shebang, '\n '.join(sorted(scripts))) detail = check.CheckDetail( checkname='python-versions.unversioned_shebangs', item=koji_build, report_type=check.ReportType.KOJI_BUILD, outcome=outcome) if outcome == 'FAILED': detail.artifact = artifact write_to_artifact(artifact, MESSAGE.format(shebang_message), INFO_URL) else: shebang_message = 'No problems found.' log.info('python-versions.unversioned_shebangs {} for {}. {}'.format( outcome, koji_build, shebang_message)) return detail
def task_unversioned_shebangs(packages, logs, koji_build, artifact): """Check if some of the binaries contain '/usr/bin/python' shebang or '/usr/bin/env python' shebang or whether those shebangs were mangled during the build. """ # libtaskotron is not available on Python 3, so we do it inside # to make the above functions testable anyway from libtaskotron import check outcome = 'PASSED' message = '' problems = '' problems = check_packages(packages) if problems: outcome = 'FAILED' message = MESSAGE.format(problems) mangled_on_arches = check_logs(logs) if mangled_on_arches: outcome = 'FAILED' message = MANGLED_MESSAGE.format(mangled_on_arches) problems = 'Shebangs mangled on: {}'.format(mangled_on_arches) detail = check.CheckDetail(checkname='unversioned_shebangs', item=koji_build, report_type=check.ReportType.KOJI_BUILD, outcome=outcome) if outcome == 'FAILED': detail.artifact = artifact write_to_artifact(artifact, message, INFO_URL) else: problems = 'No problems found.' log.info('subcheck unversioned_shebangs {} for {}. {}'.format( outcome, koji_build, problems)) return detail
def task_python_usage(logs, koji_build, artifact): """Parses the build.logs for /usr/bin/python invocation warning """ # libtaskotron is not available on Python 3, so we do it inside # to make the above functions testable anyway from libtaskotron import check outcome = 'PASSED' problem_arches = set() for buildlog in logs: # not "log" because we use that name for logging log.debug('Will parse {}'.format(buildlog)) if file_contains(buildlog, WARNING): log.debug('{} contains our warning'.format(buildlog)) _, _, arch = buildlog.rpartition('.') problem_arches.add(arch) outcome = 'FAILED' detail = check.CheckDetail(checkname='python_usage', item=koji_build, report_type=check.ReportType.KOJI_BUILD, outcome=outcome) if problem_arches: detail.artifact = artifact info = '{}: {}'.format(koji_build, ', '.join(sorted(problem_arches))) write_to_artifact(artifact, MESSAGE.format(info), INFO_URL) problems = 'Problematic architectures: ' + info else: problems = 'No problems found.' summary = 'subcheck python_usage {} for {}. {}'.format( outcome, koji_build, problems) log.info(summary) return detail
def task_py3_support(packages, koji_build, artifact): """Check that the package is packaged for Python 3, if upstream is Python 3 ready. Source of data: https://bugzilla.redhat.com/show_bug.cgi?id=1285816 """ # libtaskotron is not available on Python 3, so we do it inside # to make the above functions testable anyway from libtaskotron import check outcome = 'PASSED' message = '' srpm, packages = packages[0], packages[1:] if not ported_to_py3(packages): bugzilla_urls = get_py3_bugzillas_for(srpm.name) if bugzilla_urls: outcome = 'FAILED' log.error('This software supports Python 3 upstream,' ' but is not packaged for Python 3 in Fedora') message = ', '.join(bugzilla_urls) else: log.info('This software does not support Python 3' ' upstream, skipping Py3 support check') detail = check.CheckDetail(checkname='py3_support', item=koji_build, report_type=check.ReportType.KOJI_BUILD, outcome=outcome) if message: detail.artifact = artifact write_to_artifact(artifact, MESSAGE.format(message), INFO_URL) log.info('subcheck py3_support {} for {}'.format(outcome, koji_build)) return detail
def run(koji_build, workdir='.', artifactsdir='artifacts', testcase='dist.python-versions'): '''The main method to run from Taskotron''' workdir = os.path.abspath(workdir) results_path = os.path.join(artifactsdir, 'taskotron', 'results.yml') artifact = os.path.join(artifactsdir, 'output.log') # find files to run on files = sorted(os.listdir(workdir)) logs = [] packages = [] srpm_packages = [] for file_ in files: path = os.path.join(workdir, file_) if file_.endswith('.rpm'): try: package = Package(path) except PackageException as err: log.error('{}: {}'.format(file_, err)) else: if package.is_srpm: srpm_packages.append(package) else: packages.append(package) elif file_.startswith('build.log'): # it's build.log.{arch} logs.append(path) else: log.debug('Ignoring non-rpm, non-build.log file: {}'.format(path)) if not packages: log.warn('No binary rpm files found') if not logs: log.warn('No build.log found, that should not happen') # put all the details form subtask in this list details = [] details.append(task_two_three(packages, koji_build, artifact)) details.append(task_naming_scheme(packages, koji_build, artifact)) details.append( task_requires_naming_scheme(srpm_packages + packages, koji_build, artifact)) details.append(task_executables(packages, koji_build, artifact)) details.append( task_unversioned_shebangs(packages, logs, koji_build, artifact)) details.append( task_py3_support(srpm_packages + packages, koji_build, artifact)) details.append(task_python_usage(logs, koji_build, artifact)) # update testcase for all subtasks (use their existing testcase as a # suffix) for detail in details: detail.checkname = '{}.{}'.format(testcase, detail.checkname) # finally, the main detail with overall results outcome = 'PASSED' for detail in details: if detail.outcome == 'FAILED': outcome = 'FAILED' break overall_detail = check.CheckDetail(checkname=testcase, item=koji_build, report_type=check.ReportType.KOJI_BUILD, outcome=outcome) if outcome == 'FAILED': overall_detail.artifact = artifact details.append(overall_detail) summary = 'python-versions {} for {}.'.format(outcome, koji_build) log.info(summary) # generate output reportable to ResultsDB output = check.export_YAML(details) with open(results_path, 'w') as results_file: results_file.write(output) return 0 if overall_detail.outcome in ['PASSED', 'INFO'] else 1