def _get_result_from_cache(package_name: str,
badge_type: badge_utils.BadgeType,
commit_number: str = None) -> dict:
"""Get check result from cache."""
# Return unknown if package not in whitelist
if not utils._is_package_in_whitelist([package_name]):
result = badge_utils._build_default_result(badge_type=badge_type,
status='UNKNOWN',
details={})
# Get the result from cache, return None if not in cache
else:
package_key = '{}_{}'.format(
package_name, commit_number) if commit_number else package_name
result = cache.get('{}_{}'.format(package_key, badge_type.value))
if result is None:
result = badge_utils._build_default_result(badge_type=badge_type,
status='CALCULATING',
details={})
return result
def _self_compatibilities_to_dict(
package_name: str,
compatibility_results: List[compatibility_store.CompatibilityResult]
) -> dict:
"""Converts a CompatibilityResult into a dict containing compatibility info.
Args:
package_name: the name of the package to check (e.g.
"google-cloud-storage").
compatibility_results: a (possibly empty) list of compatibility results
representing a self-compatibility check for the given package.
Returns:
A dict containing the self compatibility status and details for any
self incompatibilities. The dict will be formatted like the following:
{
'py2': { 'status': BadgeStatus.SUCCESS, 'details': {} },
'py3': { 'status': BadgeStatus.SUCCESS, 'details': {} },
}
"""
missing_details = _get_missing_details([package_name],
compatibility_results)
if missing_details:
result_dict = badge_utils._build_default_result(
status=BadgeStatus.MISSING_DATA, details=missing_details)
return result_dict
result_dict = badge_utils._build_default_result(
status=BadgeStatus.SUCCESS,
details='The package does not support this version of python.')
for res in compatibility_results:
pyver = badge_utils.PY_VER_MAPPING[res.python_major_version]
badge_status = PACKAGE_STATUS_TO_BADGE_STATUS.get(
res.status) or BadgeStatus.SELF_INCOMPATIBLE
result_dict[pyver]['status'] = badge_status
result_dict[pyver]['details'] = res.details
if res.details is None:
result_dict[pyver]['details'] = badge_utils.EMPTY_DETAILS
return result_dict
def _get_dependency_dict(package_name: str) -> dict:
"""Returns a dict containing outdated dependencies' status and details.
Args:
package_name: the name of the package to get outdated dependencies for
(e.g. "google-cloud-storage").
Returns:
A dict containing the outdated dependency status and details for any
outdated dependencies. Note that details maps to a dict that may be
nested. The returned dict will be formatted like the following:
{
'status': BadgeStatus.OBSOLETE_DEPENDENCY,
'details': {
'google-cloud-bigquery': {
'installed_version': '1.6.1',
'latest_version': '1.10.0',
'priority': 'HIGH_PRIORITY',
'detail': ('google-cloud-bigquery is 3 or more minor'
'versions behind the latest version')
},
},
}
"""
result_dict = badge_utils._build_default_result(status=BadgeStatus.SUCCESS,
include_pyversion=False,
details={})
outdated_deps = badge_utils.highlighter.check_package(package_name)
_deps_list = badge_utils.finder.get_deprecated_dep(package_name)[1]
deprecated_deps = ', '.join(_deps_list)
outdated_depencdency_name_to_details = {}
max_level = badge_utils.priority_level.UP_TO_DATE
for dep in outdated_deps:
dep_detail = {}
level = dep.priority.level
if level.value > max_level.value:
max_level = level
dep_detail['installed_version'] = dep.installed_version
dep_detail['latest_version'] = dep.latest_version
dep_detail['priority'] = dep.priority.level.name
dep_detail['detail'] = dep.priority.details
outdated_depencdency_name_to_details[dep.name] = dep_detail
badge_status = DEPENDENCY_STATUS_TO_BADGE_STATUS[max_level]
result_dict['status'] = badge_status
result_dict['details'] = outdated_depencdency_name_to_details
result_dict['deprecated_deps'] = deprecated_deps
return result_dict
def _get_self_compatibility_dict(package_name: str) -> dict:
"""Returns a dict containing self compatibility status and details.
Args:
package_name: the name of the package to check (e.g.
"google-cloud-storage").
Returns:
A dict containing the self compatibility status and details for any
self incompatibilities. The dict will be formatted like the following:
{
'py2': { 'status': BadgeStatus.SUCCESS, 'details': {} },
'py3': { 'status': BadgeStatus.SUCCESS, 'details': {} },
}
"""
pkg = package.Package(package_name)
compatibility_results = badge_utils.store.get_self_compatibility(pkg)
missing_details = _get_missing_details([package_name],
compatibility_results)
if missing_details:
result_dict = badge_utils._build_default_result(
status=BadgeStatus.MISSING_DATA, details=missing_details)
return result_dict
result_dict = badge_utils._build_default_result(
status=BadgeStatus.SUCCESS,
details='The package does not support this version of python.')
for res in compatibility_results:
pyver = badge_utils.PY_VER_MAPPING[res.python_major_version]
badge_status = PACKAGE_STATUS_TO_BADGE_STATUS.get(
res.status) or BadgeStatus.SELF_INCOMPATIBLE
result_dict[pyver]['status'] = badge_status
result_dict[pyver]['details'] = res.details
if res.details is None:
result_dict[pyver]['details'] = badge_utils.EMPTY_DETAILS
return result_dict
def _get_check_results(package_name: str, commit_number: str = None):
"""Gets the compatibility and dependency check results.
Returns a 3 tuple: self compatibility, pair compatibility, dependency dicts
that are used to generate badge images and badge target pages.
"""
default_status = BadgeStatus.UNKNOWN_PACKAGE
unknown_details = ('This package is not a whitelisted google python '
'package; to whitelist a package, contact the python '
'team.')
self_compat_res = badge_utils._build_default_result(
status=default_status, details=unknown_details)
google_compat_res = badge_utils._build_default_result(
status=default_status, details={})
dependency_res = badge_utils._build_default_result(status=default_status,
include_pyversion=False,
details={})
# If a package is not whitelisted, return defaults
if not compat_utils._is_package_in_whitelist([package_name]):
return (self_compat_res, google_compat_res, dependency_res)
try:
self_compat_res = _get_self_compatibility_dict(package_name)
google_compat_res = _get_pair_compatibility_dict(package_name)
dependency_res = _get_dependency_dict(package_name)
except Exception:
logging.exception(
'Exception checking results for "{}"'.format(package_name))
error_status = BadgeStatus.INTERNAL_ERROR
self_compat_res, google_compat_res, dependency_res = (
badge_utils._build_default_result(status=error_status),
badge_utils._build_default_result(status=error_status, details={}),
badge_utils._build_default_result(status=error_status,
include_pyversion=False))
return (self_compat_res, google_compat_res, dependency_res)
def google_compatibility_badge_image():
"""Badge showing whether a package is compatible with Google OSS Python
packages. If all packages success, status is SUCCESS; else set status
to one of the failure types, details can be found at the target link."""
package_name = flask.request.args.get('package')
force_run_check = flask.request.args.get('force_run_check')
commit_number = flask.request.args.get('commit_number')
badge_name = flask.request.args.get('badge_name')
package_key = '{}_{}'.format(
package_name, commit_number) if commit_number else package_name
if badge_name is None:
badge_name = 'google compatibility'
def run_check():
pkg_sets = [[package_name, pkg] for pkg in configs.PKG_LIST]
if package_name in configs.PKG_LIST:
result = _get_pair_status_for_packages(pkg_sets)
else:
version_and_res = {
'py2': {
'status': 'SUCCESS',
'details': {},
},
'py3': {
'status': 'SUCCESS',
'details': {},
},
'timestamp': '',
}
for py_ver in [2, 3]:
results = list(badge_utils.checker.get_pairwise_compatibility(
py_ver, pkg_sets))
logging.warning(results)
py_version = badge_utils.PY_VER_MAPPING[py_ver]
for res in results:
res_item = res[0]
status = res_item.get('result')
package = res_item.get('packages')[1]
if status != 'SUCCESS':
# Ignore the package that not support for given py_ver
if package in \
configs.PKG_PY_VERSION_NOT_SUPPORTED.get(
py_ver):
continue
# Ignore the package that are not self compatible
self_status = _get_result_from_cache(
package_name=package_name,
badge_type=badge_utils.BadgeType.SELF_COMP_BADGE)
if self_status[py_version]['status'] not in [
'SUCCESS', 'CALCULATING']:
continue
# Status showing one of the check failures
version_and_res[
py_version]['status'] = res_item.get('result')
description = res_item.get('description')
details = badge_utils.EMPTY_DETAILS if description \
is None else description
version_and_res[
py_version]['details'][package] = details
version_and_res['timestamp'] = datetime.datetime.now().strftime(
badge_utils.TIMESTAMP_FORMAT)
result = version_and_res
# Write the result to Cloud Datastore
cache.set('{}_google_comp_badge'.format(package_key), result)
google_comp_res = cache.get('{}_google_comp_badge'.format(package_key))
if not utils._is_package_in_whitelist([package_name]):
google_comp_res = badge_utils._build_default_result(
badge_type=badge_utils.BadgeType.GOOGLE_COMP_BADGE,
status='UNKNOWN',
details={})
if google_comp_res is None:
details = badge_utils._build_default_result(
badge_type=badge_utils.BadgeType.GOOGLE_COMP_BADGE,
status='CALCULATING',
details={})
else:
details = google_comp_res
# Run the check if google_comp_res is None or forced to populate the cache
# or the cache is outdated.
if google_comp_res is None or force_run_check is not None:
threading.Thread(target=run_check).start()
elif google_comp_res is not None:
timestamp = google_comp_res.get('timestamp')
if not badge_utils._is_github_cache_valid(timestamp):
threading.Thread(target=run_check).start()
badge = badge_utils._get_badge(details, badge_name)
response = flask.make_response(badge)
response.content_type = badge_utils.SVG_CONTENT_TYPE
response.headers['Cache-Control'] = 'no-cache'
response.add_etag()
return response
def self_dependency_badge_image():
"""Badge showing whether a package is has outdated dependencies."""
package_name = flask.request.args.get('package')
force_run_check = flask.request.args.get('force_run_check')
commit_number = flask.request.args.get('commit_number')
badge_name = flask.request.args.get('badge_name')
package_key = '{}_{}'.format(
package_name, commit_number) if commit_number else package_name
if badge_name is None:
badge_name = 'dependency status'
def run_check():
res = {
'status': 'UP_TO_DATE',
'details': {},
'timestamp': '',
}
details = {}
outdated = badge_utils.highlighter.check_package(package_name)
deprecated_deps_list = badge_utils.finder.get_deprecated_dep(
package_name)[1]
deprecated_deps = ', '.join(deprecated_deps_list)
max_level = badge_utils.priority_level.UP_TO_DATE
for dep in outdated:
dep_detail = {}
level = dep.priority.level
if level.value > max_level.value:
max_level = level
dep_detail['installed_version'] = dep.installed_version
dep_detail['latest_version'] = dep.latest_version
dep_detail['priority'] = dep.priority.level.name
dep_detail['detail'] = dep.priority.details
details[dep.name] = dep_detail
res['status'] = max_level.name
res['details'] = details
res['deprecated_deps'] = deprecated_deps
res['timestamp'] = datetime.datetime.now().strftime(
badge_utils.TIMESTAMP_FORMAT)
# Write the result to Cloud Datastore
cache.set('{}_dependency_badge'.format(package_key), res)
if not utils._is_package_in_whitelist([package_name]):
dependency_res = badge_utils._build_default_result(
badge_type=badge_utils.BadgeType.DEP_BADGE,
status='UNKNOWN',
details={})
else:
dependency_res = cache.get('{}_dependency_badge'.format(package_key))
if dependency_res is None:
details = badge_utils.DEFAULT_DEPENDENCY_RESULT
else:
details = dependency_res
# Run the check if dependency_res is None or forced to populate the cache
# or the cache is outdated.
if dependency_res is None or force_run_check is not None:
threading.Thread(target=run_check).start()
elif dependency_res is not None:
timestamp = dependency_res.get('timestamp')
if not badge_utils._is_github_cache_valid(timestamp):
threading.Thread(target=run_check).start()
badge = badge_utils._get_badge(details, badge_name)
response = flask.make_response(badge)
response.content_type = badge_utils.SVG_CONTENT_TYPE
response.headers['Cache-Control'] = 'no-cache'
response.add_etag()
return response
def self_compatibility_badge_image():
"""Badge showing whether a package is compatible with itself."""
package_name = flask.request.args.get('package')
force_run_check = flask.request.args.get('force_run_check')
commit_number = flask.request.args.get('commit_number')
badge_name = flask.request.args.get('badge_name')
package_key = '{}_{}'.format(
package_name, commit_number) if commit_number else package_name
if badge_name is None:
badge_name = 'self compatibility'
version_and_res = badge_utils._build_default_result(
badge_type=badge_utils.BadgeType.SELF_COMP_BADGE,
status='CALCULATING',
details=None)
def run_check():
# First see if this package is already stored in BigQuery.
package = package_module.Package(package_name)
compatibility_status = badge_utils.store.get_self_compatibility(
package)
if compatibility_status:
for res in compatibility_status:
py_version = badge_utils.PY_VER_MAPPING[
res.python_major_version]
version_and_res[py_version]['status'] = res.status.value
version_and_res[py_version]['details'] = res.details \
if res.details is not None else badge_utils.EMPTY_DETAILS
# If not pre stored in BigQuery, run the check for the package.
else:
py2_res = badge_utils.checker.check([package_name], '2')
py3_res = badge_utils.checker.check([package_name], '3')
version_and_res['py2']['status'] = py2_res.get('result')
py2_description = py2_res.get('description')
py2_details = badge_utils.EMPTY_DETAILS if py2_description \
is None else py2_description
version_and_res['py2']['details'] = py2_details
version_and_res['py3']['status'] = py3_res.get('result')
py3_description = py3_res.get('description')
py3_details = badge_utils.EMPTY_DETAILS if py3_description \
is None else py3_description
version_and_res['py3']['details'] = py3_details
# Add the timestamp
version_and_res['timestamp'] = datetime.datetime.now().strftime(
badge_utils.TIMESTAMP_FORMAT)
# Write the result to Cloud Datastore
cache.set('{}_self_comp_badge'.format(package_key), version_and_res)
if not utils._is_package_in_whitelist([package_name]):
self_comp_res = badge_utils._build_default_result(
badge_type=badge_utils.BadgeType.SELF_COMP_BADGE,
status='UNKNOWN',
details=badge_utils.PACKAGE_NOT_SUPPORTED)
else:
self_comp_res = cache.get('{}_self_comp_badge'.format(package_key))
if self_comp_res is None:
details = version_and_res
else:
details = self_comp_res
# Run the check if details is None or forced to populate the cache or
# the cache is outdated.
if self_comp_res is None or force_run_check is not None:
threading.Thread(target=run_check).start()
elif self_comp_res is not None:
timestamp = self_comp_res.get('timestamp')
if not badge_utils._is_github_cache_valid(timestamp):
threading.Thread(target=run_check).start()
badge = badge_utils._get_badge(details, badge_name)
response = flask.make_response(badge)
response.content_type = badge_utils.SVG_CONTENT_TYPE
response.headers['Cache-Control'] = 'no-cache'
response.add_etag()
return response
def _get_pair_compatibility_dict(package_name: str) -> dict:
"""Get the pairwise dependency compatibility check result for a package.
Rules:
- Return warning status if not compatible with any of the listed
Google owned Python packages. Whole list in compatibility_lib.config.
- Ignore the warning status if:
- The package doesn't support one of the Python versions.
- The package's pair is not self compatible, which means the
pairwise conflict isn't related to the package being checked.
- Return success status if compatible with all the list of Google owned
Python packages.
Args:
package_name: the name of the package to get pairwise dependencies for
(e.g. "google-cloud-storage").
Returns:
A dict containing the pair compatibility status and details for any
pair incompatibilities. Note that details must map to another dict.
The returned dict will be formatted like the
following:
{
'py2': {'status': BadgeStatus.PAIR_INCOMPATIBLE,
'details': {'apache-beam[gcp]': 'NO DETAILS'}},
'py3': {'status': BadgeStatus.SUCCESS, 'details': {}}
}
"""
default_details = {}
result_dict = badge_utils._build_default_result(status=BadgeStatus.SUCCESS,
details=default_details)
unsupported_package_mapping = configs.PKG_PY_VERSION_NOT_SUPPORTED
pair_mapping = badge_utils.store.get_pairwise_compatibility_for_package(
package_name)
for pair, compatibility_results in pair_mapping.items():
other_package = _get_other_package_from_set(package_name, pair)
other_package_name = other_package.install_name
missing_details = _get_missing_details(
[pkg.install_name for pkg in pair], compatibility_results)
if missing_details:
result_dict = badge_utils._build_default_result(
status=BadgeStatus.MISSING_DATA,
details={other_package_name: missing_details})
return result_dict
for res in compatibility_results:
version = res.python_major_version # eg. '2', '3'
pyver = badge_utils.PY_VER_MAPPING[version] # eg. 'py2', 'py3'
if result_dict[pyver]['details'] is default_details:
result_dict[pyver]['details'] = {}
# Not all packages are supported in both Python 2 and Python 3. If
# either package is not supported in the Python version being
# checked then skip the check.
unsupported_packages = unsupported_package_mapping.get(version)
if any([pkg.install_name in unsupported_packages for pkg in pair]):
continue
# The logic after this point only handles non SUCCESS statuses.
if res.status == compatibility_store.Status.SUCCESS:
continue
# If `other_package` is not self compatible (meaning that it has a
# conflict within it's own dependencies) then skip the check since
# a pairwise comparison is only significant if both packages are
# self_compatible.
self_compat_res = _get_self_compatibility_dict(other_package_name)
if self_compat_res[pyver]['status'] != BadgeStatus.SUCCESS:
continue
details = res.details or badge_utils.EMPTY_DETAILS
badge_status = PACKAGE_STATUS_TO_BADGE_STATUS.get(
res.status) or BadgeStatus.PAIR_INCOMPATIBLE
result_dict[pyver]['status'] = badge_status
result_dict[pyver]['details'][other_package_name] = details
return result_dict
