def check(self, packages, python_version):
if not utils._is_package_in_whitelist(packages):
UNKNOWN_STATUS_RESULT['packages'] = packages
UNKNOWN_STATUS_RESULT['description'] = 'Package is not supported' \
' by our checker server.'
return UNKNOWN_STATUS_RESULT
return {
'result': 'SUCCESS',
'packages': packages,
'description': None,
'dependency_info': DEP_INFO,
}
def check(self, packages, python_version):
"""Call the checker server to get back status results."""
if not utils._is_package_in_whitelist(packages):
UNKNOWN_STATUS_RESULT['packages'] = packages
UNKNOWN_STATUS_RESULT['description'] = 'Package is not supported' \
' by our checker server.'
return UNKNOWN_STATUS_RESULT
data = {'python-version': python_version, 'package': packages}
result = requests.get(SERVER_URL, params=data)
content = result.content.decode('utf-8')
return json.loads(content)
def _get_missing_details(
package_names: List[str],
results: Iterable[compatibility_store.CompatibilityResult]
) -> Optional[str]:
"""Gets the details for any missing data (if there is any)
Args:
package_names: A list of length 1 or 2 of the package name(s) to look
up, e.g. ["tensorflow"], ["tensorflow", "opencensus"].
results: A list of length 1 or 2 of the `CompatibilityResults` for the
given package(s). The package names in the `CompatibilityResults`
should match the package_names argument.
Returns:
None if there is no missing data; a description of the package(s) and
version(s) missing otherwise.
For example, if package_names=['opencensus'], and given that
`opencensus` is compatible with both python versions 2 and 3, if
`results` only contained a result for 3, this would be returned:
"Missing data for packages=['opencensus'], versions=[2]".
"""
package_names.sort()
message = 'package_names: Expected length of 1 or 2, got {}'.format(
len(package_names))
assert len(package_names) in (1, 2), message
message = 'One of the packages in {} is not whitelisted'.format(
str(package_names))
assert compat_utils._is_package_in_whitelist(package_names), message
all_versions = (2, 3)
versions_supported = set(all_versions)
for version in all_versions:
for package_name in package_names:
if 'github.com' in package_name:
package_name = configs.WHITELIST_URLS[package_name]
if package_name in configs.PKG_PY_VERSION_NOT_SUPPORTED[version]:
versions_supported.discard(version)
break
versions_seen = {result.python_major_version for result in results}
if versions_seen.issuperset(versions_supported):
return None
missing_versions = list(versions_supported - versions_seen)
missing_details = 'Missing data for packages={}, versions={}'.format(
package_names, missing_versions)
return missing_details
def check(self, packages, python_version):
"""Call the checker server to get back status results."""
if not utils._is_package_in_whitelist(packages):
UNKNOWN_STATUS_RESULT['packages'] = packages
UNKNOWN_STATUS_RESULT['description'] = 'Package is not supported' \
' by our checker server.'
return UNKNOWN_STATUS_RESULT
data = {
'python-version': python_version,
'package': packages
}
# Set the timeout to 299 seconds, which should be less than the
# docker timeout (300 seconds).
result = requests.get(SERVER_URL, params=data, timeout=299)
content = result.content.decode('utf-8')
return json.loads(content), python_version
def _get_result_from_cache(package_name: str,
badge_type: badge_utils.BadgeType,
commit_number: str = None) -> dict:
"""Get check result from cache."""
# Return unknown if package not in whitelist
if not utils._is_package_in_whitelist([package_name]):
result = badge_utils._build_default_result(badge_type=badge_type,
status='UNKNOWN',
details={})
# Get the result from cache, return None if not in cache
else:
package_key = '{}_{}'.format(
package_name, commit_number) if commit_number else package_name
result = cache.get('{}_{}'.format(package_key, badge_type.value))
if result is None:
result = badge_utils._build_default_result(badge_type=badge_type,
status='CALCULATING',
details={})
return result
def _get_check_results(package_name: str, commit_number: str = None):
"""Gets the compatibility and dependency check results.
Returns a 3 tuple: self compatibility, pair compatibility, dependency dicts
that are used to generate badge images and badge target pages.
"""
default_status = BadgeStatus.UNKNOWN_PACKAGE
unknown_details = ('This package is not a whitelisted google python '
'package; to whitelist a package, contact the python '
'team.')
self_compat_res = badge_utils._build_default_result(
status=default_status, details=unknown_details)
google_compat_res = badge_utils._build_default_result(
status=default_status, details={})
dependency_res = badge_utils._build_default_result(status=default_status,
include_pyversion=False,
details={})
# If a package is not whitelisted, return defaults
if not compat_utils._is_package_in_whitelist([package_name]):
return (self_compat_res, google_compat_res, dependency_res)
try:
self_compat_res = _get_self_compatibility_dict(package_name)
google_compat_res = _get_pair_compatibility_dict(package_name)
dependency_res = _get_dependency_dict(package_name)
except Exception:
logging.exception(
'Exception checking results for "{}"'.format(package_name))
error_status = BadgeStatus.INTERNAL_ERROR
self_compat_res, google_compat_res, dependency_res = (
badge_utils._build_default_result(status=error_status),
badge_utils._build_default_result(status=error_status, details={}),
badge_utils._build_default_result(status=error_status,
include_pyversion=False))
return (self_compat_res, google_compat_res, dependency_res)
def check(self, packages, python_version):
"""Call the checker server to get back status results."""
if not utils._is_package_in_whitelist(packages):
UNKNOWN_STATUS_RESULT['packages'] = packages
UNKNOWN_STATUS_RESULT['description'] = 'Package is not supported' \
' by our checker server.'
return UNKNOWN_STATUS_RESULT
start_time = time.time()
data = {
'python-version': python_version,
'package': packages
}
# Set the timeout to 299 seconds, which should be less than the
# docker timeout (300 seconds).
try:
result = requests.get(SERVER_URL, params=data, timeout=299)
content = result.content.decode('utf-8')
except Exception as e:
check_time = time.time() - start_time
logging.getLogger("compatibility_lib").error(
'Checked {} in {:.1f} seconds: {}'.format(
packages, check_time, e))
raise
check_time = time.time() - start_time
if result.ok:
logging.getLogger("compatibility_lib").debug(
'Checked {} in {:.1f} seconds (success!)'.format(
packages, check_time))
else:
logging.getLogger("compatibility_lib").debug(
'Checked {} in {:.1f} seconds: {}'.format(
packages, check_time, content))
result.raise_for_status()
return json.loads(content), python_version
def google_compatibility_badge_image():
"""Badge showing whether a package is compatible with Google OSS Python
packages. If all packages success, status is SUCCESS; else set status
to one of the failure types, details can be found at the target link."""
package_name = flask.request.args.get('package')
force_run_check = flask.request.args.get('force_run_check')
commit_number = flask.request.args.get('commit_number')
badge_name = flask.request.args.get('badge_name')
package_key = '{}_{}'.format(
package_name, commit_number) if commit_number else package_name
if badge_name is None:
badge_name = 'google compatibility'
def run_check():
pkg_sets = [[package_name, pkg] for pkg in configs.PKG_LIST]
if package_name in configs.PKG_LIST:
result = _get_pair_status_for_packages(pkg_sets)
else:
version_and_res = {
'py2': {
'status': 'SUCCESS',
'details': {},
},
'py3': {
'status': 'SUCCESS',
'details': {},
},
'timestamp': '',
}
for py_ver in [2, 3]:
results = list(badge_utils.checker.get_pairwise_compatibility(
py_ver, pkg_sets))
logging.warning(results)
py_version = badge_utils.PY_VER_MAPPING[py_ver]
for res in results:
res_item = res[0]
status = res_item.get('result')
package = res_item.get('packages')[1]
if status != 'SUCCESS':
# Ignore the package that not support for given py_ver
if package in \
configs.PKG_PY_VERSION_NOT_SUPPORTED.get(
py_ver):
continue
# Ignore the package that are not self compatible
self_status = _get_result_from_cache(
package_name=package_name,
badge_type=badge_utils.BadgeType.SELF_COMP_BADGE)
if self_status[py_version]['status'] not in [
'SUCCESS', 'CALCULATING']:
continue
# Status showing one of the check failures
version_and_res[
py_version]['status'] = res_item.get('result')
description = res_item.get('description')
details = badge_utils.EMPTY_DETAILS if description \
is None else description
version_and_res[
py_version]['details'][package] = details
version_and_res['timestamp'] = datetime.datetime.now().strftime(
badge_utils.TIMESTAMP_FORMAT)
result = version_and_res
# Write the result to Cloud Datastore
cache.set('{}_google_comp_badge'.format(package_key), result)
google_comp_res = cache.get('{}_google_comp_badge'.format(package_key))
if not utils._is_package_in_whitelist([package_name]):
google_comp_res = badge_utils._build_default_result(
badge_type=badge_utils.BadgeType.GOOGLE_COMP_BADGE,
status='UNKNOWN',
details={})
if google_comp_res is None:
details = badge_utils._build_default_result(
badge_type=badge_utils.BadgeType.GOOGLE_COMP_BADGE,
status='CALCULATING',
details={})
else:
details = google_comp_res
# Run the check if google_comp_res is None or forced to populate the cache
# or the cache is outdated.
if google_comp_res is None or force_run_check is not None:
threading.Thread(target=run_check).start()
elif google_comp_res is not None:
timestamp = google_comp_res.get('timestamp')
if not badge_utils._is_github_cache_valid(timestamp):
threading.Thread(target=run_check).start()
badge = badge_utils._get_badge(details, badge_name)
response = flask.make_response(badge)
response.content_type = badge_utils.SVG_CONTENT_TYPE
response.headers['Cache-Control'] = 'no-cache'
response.add_etag()
return response
def self_dependency_badge_image():
"""Badge showing whether a package is has outdated dependencies."""
package_name = flask.request.args.get('package')
force_run_check = flask.request.args.get('force_run_check')
commit_number = flask.request.args.get('commit_number')
badge_name = flask.request.args.get('badge_name')
package_key = '{}_{}'.format(
package_name, commit_number) if commit_number else package_name
if badge_name is None:
badge_name = 'dependency status'
def run_check():
res = {
'status': 'UP_TO_DATE',
'details': {},
'timestamp': '',
}
details = {}
outdated = badge_utils.highlighter.check_package(package_name)
deprecated_deps_list = badge_utils.finder.get_deprecated_dep(
package_name)[1]
deprecated_deps = ', '.join(deprecated_deps_list)
max_level = badge_utils.priority_level.UP_TO_DATE
for dep in outdated:
dep_detail = {}
level = dep.priority.level
if level.value > max_level.value:
max_level = level
dep_detail['installed_version'] = dep.installed_version
dep_detail['latest_version'] = dep.latest_version
dep_detail['priority'] = dep.priority.level.name
dep_detail['detail'] = dep.priority.details
details[dep.name] = dep_detail
res['status'] = max_level.name
res['details'] = details
res['deprecated_deps'] = deprecated_deps
res['timestamp'] = datetime.datetime.now().strftime(
badge_utils.TIMESTAMP_FORMAT)
# Write the result to Cloud Datastore
cache.set('{}_dependency_badge'.format(package_key), res)
if not utils._is_package_in_whitelist([package_name]):
dependency_res = badge_utils._build_default_result(
badge_type=badge_utils.BadgeType.DEP_BADGE,
status='UNKNOWN',
details={})
else:
dependency_res = cache.get('{}_dependency_badge'.format(package_key))
if dependency_res is None:
details = badge_utils.DEFAULT_DEPENDENCY_RESULT
else:
details = dependency_res
# Run the check if dependency_res is None or forced to populate the cache
# or the cache is outdated.
if dependency_res is None or force_run_check is not None:
threading.Thread(target=run_check).start()
elif dependency_res is not None:
timestamp = dependency_res.get('timestamp')
if not badge_utils._is_github_cache_valid(timestamp):
threading.Thread(target=run_check).start()
badge = badge_utils._get_badge(details, badge_name)
response = flask.make_response(badge)
response.content_type = badge_utils.SVG_CONTENT_TYPE
response.headers['Cache-Control'] = 'no-cache'
response.add_etag()
return response
def self_compatibility_badge_image():
"""Badge showing whether a package is compatible with itself."""
package_name = flask.request.args.get('package')
force_run_check = flask.request.args.get('force_run_check')
commit_number = flask.request.args.get('commit_number')
badge_name = flask.request.args.get('badge_name')
package_key = '{}_{}'.format(
package_name, commit_number) if commit_number else package_name
if badge_name is None:
badge_name = 'self compatibility'
version_and_res = badge_utils._build_default_result(
badge_type=badge_utils.BadgeType.SELF_COMP_BADGE,
status='CALCULATING',
details=None)
def run_check():
# First see if this package is already stored in BigQuery.
package = package_module.Package(package_name)
compatibility_status = badge_utils.store.get_self_compatibility(
package)
if compatibility_status:
for res in compatibility_status:
py_version = badge_utils.PY_VER_MAPPING[
res.python_major_version]
version_and_res[py_version]['status'] = res.status.value
version_and_res[py_version]['details'] = res.details \
if res.details is not None else badge_utils.EMPTY_DETAILS
# If not pre stored in BigQuery, run the check for the package.
else:
py2_res = badge_utils.checker.check([package_name], '2')
py3_res = badge_utils.checker.check([package_name], '3')
version_and_res['py2']['status'] = py2_res.get('result')
py2_description = py2_res.get('description')
py2_details = badge_utils.EMPTY_DETAILS if py2_description \
is None else py2_description
version_and_res['py2']['details'] = py2_details
version_and_res['py3']['status'] = py3_res.get('result')
py3_description = py3_res.get('description')
py3_details = badge_utils.EMPTY_DETAILS if py3_description \
is None else py3_description
version_and_res['py3']['details'] = py3_details
# Add the timestamp
version_and_res['timestamp'] = datetime.datetime.now().strftime(
badge_utils.TIMESTAMP_FORMAT)
# Write the result to Cloud Datastore
cache.set('{}_self_comp_badge'.format(package_key), version_and_res)
if not utils._is_package_in_whitelist([package_name]):
self_comp_res = badge_utils._build_default_result(
badge_type=badge_utils.BadgeType.SELF_COMP_BADGE,
status='UNKNOWN',
details=badge_utils.PACKAGE_NOT_SUPPORTED)
else:
self_comp_res = cache.get('{}_self_comp_badge'.format(package_key))
if self_comp_res is None:
details = version_and_res
else:
details = self_comp_res
# Run the check if details is None or forced to populate the cache or
# the cache is outdated.
if self_comp_res is None or force_run_check is not None:
threading.Thread(target=run_check).start()
elif self_comp_res is not None:
timestamp = self_comp_res.get('timestamp')
if not badge_utils._is_github_cache_valid(timestamp):
threading.Thread(target=run_check).start()
badge = badge_utils._get_badge(details, badge_name)
response = flask.make_response(badge)
response.content_type = badge_utils.SVG_CONTENT_TYPE
response.headers['Cache-Control'] = 'no-cache'
response.add_etag()
return response
