def _get_result_from_cache(package_name: str, badge_type: badge_utils.BadgeType, commit_number: str = None) -> dict: """Get check result from cache.""" # Return unknown if package not in whitelist if not utils._is_package_in_whitelist([package_name]): result = badge_utils._build_default_result(badge_type=badge_type, status='UNKNOWN', details={}) # Get the result from cache, return None if not in cache else: package_key = '{}_{}'.format( package_name, commit_number) if commit_number else package_name result = cache.get('{}_{}'.format(package_key, badge_type.value)) if result is None: result = badge_utils._build_default_result(badge_type=badge_type, status='CALCULATING', details={}) return result
def _self_compatibilities_to_dict( package_name: str, compatibility_results: List[compatibility_store.CompatibilityResult] ) -> dict: """Converts a CompatibilityResult into a dict containing compatibility info. Args: package_name: the name of the package to check (e.g. "google-cloud-storage"). compatibility_results: a (possibly empty) list of compatibility results representing a self-compatibility check for the given package. Returns: A dict containing the self compatibility status and details for any self incompatibilities. The dict will be formatted like the following: { 'py2': { 'status': BadgeStatus.SUCCESS, 'details': {} }, 'py3': { 'status': BadgeStatus.SUCCESS, 'details': {} }, } """ missing_details = _get_missing_details([package_name], compatibility_results) if missing_details: result_dict = badge_utils._build_default_result( status=BadgeStatus.MISSING_DATA, details=missing_details) return result_dict result_dict = badge_utils._build_default_result( status=BadgeStatus.SUCCESS, details='The package does not support this version of python.') for res in compatibility_results: pyver = badge_utils.PY_VER_MAPPING[res.python_major_version] badge_status = PACKAGE_STATUS_TO_BADGE_STATUS.get( res.status) or BadgeStatus.SELF_INCOMPATIBLE result_dict[pyver]['status'] = badge_status result_dict[pyver]['details'] = res.details if res.details is None: result_dict[pyver]['details'] = badge_utils.EMPTY_DETAILS return result_dict
def _get_dependency_dict(package_name: str) -> dict: """Returns a dict containing outdated dependencies' status and details. Args: package_name: the name of the package to get outdated dependencies for (e.g. "google-cloud-storage"). Returns: A dict containing the outdated dependency status and details for any outdated dependencies. Note that details maps to a dict that may be nested. The returned dict will be formatted like the following: { 'status': BadgeStatus.OBSOLETE_DEPENDENCY, 'details': { 'google-cloud-bigquery': { 'installed_version': '1.6.1', 'latest_version': '1.10.0', 'priority': 'HIGH_PRIORITY', 'detail': ('google-cloud-bigquery is 3 or more minor' 'versions behind the latest version') }, }, } """ result_dict = badge_utils._build_default_result(status=BadgeStatus.SUCCESS, include_pyversion=False, details={}) outdated_deps = badge_utils.highlighter.check_package(package_name) _deps_list = badge_utils.finder.get_deprecated_dep(package_name)[1] deprecated_deps = ', '.join(_deps_list) outdated_depencdency_name_to_details = {} max_level = badge_utils.priority_level.UP_TO_DATE for dep in outdated_deps: dep_detail = {} level = dep.priority.level if level.value > max_level.value: max_level = level dep_detail['installed_version'] = dep.installed_version dep_detail['latest_version'] = dep.latest_version dep_detail['priority'] = dep.priority.level.name dep_detail['detail'] = dep.priority.details outdated_depencdency_name_to_details[dep.name] = dep_detail badge_status = DEPENDENCY_STATUS_TO_BADGE_STATUS[max_level] result_dict['status'] = badge_status result_dict['details'] = outdated_depencdency_name_to_details result_dict['deprecated_deps'] = deprecated_deps return result_dict
def _get_self_compatibility_dict(package_name: str) -> dict: """Returns a dict containing self compatibility status and details. Args: package_name: the name of the package to check (e.g. "google-cloud-storage"). Returns: A dict containing the self compatibility status and details for any self incompatibilities. The dict will be formatted like the following: { 'py2': { 'status': BadgeStatus.SUCCESS, 'details': {} }, 'py3': { 'status': BadgeStatus.SUCCESS, 'details': {} }, } """ pkg = package.Package(package_name) compatibility_results = badge_utils.store.get_self_compatibility(pkg) missing_details = _get_missing_details([package_name], compatibility_results) if missing_details: result_dict = badge_utils._build_default_result( status=BadgeStatus.MISSING_DATA, details=missing_details) return result_dict result_dict = badge_utils._build_default_result( status=BadgeStatus.SUCCESS, details='The package does not support this version of python.') for res in compatibility_results: pyver = badge_utils.PY_VER_MAPPING[res.python_major_version] badge_status = PACKAGE_STATUS_TO_BADGE_STATUS.get( res.status) or BadgeStatus.SELF_INCOMPATIBLE result_dict[pyver]['status'] = badge_status result_dict[pyver]['details'] = res.details if res.details is None: result_dict[pyver]['details'] = badge_utils.EMPTY_DETAILS return result_dict
def _get_check_results(package_name: str, commit_number: str = None): """Gets the compatibility and dependency check results. Returns a 3 tuple: self compatibility, pair compatibility, dependency dicts that are used to generate badge images and badge target pages. """ default_status = BadgeStatus.UNKNOWN_PACKAGE unknown_details = ('This package is not a whitelisted google python ' 'package; to whitelist a package, contact the python ' 'team.') self_compat_res = badge_utils._build_default_result( status=default_status, details=unknown_details) google_compat_res = badge_utils._build_default_result( status=default_status, details={}) dependency_res = badge_utils._build_default_result(status=default_status, include_pyversion=False, details={}) # If a package is not whitelisted, return defaults if not compat_utils._is_package_in_whitelist([package_name]): return (self_compat_res, google_compat_res, dependency_res) try: self_compat_res = _get_self_compatibility_dict(package_name) google_compat_res = _get_pair_compatibility_dict(package_name) dependency_res = _get_dependency_dict(package_name) except Exception: logging.exception( 'Exception checking results for "{}"'.format(package_name)) error_status = BadgeStatus.INTERNAL_ERROR self_compat_res, google_compat_res, dependency_res = ( badge_utils._build_default_result(status=error_status), badge_utils._build_default_result(status=error_status, details={}), badge_utils._build_default_result(status=error_status, include_pyversion=False)) return (self_compat_res, google_compat_res, dependency_res)
def google_compatibility_badge_image(): """Badge showing whether a package is compatible with Google OSS Python packages. If all packages success, status is SUCCESS; else set status to one of the failure types, details can be found at the target link.""" package_name = flask.request.args.get('package') force_run_check = flask.request.args.get('force_run_check') commit_number = flask.request.args.get('commit_number') badge_name = flask.request.args.get('badge_name') package_key = '{}_{}'.format( package_name, commit_number) if commit_number else package_name if badge_name is None: badge_name = 'google compatibility' def run_check(): pkg_sets = [[package_name, pkg] for pkg in configs.PKG_LIST] if package_name in configs.PKG_LIST: result = _get_pair_status_for_packages(pkg_sets) else: version_and_res = { 'py2': { 'status': 'SUCCESS', 'details': {}, }, 'py3': { 'status': 'SUCCESS', 'details': {}, }, 'timestamp': '', } for py_ver in [2, 3]: results = list(badge_utils.checker.get_pairwise_compatibility( py_ver, pkg_sets)) logging.warning(results) py_version = badge_utils.PY_VER_MAPPING[py_ver] for res in results: res_item = res[0] status = res_item.get('result') package = res_item.get('packages')[1] if status != 'SUCCESS': # Ignore the package that not support for given py_ver if package in \ configs.PKG_PY_VERSION_NOT_SUPPORTED.get( py_ver): continue # Ignore the package that are not self compatible self_status = _get_result_from_cache( package_name=package_name, badge_type=badge_utils.BadgeType.SELF_COMP_BADGE) if self_status[py_version]['status'] not in [ 'SUCCESS', 'CALCULATING']: continue # Status showing one of the check failures version_and_res[ py_version]['status'] = res_item.get('result') description = res_item.get('description') details = badge_utils.EMPTY_DETAILS if description \ is None else description version_and_res[ py_version]['details'][package] = details version_and_res['timestamp'] = datetime.datetime.now().strftime( badge_utils.TIMESTAMP_FORMAT) result = version_and_res # Write the result to Cloud Datastore cache.set('{}_google_comp_badge'.format(package_key), result) google_comp_res = cache.get('{}_google_comp_badge'.format(package_key)) if not utils._is_package_in_whitelist([package_name]): google_comp_res = badge_utils._build_default_result( badge_type=badge_utils.BadgeType.GOOGLE_COMP_BADGE, status='UNKNOWN', details={}) if google_comp_res is None: details = badge_utils._build_default_result( badge_type=badge_utils.BadgeType.GOOGLE_COMP_BADGE, status='CALCULATING', details={}) else: details = google_comp_res # Run the check if google_comp_res is None or forced to populate the cache # or the cache is outdated. if google_comp_res is None or force_run_check is not None: threading.Thread(target=run_check).start() elif google_comp_res is not None: timestamp = google_comp_res.get('timestamp') if not badge_utils._is_github_cache_valid(timestamp): threading.Thread(target=run_check).start() badge = badge_utils._get_badge(details, badge_name) response = flask.make_response(badge) response.content_type = badge_utils.SVG_CONTENT_TYPE response.headers['Cache-Control'] = 'no-cache' response.add_etag() return response
def self_dependency_badge_image(): """Badge showing whether a package is has outdated dependencies.""" package_name = flask.request.args.get('package') force_run_check = flask.request.args.get('force_run_check') commit_number = flask.request.args.get('commit_number') badge_name = flask.request.args.get('badge_name') package_key = '{}_{}'.format( package_name, commit_number) if commit_number else package_name if badge_name is None: badge_name = 'dependency status' def run_check(): res = { 'status': 'UP_TO_DATE', 'details': {}, 'timestamp': '', } details = {} outdated = badge_utils.highlighter.check_package(package_name) deprecated_deps_list = badge_utils.finder.get_deprecated_dep( package_name)[1] deprecated_deps = ', '.join(deprecated_deps_list) max_level = badge_utils.priority_level.UP_TO_DATE for dep in outdated: dep_detail = {} level = dep.priority.level if level.value > max_level.value: max_level = level dep_detail['installed_version'] = dep.installed_version dep_detail['latest_version'] = dep.latest_version dep_detail['priority'] = dep.priority.level.name dep_detail['detail'] = dep.priority.details details[dep.name] = dep_detail res['status'] = max_level.name res['details'] = details res['deprecated_deps'] = deprecated_deps res['timestamp'] = datetime.datetime.now().strftime( badge_utils.TIMESTAMP_FORMAT) # Write the result to Cloud Datastore cache.set('{}_dependency_badge'.format(package_key), res) if not utils._is_package_in_whitelist([package_name]): dependency_res = badge_utils._build_default_result( badge_type=badge_utils.BadgeType.DEP_BADGE, status='UNKNOWN', details={}) else: dependency_res = cache.get('{}_dependency_badge'.format(package_key)) if dependency_res is None: details = badge_utils.DEFAULT_DEPENDENCY_RESULT else: details = dependency_res # Run the check if dependency_res is None or forced to populate the cache # or the cache is outdated. if dependency_res is None or force_run_check is not None: threading.Thread(target=run_check).start() elif dependency_res is not None: timestamp = dependency_res.get('timestamp') if not badge_utils._is_github_cache_valid(timestamp): threading.Thread(target=run_check).start() badge = badge_utils._get_badge(details, badge_name) response = flask.make_response(badge) response.content_type = badge_utils.SVG_CONTENT_TYPE response.headers['Cache-Control'] = 'no-cache' response.add_etag() return response
def self_compatibility_badge_image(): """Badge showing whether a package is compatible with itself.""" package_name = flask.request.args.get('package') force_run_check = flask.request.args.get('force_run_check') commit_number = flask.request.args.get('commit_number') badge_name = flask.request.args.get('badge_name') package_key = '{}_{}'.format( package_name, commit_number) if commit_number else package_name if badge_name is None: badge_name = 'self compatibility' version_and_res = badge_utils._build_default_result( badge_type=badge_utils.BadgeType.SELF_COMP_BADGE, status='CALCULATING', details=None) def run_check(): # First see if this package is already stored in BigQuery. package = package_module.Package(package_name) compatibility_status = badge_utils.store.get_self_compatibility( package) if compatibility_status: for res in compatibility_status: py_version = badge_utils.PY_VER_MAPPING[ res.python_major_version] version_and_res[py_version]['status'] = res.status.value version_and_res[py_version]['details'] = res.details \ if res.details is not None else badge_utils.EMPTY_DETAILS # If not pre stored in BigQuery, run the check for the package. else: py2_res = badge_utils.checker.check([package_name], '2') py3_res = badge_utils.checker.check([package_name], '3') version_and_res['py2']['status'] = py2_res.get('result') py2_description = py2_res.get('description') py2_details = badge_utils.EMPTY_DETAILS if py2_description \ is None else py2_description version_and_res['py2']['details'] = py2_details version_and_res['py3']['status'] = py3_res.get('result') py3_description = py3_res.get('description') py3_details = badge_utils.EMPTY_DETAILS if py3_description \ is None else py3_description version_and_res['py3']['details'] = py3_details # Add the timestamp version_and_res['timestamp'] = datetime.datetime.now().strftime( badge_utils.TIMESTAMP_FORMAT) # Write the result to Cloud Datastore cache.set('{}_self_comp_badge'.format(package_key), version_and_res) if not utils._is_package_in_whitelist([package_name]): self_comp_res = badge_utils._build_default_result( badge_type=badge_utils.BadgeType.SELF_COMP_BADGE, status='UNKNOWN', details=badge_utils.PACKAGE_NOT_SUPPORTED) else: self_comp_res = cache.get('{}_self_comp_badge'.format(package_key)) if self_comp_res is None: details = version_and_res else: details = self_comp_res # Run the check if details is None or forced to populate the cache or # the cache is outdated. if self_comp_res is None or force_run_check is not None: threading.Thread(target=run_check).start() elif self_comp_res is not None: timestamp = self_comp_res.get('timestamp') if not badge_utils._is_github_cache_valid(timestamp): threading.Thread(target=run_check).start() badge = badge_utils._get_badge(details, badge_name) response = flask.make_response(badge) response.content_type = badge_utils.SVG_CONTENT_TYPE response.headers['Cache-Control'] = 'no-cache' response.add_etag() return response
def _get_pair_compatibility_dict(package_name: str) -> dict: """Get the pairwise dependency compatibility check result for a package. Rules: - Return warning status if not compatible with any of the listed Google owned Python packages. Whole list in compatibility_lib.config. - Ignore the warning status if: - The package doesn't support one of the Python versions. - The package's pair is not self compatible, which means the pairwise conflict isn't related to the package being checked. - Return success status if compatible with all the list of Google owned Python packages. Args: package_name: the name of the package to get pairwise dependencies for (e.g. "google-cloud-storage"). Returns: A dict containing the pair compatibility status and details for any pair incompatibilities. Note that details must map to another dict. The returned dict will be formatted like the following: { 'py2': {'status': BadgeStatus.PAIR_INCOMPATIBLE, 'details': {'apache-beam[gcp]': 'NO DETAILS'}}, 'py3': {'status': BadgeStatus.SUCCESS, 'details': {}} } """ default_details = {} result_dict = badge_utils._build_default_result(status=BadgeStatus.SUCCESS, details=default_details) unsupported_package_mapping = configs.PKG_PY_VERSION_NOT_SUPPORTED pair_mapping = badge_utils.store.get_pairwise_compatibility_for_package( package_name) for pair, compatibility_results in pair_mapping.items(): other_package = _get_other_package_from_set(package_name, pair) other_package_name = other_package.install_name missing_details = _get_missing_details( [pkg.install_name for pkg in pair], compatibility_results) if missing_details: result_dict = badge_utils._build_default_result( status=BadgeStatus.MISSING_DATA, details={other_package_name: missing_details}) return result_dict for res in compatibility_results: version = res.python_major_version # eg. '2', '3' pyver = badge_utils.PY_VER_MAPPING[version] # eg. 'py2', 'py3' if result_dict[pyver]['details'] is default_details: result_dict[pyver]['details'] = {} # Not all packages are supported in both Python 2 and Python 3. If # either package is not supported in the Python version being # checked then skip the check. unsupported_packages = unsupported_package_mapping.get(version) if any([pkg.install_name in unsupported_packages for pkg in pair]): continue # The logic after this point only handles non SUCCESS statuses. if res.status == compatibility_store.Status.SUCCESS: continue # If `other_package` is not self compatible (meaning that it has a # conflict within it's own dependencies) then skip the check since # a pairwise comparison is only significant if both packages are # self_compatible. self_compat_res = _get_self_compatibility_dict(other_package_name) if self_compat_res[pyver]['status'] != BadgeStatus.SUCCESS: continue details = res.details or badge_utils.EMPTY_DETAILS badge_status = PACKAGE_STATUS_TO_BADGE_STATUS.get( res.status) or BadgeStatus.PAIR_INCOMPATIBLE result_dict[pyver]['status'] = badge_status result_dict[pyver]['details'][other_package_name] = details return result_dict