def test_git_py2py3_fresh_nodeps_ignore_git(self): """Tests that pair results containing git packages are ignored.""" fake_results = RECENT_SUCCESS_DATA + [ compatibility_store.CompatibilityResult( [ package.Package('git+git://github.com/google/apache-beam.git'), package.Package('git+git://github.com/google/api-core.git') ], python_major_version=2, status=compatibility_store.Status.INSTALL_ERROR, timestamp=datetime.datetime(2019, 5, 7, 0, 0, 0)), compatibility_store.CompatibilityResult( [ package.Package('git+git://github.com/google/tensorflow.git'), package.Package('git+git://github.com/google/api-core.git') ], python_major_version=3, status=compatibility_store.Status.INSTALL_ERROR, timestamp=datetime.datetime(2019, 5, 7, 0, 0, 0)), ] self.fake_store.save_compatibility_statuses(fake_results) package_name = 'git+git://github.com/google/api-core.git' json_response = self.get_image_json(package_name) self.assertEqual(json_response['left_text'], 'compatibility check (master)') self.assertEqual(json_response['right_text'], 'success') self.assertEqual(json_response['right_color'], '#44CC44') self.assertLinkUrl(package_name, json_response['whole_link'])
def test__get_missing_details_missing_inputs(self): from compatibility_lib import compatibility_store from compatibility_lib import package TENSORFLOW = 'tensorflow' TENSORFLOW_RESULT_PY2 = compatibility_store.CompatibilityResult( packages=[package.Package(TENSORFLOW)], python_major_version=2, status=compatibility_store.Status.SUCCESS) TENSORFLOW_RESULT_PY3 = compatibility_store.CompatibilityResult( packages=[package.Package(TENSORFLOW)], python_major_version=3, status=compatibility_store.Status.SUCCESS) with self.assertRaises(AssertionError): package_names = [] results = [] main._get_missing_details(package_names, results) with self.assertRaises(AssertionError): package_names = [] results = [TENSORFLOW_RESULT_PY2] main._get_missing_details(package_names, results) with self.assertRaises(AssertionError): package_names = [] results = [TENSORFLOW_RESULT_PY2, TENSORFLOW_RESULT_PY3] main._get_missing_details(package_names, results)
def get_pairwise_compatibility_for_package(self, package_name: str) -> \ Mapping[FrozenSet[package.Package], List[compatibility_store.CompatibilityResult]]: """Returns a mapping between package pairs and CompatibilityResults. Args: package_name: The package to check compatibility for. Returns: A mapping between every pairing between the given package with each google cloud python package (found in configs.PKG_LIST) and their pairwise CompatibilityResults. For example: Given package_name = 'p1', configs.PKG_LIST = [p2, p3, p4] => { frozenset([p1, p2]): [CompatibilityResult...], frozenset([p1, p3]): [CompatibilityResult...], frozenset([p1, p4]): [CompatibilityResult...], }. """ package_pairs = [frozenset([package.Package(package_name), package.Package(name)]) for name in configs.PKG_LIST if package_name != name] results = {pair: self.get_pair_compatibility(pair) for pair in package_pairs if self.get_pair_compatibility(pair)} return results
def test__get_pair_compatibility_dict_internal_error(self): from compatibility_lib import compatibility_store from compatibility_lib import package expected = { 'py2': { 'status': main.BadgeStatus.PAIR_INCOMPATIBLE, 'details': { 'package2': {} } }, 'py3': { 'status': main.BadgeStatus.PAIR_INCOMPATIBLE, 'details': { 'package2': {} } }, } PACKAGE_1 = package.Package("package1") PACKAGE_2 = package.Package("package2") cr_py2 = compatibility_store.CompatibilityResult( packages=[PACKAGE_1, PACKAGE_2], python_major_version=2, status=compatibility_store.Status.CHECK_WARNING) cr_py3 = compatibility_store.CompatibilityResult( packages=[PACKAGE_1, PACKAGE_2], python_major_version=3, status=compatibility_store.Status.CHECK_WARNING) pair_result = [cr_py2, cr_py3] self.fake_store._packages_to_compatibility_result[frozenset( [PACKAGE_1, PACKAGE_2])] = pair_result mock_self_res = mock.Mock() self_res = { 'py2': { 'status': main.BadgeStatus.SUCCESS, 'details': {} }, 'py3': { 'status': main.BadgeStatus.SUCCESS, 'details': {} }, } mock_self_res.return_value = self_res patch_self_status = mock.patch('main._get_self_compatibility_dict', mock_self_res) pkgs = ['package2'] patch_configs = mock.patch('main.configs.PKG_LIST', pkgs) with self.patch_checker, self.patch_store, patch_self_status, \ patch_configs: result_dict = main._get_pair_compatibility_dict('package1') self.assertEqual(result_dict, expected)
def test__get_pair_compatibility_dict_self_conflict(self): # If the pair package is not self compatible, the package being checked # should not be marked as `INTERNAL_ERROR`. from compatibility_lib import compatibility_store from compatibility_lib import package expected = { 'py2': { 'status': main.BadgeStatus.SUCCESS, 'details': {} }, 'py3': { 'status': main.BadgeStatus.SUCCESS, 'details': {} }, } PACKAGE_1 = package.Package("opencensus") PACKAGE_2 = package.Package("tensorflow") cr_py2 = compatibility_store.CompatibilityResult( packages=[PACKAGE_1, PACKAGE_2], python_major_version=2, status=compatibility_store.Status.CHECK_WARNING) cr_py3 = compatibility_store.CompatibilityResult( packages=[PACKAGE_1, PACKAGE_2], python_major_version=3, status=compatibility_store.Status.CHECK_WARNING) pair_result = [cr_py2, cr_py3] self.fake_store._packages_to_compatibility_result[frozenset( [PACKAGE_1, PACKAGE_2])] = pair_result mock_self_res = mock.Mock() self_res = { 'py2': { 'status': main.BadgeStatus.SELF_INCOMPATIBLE, 'details': {} }, 'py3': { 'status': main.BadgeStatus.SELF_INCOMPATIBLE, 'details': {} }, } mock_self_res.return_value = self_res patch_self_status = mock.patch('main._get_self_compatibility_dict', mock_self_res) pkgs = [p.install_name for p in (PACKAGE_1, PACKAGE_2)] patch_configs = mock.patch('main.configs.PKG_LIST', pkgs) with self.patch_checker, self.patch_store, patch_self_status, \ patch_configs: result_dict = main._get_pair_compatibility_dict( PACKAGE_1.install_name) self.assertEqual(result_dict, expected)
def test__get_pair_status_for_packages_warning(self): from compatibility_lib import compatibility_store from compatibility_lib import package PACKAGE_1 = package.Package("package1") PACKAGE_2 = package.Package("package2") pkg_sets = [ ['package1', 'package2'], ] expected = { 'py2': { 'status': 'CHECK_WARNING', 'details': { 'package2': 'NO DETAILS' } }, 'py3': { 'status': 'CHECK_WARNING', 'details': { 'package2': 'NO DETAILS' } } } cr_py2 = compatibility_store.CompatibilityResult( packages=[PACKAGE_1, PACKAGE_2], python_major_version=2, status=compatibility_store.Status.CHECK_WARNING) cr_py3 = compatibility_store.CompatibilityResult( packages=[PACKAGE_1, PACKAGE_2], python_major_version=3, status=compatibility_store.Status.CHECK_WARNING) pair_result = [cr_py2, cr_py3] self.fake_store._packages_to_compatibility_result[frozenset( [PACKAGE_1, PACKAGE_2])] = pair_result mock_self_res = mock.Mock() self_res = { 'py2': { 'status': 'SUCCESS', 'details': {} }, 'py3': { 'status': 'SUCCESS', 'details': {} } } mock_self_res.return_value = self_res patch_self_status = mock.patch('main._get_result_from_cache', mock_self_res) with self.patch_checker, self.patch_store, patch_self_status: version_and_res = main._get_pair_status_for_packages(pkg_sets) self.assertEqual(version_and_res, expected)
def test__get_pair_status_for_packages_self_conflict(self): # If the pair package is not self compatible, the package being checked # should not be marked as CHECK_WARNING. from compatibility_lib import compatibility_store from compatibility_lib import package PACKAGE_1 = package.Package("package1") PACKAGE_2 = package.Package("tensorflow") pkg_sets = [ ['package1', 'tensorflow'], ] expected = { 'py2': { 'status': 'SUCCESS', 'details': {} }, 'py3': { 'status': 'SUCCESS', 'details': {} } } cr_py2 = compatibility_store.CompatibilityResult( packages=[PACKAGE_1, PACKAGE_2], python_major_version=2, status=compatibility_store.Status.CHECK_WARNING) cr_py3 = compatibility_store.CompatibilityResult( packages=[PACKAGE_1, PACKAGE_2], python_major_version=3, status=compatibility_store.Status.CHECK_WARNING) pair_result = [cr_py2, cr_py3] self.fake_store._packages_to_compatibility_result[frozenset( [PACKAGE_1, PACKAGE_2])] = pair_result mock_self_res = mock.Mock() self_res = { 'py2': { 'status': 'CHECK_WARNING', 'details': {} }, 'py3': { 'status': 'CHECK_WARNING', 'details': {} } } mock_self_res.return_value = self_res patch_self_status = mock.patch('main._get_result_from_cache', mock_self_res) with self.patch_checker, self.patch_store, patch_self_status: version_and_res = main._get_pair_status_for_packages(pkg_sets) self.assertEqual(version_and_res, expected)
def get_pairwise_compatibility_for_package(self, package_name) -> \ Mapping[FrozenSet[package.Package], List[CompatibilityResult]]: """Returns a mapping between package pairs and CompatibilityResults. Args: package_name: The package to check compatibility for. Returns: A mapping between every pairing between the given package with each google cloud python package (found in configs.PKG_LIST) and their pairwise CompatibilityResults. The returned CompatibilityResults do not include a set `dependency_info`. For example: Given package_name = 'p1', configs.PKG_LIST = [p2, p3, p4] => { frozenset([p1, p2]): [CompatibilityResult...], frozenset([p1, p3]): [CompatibilityResult...], frozenset([p1, p4]): [CompatibilityResult...], }. """ pkg_sets = [sorted([package_name, pkg]) for pkg in configs.PKG_LIST] install_names_lower = [pair[0] for pair in pkg_sets] install_names_higher = [pair[1] for pair in pkg_sets] packages_to_results = {} query = ('SELECT * ' 'FROM' '(SELECT *' ' FROM pairwise_compatibility_status' ' WHERE install_name_lower IN %s' ' AND install_name_higher IN %s) t1 ' 'WHERE t1.install_name_lower=%s ' 'OR t1.install_name_higher=%s') with closing(self.connect()) as conn: with closing(conn.cursor()) as cursor: cursor.execute(query, (install_names_lower, install_names_higher, package_name, package_name)) results = cursor.fetchall() for row in results: install_name_lower, install_name_higher, _, _, _, _ = row p_lower = package.Package(install_name_lower) p_higher = package.Package(install_name_higher) key = frozenset([p_lower, p_higher]) if not packages_to_results.get(key): packages_to_results[key] = [] packages_to_results[key].append( self._row_to_compatibility_status([p_lower, p_higher], row)) return packages_to_results
def get_packages(self) -> Iterable[package.Package]: """Returns all packages tracked by the system.""" query = 'SELECT DISTINCT install_name FROM {}'.format( self._self_table_id) query_job = self._client.query(query) for row in query_job: yield package.Package(install_name=row[0])
def run_check(): # First see if this package is already stored in BigQuery. package = package_module.Package(package_name) compatibility_status = store.get_self_compatibility(package) if compatibility_status: for res in compatibility_status: py_version = PY_VER_MAPPING[res.python_major_version] version_and_res[py_version]['status'] = res.status.value version_and_res[py_version]['details'] = res.details \ if res.details is not None else EMPTY_DETAILS # If not pre stored in BigQuery, run the check for the package. else: py2_res = checker.check([package_name], '2') py3_res = checker.check([package_name], '3') version_and_res['py2']['status'] = py2_res.get('result') py2_description = py2_res.get('description') py2_details = EMPTY_DETAILS if py2_description is None \ else py2_description version_and_res['py2']['details'] = py2_details version_and_res['py3']['status'] = py3_res.get('result') py3_description = py3_res.get('description') py3_details = EMPTY_DETAILS if py3_description is None \ else py3_description version_and_res['py3']['details'] = py3_details url = _get_badge_url(version_and_res, package_name) # Write the result to memory store redis_client.set( '{}_self_comp_badge'.format(package_name), version_and_res) return requests.get(url).text
def run_check(): # First see if this package is already stored in BigQuery. package = package_module.Package(package_name) compatibility_status = badge_utils.store.get_self_compatibility( package) if compatibility_status: for res in compatibility_status: py_version = badge_utils.PY_VER_MAPPING[ res.python_major_version] version_and_res[py_version]['status'] = res.status.value version_and_res[py_version]['details'] = res.details \ if res.details is not None else badge_utils.EMPTY_DETAILS # If not pre stored in BigQuery, run the check for the package. else: py2_res = badge_utils.checker.check([package_name], '2') py3_res = badge_utils.checker.check([package_name], '3') version_and_res['py2']['status'] = py2_res.get('result') py2_description = py2_res.get('description') py2_details = badge_utils.EMPTY_DETAILS if py2_description \ is None else py2_description version_and_res['py2']['details'] = py2_details version_and_res['py3']['status'] = py3_res.get('result') py3_description = py3_res.get('description') py3_details = badge_utils.EMPTY_DETAILS if py3_description \ is None else py3_description version_and_res['py3']['details'] = py3_details # Add the timestamp version_and_res['timestamp'] = datetime.datetime.now().strftime( badge_utils.TIMESTAMP_FORMAT) # Write the result to Cloud Datastore cache.set('{}_self_comp_badge'.format(package_key), version_and_res)
def main(): parser = argparse.ArgumentParser( description='Display a grid show the dependency compatibility ' + 'between Python packages') parser.add_argument('--packages', nargs='+', default=_DEFAULT_INSTALL_NAMES, help='the packages to display compatibility ' + 'information for') parser.add_argument('--browser', action='store_true', default=False, help='display the grid in a browser tab') args = parser.parse_args() store = compatibility_store.CompatibilityStore() grid_builder = GridBuilder(store) grid_html = grid_builder.build_grid( (package.Package(install_name) for install_name in args.packages)) if args.browser: _, grid_path = tempfile.mkstemp(suffix='.html') with open(grid_path, 'wt') as f: f.write(grid_html) webbrowser.open_new_tab('file://' + grid_path) else: print(grid_html, end='')
def get_package_details(self, p: package.Package): """Return the dict of package check summary. { 'self_conflict': True, 'pairwise_conflict': ['package1', 'package2'], 'latest_version': '0.1.0', } """ latest_version = self._package_with_dependency_info[p.install_name][ p.install_name]['latest_version'] pairwise_conflict = [] # Initialize the values result = { 'self_conflict': False, 'pairwise_conflict': pairwise_conflict, 'latest_version': latest_version, } for pair_pkg in configs.PKG_LIST: check_result = self.get_result(p, package.Package(pair_pkg)) # Get self compatibility status if pair_pkg == p.install_name: result['self_conflict'] = True \ if check_result['status_type'] != 'self-success' else False # Get pairwise compatibility status else: if check_result['status_type'] != 'pairwise-success' \ and self.has_issues(p): pairwise_conflict.append(pair_pkg) result['pairwise_conflict'] = pairwise_conflict return result
def test_get_deprecated_dep(self): # TODO: use a more convincing mock that doesn't say that *all* packages # are deprecated. mock_call_pypi_json_api = mock.Mock(autospec=True) mock_call_pypi_json_api.return_value = self.PKG_INFO self.fake_store.save_compatibility_statuses([ compatibility_store.CompatibilityResult( packages=[package.Package('opencencus')], python_major_version='3', status=compatibility_store.Status.SUCCESS, details=None, dependency_info=self.DEP_INFO), ]) patch_utils = mock.patch( 'compatibility_lib.deprecated_dep_finder.utils.call_pypi_json_api', mock_call_pypi_json_api) with patch_utils: finder = deprecated_dep_finder.DeprecatedDepFinder( checker=self.mock_checker, store=self.fake_store) deprecated_deps = finder.get_deprecated_dep('opencencus') expected_deprecated_deps = set(['dep1', 'dep2']) self.assertEqual(set(deprecated_deps[1]), expected_deprecated_deps)
def test__get_self_compatibility_dict(self): from compatibility_lib import compatibility_store from compatibility_lib import package expected = { 'py2': { 'status': main.BadgeStatus.SUCCESS, 'details': 'The package does not support this version of python.' }, 'py3': { 'status': main.BadgeStatus.SUCCESS, 'details': 'NO DETAILS' }, } PACKAGE = package.Package('tensorflow') cr_py3 = compatibility_store.CompatibilityResult( packages=[PACKAGE], python_major_version=3, status=compatibility_store.Status.SUCCESS) self.fake_store._packages_to_compatibility_result[frozenset( [PACKAGE])] = [cr_py3] with self.patch_checker, self.patch_store: result_dict = main._get_self_compatibility_dict('tensorflow') self.assertEqual(result_dict, expected)
def _get_pair_status_for_packages(pkg_sets): version_and_res = { 'py2': { 'status': 'SUCCESS', 'details': {}, }, 'py3': { 'status': 'SUCCESS', 'details': {}, } } for pkg_set in pkg_sets: pkgs = [package_module.Package(pkg) for pkg in pkg_set] pair_res = store.get_pair_compatibility(pkgs) for res in pair_res: py_version = PY_VER_MAPPING[res.python_major_version] # Status showing one of the check failures if res.status.value != 'SUCCESS': # Ignore the package that not support for given py_ver if pkg_set[1] in \ configs.PKG_PY_VERSION_NOT_SUPPORTED.get( res.python_major_version): continue version_and_res[py_version]['status'] = res.status.value version_and_res[py_version]['details'][pkg_set[1]] = \ res.details if res.details is not None else EMPTY_DETAILS return version_and_res
def main(): parser = argparse.ArgumentParser( description='Display a grid show the dependency compatibility ' + 'between Python packages') parser.add_argument('--packages', nargs='+', default=_DEFAULT_INSTALL_NAMES, help='the packages to display compatibility ' + 'information for') parser.add_argument('--browser', action='store_true', default=False, help='display the grid in a browser tab') args = parser.parse_args() checker = compatibility_checker.CompatibilityChecker() store = compatibility_store.CompatibilityStore() packages = [ package.Package(install_name) for install_name in args.packages ] logging.info("Getting self compatibility results...") package_to_results = store.get_self_compatibilities(packages) logging.info("Getting pairwise compatibility results...") pairwise_to_results = store.get_compatibility_combinations(packages) package_with_dependency_info = {} for pkg in configs.PKG_LIST: dep_info = store.get_dependency_info(pkg) package_with_dependency_info[pkg] = dep_info results = _ResultHolder(package_to_results, pairwise_to_results, package_with_dependency_info, checker, store) dashboard_builder = DashboardBuilder(packages, results) # Build the pairwise grid dashboard logging.info('Starting build the grid...') grid_html = dashboard_builder.build_dashboard( 'dashboard/grid-template.html') grid_path = os.path.dirname(os.path.abspath(__file__)) + '/grid.html' with open(grid_path, 'wt') as f: f.write(grid_html) # Build the dashboard main page logging.info('Starting build the main dashboard...') main_html = dashboard_builder.build_dashboard( 'dashboard/main-template.html') main_path = os.path.dirname(os.path.abspath(__file__)) + '/index.html' with open(main_path, 'wt') as f: f.write(main_html) if args.browser: webbrowser.open_new_tab('file://' + main_path)
def get_packages(self) -> Iterable[package.Package]: """Returns all packages tracked by the system.""" query = 'SELECT DISTINCT install_name FROM self_compatibility_status' with closing(self.connect()) as conn: with closing(conn.cursor()) as cursor: cursor.execute(query) results = cursor.fetchall() for row in results: yield package.Package(install_name=row[0])
def test__get_missing_details_for_self_compatibility(self): from compatibility_lib import compatibility_store from compatibility_lib import configs from compatibility_lib import package for package_name in configs.WHITELIST_PKGS: results = [] if package_name not in ('tensorflow'): results.append( compatibility_store.CompatibilityResult( packages=[package.Package(p) for p in package_name], python_major_version=2, status=compatibility_store.Status.SUCCESS)) if package_name not in ('apache-beam[gcp]', 'gsutil'): results.append( compatibility_store.CompatibilityResult( packages=[package.Package(p) for p in package_name], python_major_version=3, status=compatibility_store.Status.SUCCESS)) details = main._get_missing_details([package_name], results) self.assertEqual(details, None)
def test__get_missing_details_for_pair_compatibility(self): from compatibility_lib import compatibility_store from compatibility_lib import configs from compatibility_lib import package import itertools for p1, p2 in itertools.combinations(configs.WHITELIST_PKGS, r=2): pkgs = [p1, p2] results = [] if all([p not in ('tensorflow') for p in pkgs]): results.append( compatibility_store.CompatibilityResult( packages=[package.Package(p) for p in pkgs], python_major_version=2, status=compatibility_store.Status.SUCCESS)) if all([p not in ('apache-beam[gcp]', 'gsutil') for p in pkgs]): results.append( compatibility_store.CompatibilityResult( packages=[package.Package(p) for p in pkgs], python_major_version=3, status=compatibility_store.Status.SUCCESS)) details = main._get_missing_details(pkgs, results) self.assertEqual(details, None)
def test__get_missing_details_unsupported_packages(self): from compatibility_lib import compatibility_store from compatibility_lib import package TENSORFLOW = 'tensorflow' UNSUPPORTED = 'unsupported' UNSUPPORTED_RESULT_PY2 = compatibility_store.CompatibilityResult( packages=[package.Package(UNSUPPORTED)], python_major_version=2, status=compatibility_store.Status.UNKNOWN) PAIR_RESULT_PY3 = compatibility_store.CompatibilityResult( packages=[package.Package(p) for p in (TENSORFLOW, UNSUPPORTED)], python_major_version=3, status=compatibility_store.Status.UNKNOWN) with self.assertRaises(AssertionError): package_names = [UNSUPPORTED] results = [UNSUPPORTED_RESULT_PY2] main._get_missing_details(package_names, results) with self.assertRaises(AssertionError): package_names = [TENSORFLOW, UNSUPPORTED] results = [PAIR_RESULT_PY3] main._get_missing_details(package_names, results)
def test__get_missing_details_pair_fail(self): from compatibility_lib import compatibility_store from compatibility_lib import package package_names = ['opencensus', 'compatibility-lib'] results = [ compatibility_store.CompatibilityResult( packages=[package.Package(name) for name in package_names], python_major_version=2, status=compatibility_store.Status.SUCCESS) ] details = main._get_missing_details(package_names, results) expected_details = ("Missing data for packages=['opencensus', " "'compatibility-lib'], versions=[3]") self.assertEqual(details, expected_details)
def test__get_pair_status_for_packages_install_error(self): from compatibility_lib import compatibility_store from compatibility_lib import package PACKAGE_1 = package.Package("package1") PACKAGE_2 = package.Package("tensorflow") pkg_sets = [ ['package1', 'tensorflow'], ] expected = { 'py2': { 'status': 'SUCCESS', 'details': {} }, 'py3': { 'status': 'SUCCESS', 'details': {} } } cr_py2 = compatibility_store.CompatibilityResult( packages=[PACKAGE_1, PACKAGE_2], python_major_version=2, status=compatibility_store.Status.INSTALL_ERROR) cr_py3 = compatibility_store.CompatibilityResult( packages=[PACKAGE_1, PACKAGE_2], python_major_version=3, status=compatibility_store.Status.SUCCESS) pair_result = [cr_py2, cr_py3] self.fake_store._packages_to_compatibility_result[frozenset( [PACKAGE_1, PACKAGE_2])] = pair_result with self.patch_checker, self.patch_store: version_and_res = main._get_pair_status_for_packages(pkg_sets) self.assertEqual(version_and_res, expected)
def _get_pair_status_for_packages(pkg_sets): """Get the pairwise dependency compatibility check result for packages. Rules: - Return warning status if not compatible with any of the listed Google owned Python packages. Whole list in compatibility_lib.config. - Ignore the warning status if: - The package doesn't support one of the Python versions. - The package's pair is not self compatible, which means the pairwise conflict isn't related to the package being checked. - Return success status if compatible with all the list of Google owned Python packages. """ version_and_res = { 'py2': { 'status': 'SUCCESS', 'details': {}, }, 'py3': { 'status': 'SUCCESS', 'details': {}, } } for pkg_set in pkg_sets: pkgs = [package_module.Package(pkg) for pkg in pkg_set] pair_res = badge_utils.store.get_pair_compatibility(pkgs) for res in pair_res: py_version = badge_utils.PY_VER_MAPPING[res.python_major_version] # Status showing one of the check failures if res.status.value != 'SUCCESS': # Ignore the package that not support for given py_ver if pkg_set[1] in \ configs.PKG_PY_VERSION_NOT_SUPPORTED.get( res.python_major_version): continue # Ignore the package that are not self compatible self_status = _get_result_from_cache( package_name=pkg_set[1], badge_type=badge_utils.BadgeType.SELF_COMP_BADGE) if self_status[py_version]['status'] != 'SUCCESS': continue version_and_res[py_version]['status'] = res.status.value version_and_res[py_version]['details'][pkg_set[1]] = \ res.details if res.details is not None \ else badge_utils.EMPTY_DETAILS return version_and_res
def _get_self_compatibility_dict(package_name: str) -> dict: """Returns a dict containing self compatibility status and details. Args: package_name: the name of the package to check (e.g. "google-cloud-storage"). Returns: A dict containing the self compatibility status and details for any self incompatibilities. The dict will be formatted like the following: { 'py2': { 'status': BadgeStatus.SUCCESS, 'details': {} }, 'py3': { 'status': BadgeStatus.SUCCESS, 'details': {} }, } """ pkg = package.Package(package_name) compatibility_results = badge_utils.store.get_self_compatibility(pkg) return _self_compatibilities_to_dict(package_name, compatibility_results)
def get_package_details(self, p: package.Package): """Return the dict of package check summary. { 'self_conflict': True, 'pairwise_conflict': ['package1', 'package2'], 'latest_version': '0.1.0', } """ # The package being checked will appear in the dep list, but for # apache-beam[gcp], the package in dep list will just be apache-beam. self_dep_name = p.install_name if '[' in p.install_name: self_dep_name = p.install_name.split('[')[0] latest_version = self._package_with_dependency_info[ p.install_name][self_dep_name]['latest_version'] pairwise_conflict = [] # Initialize the values result = { 'self_conflict': False, 'pairwise_conflict': pairwise_conflict, 'latest_version': latest_version, } for pair_pkg in configs.PKG_LIST: check_result = self.get_result(p, package.Package(pair_pkg)) # Get self compatibility status if pair_pkg == p.install_name: result['self_conflict'] = True \ if check_result['status_type'] != 'self-success' else False # Get pairwise compatibility status else: if check_result['status_type'] != 'pairwise-success' \ and self.has_issues(p): pairwise_conflict.append(pair_pkg) result['pairwise_conflict'] = pairwise_conflict return result
def _get_pair_status_for_packages(pkg_sets): version_and_res = { 'py2': { 'status': 'SUCCESS', 'details': {}, }, 'py3': { 'status': 'SUCCESS', 'details': {}, } } for pkg_set in pkg_sets: pkgs = [package_module.Package(pkg) for pkg in pkg_set] pair_res = store.get_pair_compatibility(pkgs) for res in pair_res: py_version = PY_VER_MAPPING[res.python_major_version] # Status showing one of the check failures if res.status.value != 'SUCCESS': version_and_res[py_version]['status'] = res.status.value version_and_res[py_version]['details'][pkg_set[1]] = \ res.details if res.details is not None else EMPTY_DETAILS return version_and_res
def _result_dict_to_compatibility_result(results, python_version): res_list = [] for item in results: res_dict = item[0] check_result = res_dict.get('result') packages_list = [ package.Package(pkg) for pkg in res_dict.get('packages') ] details = res_dict.get('description') timestamp = datetime.datetime.now().isoformat() dependency_info = res_dict.get('dependency_info') compatibility_result = compatibility_store.CompatibilityResult( packages=packages_list, python_major_version=python_version, status=compatibility_store.Status(check_result), details=details, timestamp=timestamp, dependency_info=dependency_info) res_list.append(compatibility_result) return res_list
def _get_self_compatibility_dict(package_name: str) -> dict: """Returns a dict containing self compatibility status and details. Args: package_name: the name of the package to check (e.g. "google-cloud-storage"). Returns: A dict containing the self compatibility status and details for any self incompatibilities. The dict will be formatted like the following: { 'py2': { 'status': BadgeStatus.SUCCESS, 'details': {} }, 'py3': { 'status': BadgeStatus.SUCCESS, 'details': {} }, } """ pkg = package.Package(package_name) compatibility_results = badge_utils.store.get_self_compatibility(pkg) missing_details = _get_missing_details([package_name], compatibility_results) if missing_details: result_dict = badge_utils._build_default_result( status=BadgeStatus.MISSING_DATA, details=missing_details) return result_dict result_dict = badge_utils._build_default_result( status=BadgeStatus.SUCCESS, details='The package does not support this version of python.') for res in compatibility_results: pyver = badge_utils.PY_VER_MAPPING[res.python_major_version] badge_status = PACKAGE_STATUS_TO_BADGE_STATUS.get( res.status) or BadgeStatus.SELF_INCOMPATIBLE result_dict[pyver]['status'] = badge_status result_dict[pyver]['details'] = res.details if res.details is None: result_dict[pyver]['details'] = badge_utils.EMPTY_DETAILS return result_dict
def main(): parser = argparse.ArgumentParser( description='Display a grid show the dependency compatibility ' + 'between Python packages') parser.add_argument('--packages', nargs='+', default=_DEFAULT_INSTALL_NAMES, help='the packages to display compatibility ' + 'information for') parser.add_argument('--browser', action='store_true', default=False, help='display the grid in a browser tab') args = parser.parse_args() checker = compatibility_checker.CompatibilityChecker() store = compatibility_store.CompatibilityStore() instance_flag = '-instances={}=tcp:{}'.format(INSTANCE_CONNECTION_NAME, PORT) cloud_sql_proxy_path = './cloud_sql_proxy' try: # Run cloud_sql_proxy process = popen_spawn.PopenSpawn([cloud_sql_proxy_path, instance_flag]) process.expect('Ready for new connection', timeout=5) packages = [ package.Package(install_name) for install_name in args.packages ] logging.info('Getting self compatibility results...') package_to_results = store.get_self_compatibilities(packages) logging.info('Getting pairwise compatibility results...') pairwise_to_results = store.get_compatibility_combinations(packages) package_with_dependency_info = {} for pkg in configs.PKG_LIST: dep_info = store.get_dependency_info(pkg) package_with_dependency_info[pkg] = dep_info results = _ResultHolder(package_to_results, pairwise_to_results, package_with_dependency_info, checker, store) dashboard_builder = DashboardBuilder(packages, results) # Build the pairwise grid dashboard logging.info('Starting build the grid...') grid_html = dashboard_builder.build_dashboard( 'dashboard/grid-template.html') grid_path = os.path.dirname(os.path.abspath(__file__)) + '/grid.html' with open(grid_path, 'wt') as f: f.write(grid_html) # Build the dashboard main page logging.info('Starting build the main dashboard...') main_html = dashboard_builder.build_dashboard( 'dashboard/main-template.html') main_path = os.path.dirname(os.path.abspath(__file__)) + '/index.html' with open(main_path, 'wt') as f: f.write(main_html) except Exception: raise DashboardBuilderError('Error occurs when building dashboard.' 'Output: {}'.format(process.before)) finally: process.kill(signal.SIGTERM) if args.browser: webbrowser.open_new_tab('file://' + main_path)