def test_git_py2py3_fresh_nodeps_ignore_git(self):
"""Tests that pair results containing git packages are ignored."""
fake_results = RECENT_SUCCESS_DATA + [
compatibility_store.CompatibilityResult(
[
package.Package('git+git://github.com/google/apache-beam.git'),
package.Package('git+git://github.com/google/api-core.git')
],
python_major_version=2,
status=compatibility_store.Status.INSTALL_ERROR,
timestamp=datetime.datetime(2019, 5, 7, 0, 0, 0)),
compatibility_store.CompatibilityResult(
[
package.Package('git+git://github.com/google/tensorflow.git'),
package.Package('git+git://github.com/google/api-core.git')
],
python_major_version=3,
status=compatibility_store.Status.INSTALL_ERROR,
timestamp=datetime.datetime(2019, 5, 7, 0, 0, 0)),
]
self.fake_store.save_compatibility_statuses(fake_results)
package_name = 'git+git://github.com/google/api-core.git'
json_response = self.get_image_json(package_name)
self.assertEqual(json_response['left_text'],
'compatibility check (master)')
self.assertEqual(json_response['right_text'], 'success')
self.assertEqual(json_response['right_color'], '#44CC44')
self.assertLinkUrl(package_name, json_response['whole_link'])
def test__get_missing_details_missing_inputs(self):
from compatibility_lib import compatibility_store
from compatibility_lib import package
TENSORFLOW = 'tensorflow'
TENSORFLOW_RESULT_PY2 = compatibility_store.CompatibilityResult(
packages=[package.Package(TENSORFLOW)],
python_major_version=2,
status=compatibility_store.Status.SUCCESS)
TENSORFLOW_RESULT_PY3 = compatibility_store.CompatibilityResult(
packages=[package.Package(TENSORFLOW)],
python_major_version=3,
status=compatibility_store.Status.SUCCESS)
with self.assertRaises(AssertionError):
package_names = []
results = []
main._get_missing_details(package_names, results)
with self.assertRaises(AssertionError):
package_names = []
results = [TENSORFLOW_RESULT_PY2]
main._get_missing_details(package_names, results)
with self.assertRaises(AssertionError):
package_names = []
results = [TENSORFLOW_RESULT_PY2, TENSORFLOW_RESULT_PY3]
main._get_missing_details(package_names, results)
def get_pairwise_compatibility_for_package(self, package_name: str) -> \
Mapping[FrozenSet[package.Package],
List[compatibility_store.CompatibilityResult]]:
"""Returns a mapping between package pairs and CompatibilityResults.
Args:
package_name: The package to check compatibility for.
Returns:
A mapping between every pairing between the given package with
each google cloud python package (found in configs.PKG_LIST) and
their pairwise CompatibilityResults. For example:
Given package_name = 'p1', configs.PKG_LIST = [p2, p3, p4] =>
{
frozenset([p1, p2]): [CompatibilityResult...],
frozenset([p1, p3]): [CompatibilityResult...],
frozenset([p1, p4]): [CompatibilityResult...],
}.
"""
package_pairs = [frozenset([package.Package(package_name),
package.Package(name)])
for name in configs.PKG_LIST
if package_name != name]
results = {pair: self.get_pair_compatibility(pair)
for pair in package_pairs
if self.get_pair_compatibility(pair)}
return results
def test__get_pair_compatibility_dict_internal_error(self):
from compatibility_lib import compatibility_store
from compatibility_lib import package
expected = {
'py2': {
'status': main.BadgeStatus.PAIR_INCOMPATIBLE,
'details': {
'package2': {}
}
},
'py3': {
'status': main.BadgeStatus.PAIR_INCOMPATIBLE,
'details': {
'package2': {}
}
},
}
PACKAGE_1 = package.Package("package1")
PACKAGE_2 = package.Package("package2")
cr_py2 = compatibility_store.CompatibilityResult(
packages=[PACKAGE_1, PACKAGE_2],
python_major_version=2,
status=compatibility_store.Status.CHECK_WARNING)
cr_py3 = compatibility_store.CompatibilityResult(
packages=[PACKAGE_1, PACKAGE_2],
python_major_version=3,
status=compatibility_store.Status.CHECK_WARNING)
pair_result = [cr_py2, cr_py3]
self.fake_store._packages_to_compatibility_result[frozenset(
[PACKAGE_1, PACKAGE_2])] = pair_result
mock_self_res = mock.Mock()
self_res = {
'py2': {
'status': main.BadgeStatus.SUCCESS,
'details': {}
},
'py3': {
'status': main.BadgeStatus.SUCCESS,
'details': {}
},
}
mock_self_res.return_value = self_res
patch_self_status = mock.patch('main._get_self_compatibility_dict',
mock_self_res)
pkgs = ['package2']
patch_configs = mock.patch('main.configs.PKG_LIST', pkgs)
with self.patch_checker, self.patch_store, patch_self_status, \
patch_configs:
result_dict = main._get_pair_compatibility_dict('package1')
self.assertEqual(result_dict, expected)
def test__get_pair_compatibility_dict_self_conflict(self):
# If the pair package is not self compatible, the package being checked
# should not be marked as `INTERNAL_ERROR`.
from compatibility_lib import compatibility_store
from compatibility_lib import package
expected = {
'py2': {
'status': main.BadgeStatus.SUCCESS,
'details': {}
},
'py3': {
'status': main.BadgeStatus.SUCCESS,
'details': {}
},
}
PACKAGE_1 = package.Package("opencensus")
PACKAGE_2 = package.Package("tensorflow")
cr_py2 = compatibility_store.CompatibilityResult(
packages=[PACKAGE_1, PACKAGE_2],
python_major_version=2,
status=compatibility_store.Status.CHECK_WARNING)
cr_py3 = compatibility_store.CompatibilityResult(
packages=[PACKAGE_1, PACKAGE_2],
python_major_version=3,
status=compatibility_store.Status.CHECK_WARNING)
pair_result = [cr_py2, cr_py3]
self.fake_store._packages_to_compatibility_result[frozenset(
[PACKAGE_1, PACKAGE_2])] = pair_result
mock_self_res = mock.Mock()
self_res = {
'py2': {
'status': main.BadgeStatus.SELF_INCOMPATIBLE,
'details': {}
},
'py3': {
'status': main.BadgeStatus.SELF_INCOMPATIBLE,
'details': {}
},
}
mock_self_res.return_value = self_res
patch_self_status = mock.patch('main._get_self_compatibility_dict',
mock_self_res)
pkgs = [p.install_name for p in (PACKAGE_1, PACKAGE_2)]
patch_configs = mock.patch('main.configs.PKG_LIST', pkgs)
with self.patch_checker, self.patch_store, patch_self_status, \
patch_configs:
result_dict = main._get_pair_compatibility_dict(
PACKAGE_1.install_name)
self.assertEqual(result_dict, expected)
def test__get_pair_status_for_packages_warning(self):
from compatibility_lib import compatibility_store
from compatibility_lib import package
PACKAGE_1 = package.Package("package1")
PACKAGE_2 = package.Package("package2")
pkg_sets = [
['package1', 'package2'],
]
expected = {
'py2': {
'status': 'CHECK_WARNING',
'details': {
'package2': 'NO DETAILS'
}
},
'py3': {
'status': 'CHECK_WARNING',
'details': {
'package2': 'NO DETAILS'
}
}
}
cr_py2 = compatibility_store.CompatibilityResult(
packages=[PACKAGE_1, PACKAGE_2],
python_major_version=2,
status=compatibility_store.Status.CHECK_WARNING)
cr_py3 = compatibility_store.CompatibilityResult(
packages=[PACKAGE_1, PACKAGE_2],
python_major_version=3,
status=compatibility_store.Status.CHECK_WARNING)
pair_result = [cr_py2, cr_py3]
self.fake_store._packages_to_compatibility_result[frozenset(
[PACKAGE_1, PACKAGE_2])] = pair_result
mock_self_res = mock.Mock()
self_res = {
'py2': {
'status': 'SUCCESS',
'details': {}
},
'py3': {
'status': 'SUCCESS',
'details': {}
}
}
mock_self_res.return_value = self_res
patch_self_status = mock.patch('main._get_result_from_cache',
mock_self_res)
with self.patch_checker, self.patch_store, patch_self_status:
version_and_res = main._get_pair_status_for_packages(pkg_sets)
self.assertEqual(version_and_res, expected)
def test__get_pair_status_for_packages_self_conflict(self):
# If the pair package is not self compatible, the package being checked
# should not be marked as CHECK_WARNING.
from compatibility_lib import compatibility_store
from compatibility_lib import package
PACKAGE_1 = package.Package("package1")
PACKAGE_2 = package.Package("tensorflow")
pkg_sets = [
['package1', 'tensorflow'],
]
expected = {
'py2': {
'status': 'SUCCESS',
'details': {}
},
'py3': {
'status': 'SUCCESS',
'details': {}
}
}
cr_py2 = compatibility_store.CompatibilityResult(
packages=[PACKAGE_1, PACKAGE_2],
python_major_version=2,
status=compatibility_store.Status.CHECK_WARNING)
cr_py3 = compatibility_store.CompatibilityResult(
packages=[PACKAGE_1, PACKAGE_2],
python_major_version=3,
status=compatibility_store.Status.CHECK_WARNING)
pair_result = [cr_py2, cr_py3]
self.fake_store._packages_to_compatibility_result[frozenset(
[PACKAGE_1, PACKAGE_2])] = pair_result
mock_self_res = mock.Mock()
self_res = {
'py2': {
'status': 'CHECK_WARNING',
'details': {}
},
'py3': {
'status': 'CHECK_WARNING',
'details': {}
}
}
mock_self_res.return_value = self_res
patch_self_status = mock.patch('main._get_result_from_cache',
mock_self_res)
with self.patch_checker, self.patch_store, patch_self_status:
version_and_res = main._get_pair_status_for_packages(pkg_sets)
self.assertEqual(version_and_res, expected)
def get_pairwise_compatibility_for_package(self, package_name) -> \
Mapping[FrozenSet[package.Package], List[CompatibilityResult]]:
"""Returns a mapping between package pairs and CompatibilityResults.
Args:
package_name: The package to check compatibility for.
Returns:
A mapping between every pairing between the given package with
each google cloud python package (found in configs.PKG_LIST) and
their pairwise CompatibilityResults. The returned
CompatibilityResults do not include a set `dependency_info`.
For example:
Given package_name = 'p1', configs.PKG_LIST = [p2, p3, p4] =>
{
frozenset([p1, p2]): [CompatibilityResult...],
frozenset([p1, p3]): [CompatibilityResult...],
frozenset([p1, p4]): [CompatibilityResult...],
}.
"""
pkg_sets = [sorted([package_name, pkg]) for pkg in configs.PKG_LIST]
install_names_lower = [pair[0] for pair in pkg_sets]
install_names_higher = [pair[1] for pair in pkg_sets]
packages_to_results = {}
query = ('SELECT * '
'FROM'
'(SELECT *'
' FROM pairwise_compatibility_status'
' WHERE install_name_lower IN %s'
' AND install_name_higher IN %s) t1 '
'WHERE t1.install_name_lower=%s '
'OR t1.install_name_higher=%s')
with closing(self.connect()) as conn:
with closing(conn.cursor()) as cursor:
cursor.execute(query,
(install_names_lower, install_names_higher,
package_name, package_name))
results = cursor.fetchall()
for row in results:
install_name_lower, install_name_higher, _, _, _, _ = row
p_lower = package.Package(install_name_lower)
p_higher = package.Package(install_name_higher)
key = frozenset([p_lower, p_higher])
if not packages_to_results.get(key):
packages_to_results[key] = []
packages_to_results[key].append(
self._row_to_compatibility_status([p_lower, p_higher], row))
return packages_to_results
def get_packages(self) -> Iterable[package.Package]:
"""Returns all packages tracked by the system."""
query = 'SELECT DISTINCT install_name FROM {}'.format(
self._self_table_id)
query_job = self._client.query(query)
for row in query_job:
yield package.Package(install_name=row[0])
def run_check():
# First see if this package is already stored in BigQuery.
package = package_module.Package(package_name)
compatibility_status = store.get_self_compatibility(package)
if compatibility_status:
for res in compatibility_status:
py_version = PY_VER_MAPPING[res.python_major_version]
version_and_res[py_version]['status'] = res.status.value
version_and_res[py_version]['details'] = res.details \
if res.details is not None else EMPTY_DETAILS
# If not pre stored in BigQuery, run the check for the package.
else:
py2_res = checker.check([package_name], '2')
py3_res = checker.check([package_name], '3')
version_and_res['py2']['status'] = py2_res.get('result')
py2_description = py2_res.get('description')
py2_details = EMPTY_DETAILS if py2_description is None \
else py2_description
version_and_res['py2']['details'] = py2_details
version_and_res['py3']['status'] = py3_res.get('result')
py3_description = py3_res.get('description')
py3_details = EMPTY_DETAILS if py3_description is None \
else py3_description
version_and_res['py3']['details'] = py3_details
url = _get_badge_url(version_and_res, package_name)
# Write the result to memory store
redis_client.set(
'{}_self_comp_badge'.format(package_name), version_and_res)
return requests.get(url).text
def run_check():
# First see if this package is already stored in BigQuery.
package = package_module.Package(package_name)
compatibility_status = badge_utils.store.get_self_compatibility(
package)
if compatibility_status:
for res in compatibility_status:
py_version = badge_utils.PY_VER_MAPPING[
res.python_major_version]
version_and_res[py_version]['status'] = res.status.value
version_and_res[py_version]['details'] = res.details \
if res.details is not None else badge_utils.EMPTY_DETAILS
# If not pre stored in BigQuery, run the check for the package.
else:
py2_res = badge_utils.checker.check([package_name], '2')
py3_res = badge_utils.checker.check([package_name], '3')
version_and_res['py2']['status'] = py2_res.get('result')
py2_description = py2_res.get('description')
py2_details = badge_utils.EMPTY_DETAILS if py2_description \
is None else py2_description
version_and_res['py2']['details'] = py2_details
version_and_res['py3']['status'] = py3_res.get('result')
py3_description = py3_res.get('description')
py3_details = badge_utils.EMPTY_DETAILS if py3_description \
is None else py3_description
version_and_res['py3']['details'] = py3_details
# Add the timestamp
version_and_res['timestamp'] = datetime.datetime.now().strftime(
badge_utils.TIMESTAMP_FORMAT)
# Write the result to Cloud Datastore
cache.set('{}_self_comp_badge'.format(package_key), version_and_res)
def main():
parser = argparse.ArgumentParser(
description='Display a grid show the dependency compatibility ' +
'between Python packages')
parser.add_argument('--packages',
nargs='+',
default=_DEFAULT_INSTALL_NAMES,
help='the packages to display compatibility ' +
'information for')
parser.add_argument('--browser',
action='store_true',
default=False,
help='display the grid in a browser tab')
args = parser.parse_args()
store = compatibility_store.CompatibilityStore()
grid_builder = GridBuilder(store)
grid_html = grid_builder.build_grid(
(package.Package(install_name) for install_name in args.packages))
if args.browser:
_, grid_path = tempfile.mkstemp(suffix='.html')
with open(grid_path, 'wt') as f:
f.write(grid_html)
webbrowser.open_new_tab('file://' + grid_path)
else:
print(grid_html, end='')
def get_package_details(self, p: package.Package):
"""Return the dict of package check summary.
{
'self_conflict': True,
'pairwise_conflict': ['package1', 'package2'],
'latest_version': '0.1.0',
}
"""
latest_version = self._package_with_dependency_info[p.install_name][
p.install_name]['latest_version']
pairwise_conflict = []
# Initialize the values
result = {
'self_conflict': False,
'pairwise_conflict': pairwise_conflict,
'latest_version': latest_version,
}
for pair_pkg in configs.PKG_LIST:
check_result = self.get_result(p, package.Package(pair_pkg))
# Get self compatibility status
if pair_pkg == p.install_name:
result['self_conflict'] = True \
if check_result['status_type'] != 'self-success' else False
# Get pairwise compatibility status
else:
if check_result['status_type'] != 'pairwise-success' \
and self.has_issues(p):
pairwise_conflict.append(pair_pkg)
result['pairwise_conflict'] = pairwise_conflict
return result
def test_get_deprecated_dep(self):
# TODO: use a more convincing mock that doesn't say that *all* packages
# are deprecated.
mock_call_pypi_json_api = mock.Mock(autospec=True)
mock_call_pypi_json_api.return_value = self.PKG_INFO
self.fake_store.save_compatibility_statuses([
compatibility_store.CompatibilityResult(
packages=[package.Package('opencencus')],
python_major_version='3',
status=compatibility_store.Status.SUCCESS,
details=None,
dependency_info=self.DEP_INFO),
])
patch_utils = mock.patch(
'compatibility_lib.deprecated_dep_finder.utils.call_pypi_json_api',
mock_call_pypi_json_api)
with patch_utils:
finder = deprecated_dep_finder.DeprecatedDepFinder(
checker=self.mock_checker, store=self.fake_store)
deprecated_deps = finder.get_deprecated_dep('opencencus')
expected_deprecated_deps = set(['dep1', 'dep2'])
self.assertEqual(set(deprecated_deps[1]), expected_deprecated_deps)
def test__get_self_compatibility_dict(self):
from compatibility_lib import compatibility_store
from compatibility_lib import package
expected = {
'py2': {
'status': main.BadgeStatus.SUCCESS,
'details':
'The package does not support this version of python.'
},
'py3': {
'status': main.BadgeStatus.SUCCESS,
'details': 'NO DETAILS'
},
}
PACKAGE = package.Package('tensorflow')
cr_py3 = compatibility_store.CompatibilityResult(
packages=[PACKAGE],
python_major_version=3,
status=compatibility_store.Status.SUCCESS)
self.fake_store._packages_to_compatibility_result[frozenset(
[PACKAGE])] = [cr_py3]
with self.patch_checker, self.patch_store:
result_dict = main._get_self_compatibility_dict('tensorflow')
self.assertEqual(result_dict, expected)
def _get_pair_status_for_packages(pkg_sets):
version_and_res = {
'py2': {
'status': 'SUCCESS',
'details': {},
},
'py3': {
'status': 'SUCCESS',
'details': {},
}
}
for pkg_set in pkg_sets:
pkgs = [package_module.Package(pkg) for pkg in pkg_set]
pair_res = store.get_pair_compatibility(pkgs)
for res in pair_res:
py_version = PY_VER_MAPPING[res.python_major_version]
# Status showing one of the check failures
if res.status.value != 'SUCCESS':
# Ignore the package that not support for given py_ver
if pkg_set[1] in \
configs.PKG_PY_VERSION_NOT_SUPPORTED.get(
res.python_major_version):
continue
version_and_res[py_version]['status'] = res.status.value
version_and_res[py_version]['details'][pkg_set[1]] = \
res.details if res.details is not None else EMPTY_DETAILS
return version_and_res
def main():
parser = argparse.ArgumentParser(
description='Display a grid show the dependency compatibility ' +
'between Python packages')
parser.add_argument('--packages',
nargs='+',
default=_DEFAULT_INSTALL_NAMES,
help='the packages to display compatibility ' +
'information for')
parser.add_argument('--browser',
action='store_true',
default=False,
help='display the grid in a browser tab')
args = parser.parse_args()
checker = compatibility_checker.CompatibilityChecker()
store = compatibility_store.CompatibilityStore()
packages = [
package.Package(install_name) for install_name in args.packages
]
logging.info("Getting self compatibility results...")
package_to_results = store.get_self_compatibilities(packages)
logging.info("Getting pairwise compatibility results...")
pairwise_to_results = store.get_compatibility_combinations(packages)
package_with_dependency_info = {}
for pkg in configs.PKG_LIST:
dep_info = store.get_dependency_info(pkg)
package_with_dependency_info[pkg] = dep_info
results = _ResultHolder(package_to_results, pairwise_to_results,
package_with_dependency_info, checker, store)
dashboard_builder = DashboardBuilder(packages, results)
# Build the pairwise grid dashboard
logging.info('Starting build the grid...')
grid_html = dashboard_builder.build_dashboard(
'dashboard/grid-template.html')
grid_path = os.path.dirname(os.path.abspath(__file__)) + '/grid.html'
with open(grid_path, 'wt') as f:
f.write(grid_html)
# Build the dashboard main page
logging.info('Starting build the main dashboard...')
main_html = dashboard_builder.build_dashboard(
'dashboard/main-template.html')
main_path = os.path.dirname(os.path.abspath(__file__)) + '/index.html'
with open(main_path, 'wt') as f:
f.write(main_html)
if args.browser:
webbrowser.open_new_tab('file://' + main_path)
def get_packages(self) -> Iterable[package.Package]:
"""Returns all packages tracked by the system."""
query = 'SELECT DISTINCT install_name FROM self_compatibility_status'
with closing(self.connect()) as conn:
with closing(conn.cursor()) as cursor:
cursor.execute(query)
results = cursor.fetchall()
for row in results:
yield package.Package(install_name=row[0])
def test__get_missing_details_for_self_compatibility(self):
from compatibility_lib import compatibility_store
from compatibility_lib import configs
from compatibility_lib import package
for package_name in configs.WHITELIST_PKGS:
results = []
if package_name not in ('tensorflow'):
results.append(
compatibility_store.CompatibilityResult(
packages=[package.Package(p) for p in package_name],
python_major_version=2,
status=compatibility_store.Status.SUCCESS))
if package_name not in ('apache-beam[gcp]', 'gsutil'):
results.append(
compatibility_store.CompatibilityResult(
packages=[package.Package(p) for p in package_name],
python_major_version=3,
status=compatibility_store.Status.SUCCESS))
details = main._get_missing_details([package_name], results)
self.assertEqual(details, None)
def test__get_missing_details_for_pair_compatibility(self):
from compatibility_lib import compatibility_store
from compatibility_lib import configs
from compatibility_lib import package
import itertools
for p1, p2 in itertools.combinations(configs.WHITELIST_PKGS, r=2):
pkgs = [p1, p2]
results = []
if all([p not in ('tensorflow') for p in pkgs]):
results.append(
compatibility_store.CompatibilityResult(
packages=[package.Package(p) for p in pkgs],
python_major_version=2,
status=compatibility_store.Status.SUCCESS))
if all([p not in ('apache-beam[gcp]', 'gsutil') for p in pkgs]):
results.append(
compatibility_store.CompatibilityResult(
packages=[package.Package(p) for p in pkgs],
python_major_version=3,
status=compatibility_store.Status.SUCCESS))
details = main._get_missing_details(pkgs, results)
self.assertEqual(details, None)
def test__get_missing_details_unsupported_packages(self):
from compatibility_lib import compatibility_store
from compatibility_lib import package
TENSORFLOW = 'tensorflow'
UNSUPPORTED = 'unsupported'
UNSUPPORTED_RESULT_PY2 = compatibility_store.CompatibilityResult(
packages=[package.Package(UNSUPPORTED)],
python_major_version=2,
status=compatibility_store.Status.UNKNOWN)
PAIR_RESULT_PY3 = compatibility_store.CompatibilityResult(
packages=[package.Package(p) for p in (TENSORFLOW, UNSUPPORTED)],
python_major_version=3,
status=compatibility_store.Status.UNKNOWN)
with self.assertRaises(AssertionError):
package_names = [UNSUPPORTED]
results = [UNSUPPORTED_RESULT_PY2]
main._get_missing_details(package_names, results)
with self.assertRaises(AssertionError):
package_names = [TENSORFLOW, UNSUPPORTED]
results = [PAIR_RESULT_PY3]
main._get_missing_details(package_names, results)
def test__get_missing_details_pair_fail(self):
from compatibility_lib import compatibility_store
from compatibility_lib import package
package_names = ['opencensus', 'compatibility-lib']
results = [
compatibility_store.CompatibilityResult(
packages=[package.Package(name) for name in package_names],
python_major_version=2,
status=compatibility_store.Status.SUCCESS)
]
details = main._get_missing_details(package_names, results)
expected_details = ("Missing data for packages=['opencensus', "
"'compatibility-lib'], versions=[3]")
self.assertEqual(details, expected_details)
def test__get_pair_status_for_packages_install_error(self):
from compatibility_lib import compatibility_store
from compatibility_lib import package
PACKAGE_1 = package.Package("package1")
PACKAGE_2 = package.Package("tensorflow")
pkg_sets = [
['package1', 'tensorflow'],
]
expected = {
'py2': {
'status': 'SUCCESS',
'details': {}
},
'py3': {
'status': 'SUCCESS',
'details': {}
}
}
cr_py2 = compatibility_store.CompatibilityResult(
packages=[PACKAGE_1, PACKAGE_2],
python_major_version=2,
status=compatibility_store.Status.INSTALL_ERROR)
cr_py3 = compatibility_store.CompatibilityResult(
packages=[PACKAGE_1, PACKAGE_2],
python_major_version=3,
status=compatibility_store.Status.SUCCESS)
pair_result = [cr_py2, cr_py3]
self.fake_store._packages_to_compatibility_result[frozenset(
[PACKAGE_1, PACKAGE_2])] = pair_result
with self.patch_checker, self.patch_store:
version_and_res = main._get_pair_status_for_packages(pkg_sets)
self.assertEqual(version_and_res, expected)
def _get_pair_status_for_packages(pkg_sets):
"""Get the pairwise dependency compatibility check result for packages.
Rules:
- Return warning status if not compatible with any of the listed
Google owned Python packages. Whole list in compatibility_lib.config.
- Ignore the warning status if:
- The package doesn't support one of the Python versions.
- The package's pair is not self compatible, which means the
pairwise conflict isn't related to the package being checked.
- Return success status if compatible with all the list of Google owned
Python packages.
"""
version_and_res = {
'py2': {
'status': 'SUCCESS',
'details': {},
},
'py3': {
'status': 'SUCCESS',
'details': {},
}
}
for pkg_set in pkg_sets:
pkgs = [package_module.Package(pkg) for pkg in pkg_set]
pair_res = badge_utils.store.get_pair_compatibility(pkgs)
for res in pair_res:
py_version = badge_utils.PY_VER_MAPPING[res.python_major_version]
# Status showing one of the check failures
if res.status.value != 'SUCCESS':
# Ignore the package that not support for given py_ver
if pkg_set[1] in \
configs.PKG_PY_VERSION_NOT_SUPPORTED.get(
res.python_major_version):
continue
# Ignore the package that are not self compatible
self_status = _get_result_from_cache(
package_name=pkg_set[1],
badge_type=badge_utils.BadgeType.SELF_COMP_BADGE)
if self_status[py_version]['status'] != 'SUCCESS':
continue
version_and_res[py_version]['status'] = res.status.value
version_and_res[py_version]['details'][pkg_set[1]] = \
res.details if res.details is not None \
else badge_utils.EMPTY_DETAILS
return version_and_res
def _get_self_compatibility_dict(package_name: str) -> dict:
"""Returns a dict containing self compatibility status and details.
Args:
package_name: the name of the package to check (e.g.
"google-cloud-storage").
Returns:
A dict containing the self compatibility status and details for any
self incompatibilities. The dict will be formatted like the following:
{
'py2': { 'status': BadgeStatus.SUCCESS, 'details': {} },
'py3': { 'status': BadgeStatus.SUCCESS, 'details': {} },
}
"""
pkg = package.Package(package_name)
compatibility_results = badge_utils.store.get_self_compatibility(pkg)
return _self_compatibilities_to_dict(package_name, compatibility_results)
def get_package_details(self, p: package.Package):
"""Return the dict of package check summary.
{
'self_conflict': True,
'pairwise_conflict': ['package1', 'package2'],
'latest_version': '0.1.0',
}
"""
# The package being checked will appear in the dep list, but for
# apache-beam[gcp], the package in dep list will just be apache-beam.
self_dep_name = p.install_name
if '[' in p.install_name:
self_dep_name = p.install_name.split('[')[0]
latest_version = self._package_with_dependency_info[
p.install_name][self_dep_name]['latest_version']
pairwise_conflict = []
# Initialize the values
result = {
'self_conflict': False,
'pairwise_conflict': pairwise_conflict,
'latest_version': latest_version,
}
for pair_pkg in configs.PKG_LIST:
check_result = self.get_result(p, package.Package(pair_pkg))
# Get self compatibility status
if pair_pkg == p.install_name:
result['self_conflict'] = True \
if check_result['status_type'] != 'self-success' else False
# Get pairwise compatibility status
else:
if check_result['status_type'] != 'pairwise-success' \
and self.has_issues(p):
pairwise_conflict.append(pair_pkg)
result['pairwise_conflict'] = pairwise_conflict
return result
def _get_pair_status_for_packages(pkg_sets):
version_and_res = {
'py2': {
'status': 'SUCCESS',
'details': {},
},
'py3': {
'status': 'SUCCESS',
'details': {},
}
}
for pkg_set in pkg_sets:
pkgs = [package_module.Package(pkg) for pkg in pkg_set]
pair_res = store.get_pair_compatibility(pkgs)
for res in pair_res:
py_version = PY_VER_MAPPING[res.python_major_version]
# Status showing one of the check failures
if res.status.value != 'SUCCESS':
version_and_res[py_version]['status'] = res.status.value
version_and_res[py_version]['details'][pkg_set[1]] = \
res.details if res.details is not None else EMPTY_DETAILS
return version_and_res
def _result_dict_to_compatibility_result(results, python_version):
res_list = []
for item in results:
res_dict = item[0]
check_result = res_dict.get('result')
packages_list = [
package.Package(pkg) for pkg in res_dict.get('packages')
]
details = res_dict.get('description')
timestamp = datetime.datetime.now().isoformat()
dependency_info = res_dict.get('dependency_info')
compatibility_result = compatibility_store.CompatibilityResult(
packages=packages_list,
python_major_version=python_version,
status=compatibility_store.Status(check_result),
details=details,
timestamp=timestamp,
dependency_info=dependency_info)
res_list.append(compatibility_result)
return res_list
def _get_self_compatibility_dict(package_name: str) -> dict:
"""Returns a dict containing self compatibility status and details.
Args:
package_name: the name of the package to check (e.g.
"google-cloud-storage").
Returns:
A dict containing the self compatibility status and details for any
self incompatibilities. The dict will be formatted like the following:
{
'py2': { 'status': BadgeStatus.SUCCESS, 'details': {} },
'py3': { 'status': BadgeStatus.SUCCESS, 'details': {} },
}
"""
pkg = package.Package(package_name)
compatibility_results = badge_utils.store.get_self_compatibility(pkg)
missing_details = _get_missing_details([package_name],
compatibility_results)
if missing_details:
result_dict = badge_utils._build_default_result(
status=BadgeStatus.MISSING_DATA, details=missing_details)
return result_dict
result_dict = badge_utils._build_default_result(
status=BadgeStatus.SUCCESS,
details='The package does not support this version of python.')
for res in compatibility_results:
pyver = badge_utils.PY_VER_MAPPING[res.python_major_version]
badge_status = PACKAGE_STATUS_TO_BADGE_STATUS.get(
res.status) or BadgeStatus.SELF_INCOMPATIBLE
result_dict[pyver]['status'] = badge_status
result_dict[pyver]['details'] = res.details
if res.details is None:
result_dict[pyver]['details'] = badge_utils.EMPTY_DETAILS
return result_dict
def main():
parser = argparse.ArgumentParser(
description='Display a grid show the dependency compatibility ' +
'between Python packages')
parser.add_argument('--packages',
nargs='+',
default=_DEFAULT_INSTALL_NAMES,
help='the packages to display compatibility ' +
'information for')
parser.add_argument('--browser',
action='store_true',
default=False,
help='display the grid in a browser tab')
args = parser.parse_args()
checker = compatibility_checker.CompatibilityChecker()
store = compatibility_store.CompatibilityStore()
instance_flag = '-instances={}=tcp:{}'.format(INSTANCE_CONNECTION_NAME,
PORT)
cloud_sql_proxy_path = './cloud_sql_proxy'
try:
# Run cloud_sql_proxy
process = popen_spawn.PopenSpawn([cloud_sql_proxy_path, instance_flag])
process.expect('Ready for new connection', timeout=5)
packages = [
package.Package(install_name) for install_name in args.packages
]
logging.info('Getting self compatibility results...')
package_to_results = store.get_self_compatibilities(packages)
logging.info('Getting pairwise compatibility results...')
pairwise_to_results = store.get_compatibility_combinations(packages)
package_with_dependency_info = {}
for pkg in configs.PKG_LIST:
dep_info = store.get_dependency_info(pkg)
package_with_dependency_info[pkg] = dep_info
results = _ResultHolder(package_to_results, pairwise_to_results,
package_with_dependency_info, checker, store)
dashboard_builder = DashboardBuilder(packages, results)
# Build the pairwise grid dashboard
logging.info('Starting build the grid...')
grid_html = dashboard_builder.build_dashboard(
'dashboard/grid-template.html')
grid_path = os.path.dirname(os.path.abspath(__file__)) + '/grid.html'
with open(grid_path, 'wt') as f:
f.write(grid_html)
# Build the dashboard main page
logging.info('Starting build the main dashboard...')
main_html = dashboard_builder.build_dashboard(
'dashboard/main-template.html')
main_path = os.path.dirname(os.path.abspath(__file__)) + '/index.html'
with open(main_path, 'wt') as f:
f.write(main_html)
except Exception:
raise DashboardBuilderError('Error occurs when building dashboard.'
'Output: {}'.format(process.before))
finally:
process.kill(signal.SIGTERM)
if args.browser:
webbrowser.open_new_tab('file://' + main_path)
