def _get_third_party_python_libs_directory_contents():
"""Returns a dictionary containing all of the normalized libraries name
strings with their corresponding version strings installed in the
'third_party/python_libs' directory.
Returns:
dict(str, str). Dictionary with the normalized name of the library
installed as the key and the version string of that library as the
value.
"""
direct_url_packages, standard_packages = utils.partition(
pkg_resources.find_distributions(common.THIRD_PARTY_PYTHON_LIBS_DIR),
predicate=lambda dist: dist.has_metadata('direct_url.json'))
installed_packages = {
pkg.project_name: pkg.version for pkg in standard_packages
}
for pkg in direct_url_packages:
metadata = json.loads(pkg.get_metadata('direct_url.json'))
version_string = '%s+%s@%s' % (
metadata['vcs_info']['vcs'], metadata['url'],
metadata['vcs_info']['commit_id'])
installed_packages[pkg.project_name] = version_string
# Libraries with different case are considered equivalent libraries:
# e.g 'Flask' is the same library as 'flask'. Therefore, we
# normalize all library names in order to compare libraries without
# ambiguities.
directory_contents = {
normalize_python_library_name(library_name): version_string
for library_name, version_string in installed_packages.items()
}
return directory_contents
def test_partition(self) -> None:
is_even = lambda n: (n % 2) == 0
evens, odds = (utils.partition([10, 8, 1, 5, 6, 4, 3, 7],
predicate=is_even))
self.assertEqual(list(evens), [10, 8, 6, 4])
self.assertEqual(list(odds), [1, 5, 3, 7])
def test_enumerated_partition(self) -> None:
logs = ['ERROR: foo', 'INFO: bar', 'INFO: fee', 'ERROR: fie']
is_error = lambda msg: msg.startswith('ERROR: ')
errors, others = (
utils.partition(logs, predicate=is_error, enumerated=True))
self.assertEqual(list(errors), [(0, 'ERROR: foo'), (3, 'ERROR: fie')])
self.assertEqual(list(others), [(1, 'INFO: bar'), (2, 'INFO: fee')])
def _rectify_third_party_directory(mismatches):
"""Rectifies the 'third_party/python_libs' directory state to reflect the
current 'requirements.txt' file requirements. It takes a list of mismatches
and corrects those mismatches by installing or uninstalling packages.
Args:
mismatches: dict(str, tuple(str|None, str|None)). Dictionary
with the normalized library names as keys and a tuple as values. The
1st element of the tuple is the version string of the library
required by the requirements.txt file while the 2nd element is the
version string of the library currently installed in the
'third_party/python_libs' directory. If the library doesn't exist,
the corresponding tuple element will be None. For example, this
dictionary signifies that 'requirements.txt' requires flask with
version 1.0.1 while the 'third_party/python_libs' directory contains
flask 1.1.1:
{
flask: ('1.0.1', '1.1.1')
}
"""
# Handling 5 or more mismatches requires 5 or more individual `pip install`
# commands, which is slower than just reinstalling all of the libraries
# using `pip install -r requirements.txt`.
if len(mismatches) >= 5:
if os.path.isdir(common.THIRD_PARTY_PYTHON_LIBS_DIR):
shutil.rmtree(common.THIRD_PARTY_PYTHON_LIBS_DIR)
_reinstall_all_dependencies()
return
# The library is installed in the directory but is not listed in
# requirements. We don't have functionality to remove a library cleanly, and
# if we ignore the library, this might cause issues when pushing the branch
# to develop as there might be possible hidden use cases of a deleted
# library that the developer did not catch. The only way to enforce the
# removal of a library is to clean out the folder and reinstall everything
# from scratch.
if any(required is None for required, _ in mismatches.values()):
if os.path.isdir(common.THIRD_PARTY_PYTHON_LIBS_DIR):
shutil.rmtree(common.THIRD_PARTY_PYTHON_LIBS_DIR)
_reinstall_all_dependencies()
return
git_mismatches, pip_mismatches = (utils.partition(
mismatches.items(), predicate=_is_git_url_mismatch))
for normalized_library_name, versions in git_mismatches:
requirements_version, directory_version = versions
# The library listed in 'requirements.txt' is not in the
# 'third_party/python_libs' directory.
if not directory_version or requirements_version != directory_version:
_install_direct_url(normalized_library_name, requirements_version)
for normalized_library_name, versions in pip_mismatches:
requirements_version = (pkg_resources.parse_version(versions[0])
if versions[0] else None)
directory_version = (pkg_resources.parse_version(versions[1])
if versions[1] else None)
# The library listed in 'requirements.txt' is not in the
# 'third_party/python_libs' directory.
if not directory_version:
_install_library(normalized_library_name,
python_utils.UNICODE(requirements_version))
# The currently installed library version is not equal to the required
# 'requirements.txt' version.
elif requirements_version != directory_version:
_install_library(normalized_library_name,
python_utils.UNICODE(requirements_version))
_remove_metadata(normalized_library_name,
python_utils.UNICODE(directory_version))
