def _get_third_party_python_libs_directory_contents(): """Returns a dictionary containing all of the normalized libraries name strings with their corresponding version strings installed in the 'third_party/python_libs' directory. Returns: dict(str, str). Dictionary with the normalized name of the library installed as the key and the version string of that library as the value. """ direct_url_packages, standard_packages = utils.partition( pkg_resources.find_distributions(common.THIRD_PARTY_PYTHON_LIBS_DIR), predicate=lambda dist: dist.has_metadata('direct_url.json')) installed_packages = { pkg.project_name: pkg.version for pkg in standard_packages } for pkg in direct_url_packages: metadata = json.loads(pkg.get_metadata('direct_url.json')) version_string = '%s+%s@%s' % ( metadata['vcs_info']['vcs'], metadata['url'], metadata['vcs_info']['commit_id']) installed_packages[pkg.project_name] = version_string # Libraries with different case are considered equivalent libraries: # e.g 'Flask' is the same library as 'flask'. Therefore, we # normalize all library names in order to compare libraries without # ambiguities. directory_contents = { normalize_python_library_name(library_name): version_string for library_name, version_string in installed_packages.items() } return directory_contents
def test_partition(self) -> None: is_even = lambda n: (n % 2) == 0 evens, odds = (utils.partition([10, 8, 1, 5, 6, 4, 3, 7], predicate=is_even)) self.assertEqual(list(evens), [10, 8, 6, 4]) self.assertEqual(list(odds), [1, 5, 3, 7])
def test_enumerated_partition(self) -> None: logs = ['ERROR: foo', 'INFO: bar', 'INFO: fee', 'ERROR: fie'] is_error = lambda msg: msg.startswith('ERROR: ') errors, others = ( utils.partition(logs, predicate=is_error, enumerated=True)) self.assertEqual(list(errors), [(0, 'ERROR: foo'), (3, 'ERROR: fie')]) self.assertEqual(list(others), [(1, 'INFO: bar'), (2, 'INFO: fee')])
def _rectify_third_party_directory(mismatches): """Rectifies the 'third_party/python_libs' directory state to reflect the current 'requirements.txt' file requirements. It takes a list of mismatches and corrects those mismatches by installing or uninstalling packages. Args: mismatches: dict(str, tuple(str|None, str|None)). Dictionary with the normalized library names as keys and a tuple as values. The 1st element of the tuple is the version string of the library required by the requirements.txt file while the 2nd element is the version string of the library currently installed in the 'third_party/python_libs' directory. If the library doesn't exist, the corresponding tuple element will be None. For example, this dictionary signifies that 'requirements.txt' requires flask with version 1.0.1 while the 'third_party/python_libs' directory contains flask 1.1.1: { flask: ('1.0.1', '1.1.1') } """ # Handling 5 or more mismatches requires 5 or more individual `pip install` # commands, which is slower than just reinstalling all of the libraries # using `pip install -r requirements.txt`. if len(mismatches) >= 5: if os.path.isdir(common.THIRD_PARTY_PYTHON_LIBS_DIR): shutil.rmtree(common.THIRD_PARTY_PYTHON_LIBS_DIR) _reinstall_all_dependencies() return # The library is installed in the directory but is not listed in # requirements. We don't have functionality to remove a library cleanly, and # if we ignore the library, this might cause issues when pushing the branch # to develop as there might be possible hidden use cases of a deleted # library that the developer did not catch. The only way to enforce the # removal of a library is to clean out the folder and reinstall everything # from scratch. if any(required is None for required, _ in mismatches.values()): if os.path.isdir(common.THIRD_PARTY_PYTHON_LIBS_DIR): shutil.rmtree(common.THIRD_PARTY_PYTHON_LIBS_DIR) _reinstall_all_dependencies() return git_mismatches, pip_mismatches = (utils.partition( mismatches.items(), predicate=_is_git_url_mismatch)) for normalized_library_name, versions in git_mismatches: requirements_version, directory_version = versions # The library listed in 'requirements.txt' is not in the # 'third_party/python_libs' directory. if not directory_version or requirements_version != directory_version: _install_direct_url(normalized_library_name, requirements_version) for normalized_library_name, versions in pip_mismatches: requirements_version = (pkg_resources.parse_version(versions[0]) if versions[0] else None) directory_version = (pkg_resources.parse_version(versions[1]) if versions[1] else None) # The library listed in 'requirements.txt' is not in the # 'third_party/python_libs' directory. if not directory_version: _install_library(normalized_library_name, python_utils.UNICODE(requirements_version)) # The currently installed library version is not equal to the required # 'requirements.txt' version. elif requirements_version != directory_version: _install_library(normalized_library_name, python_utils.UNICODE(requirements_version)) _remove_metadata(normalized_library_name, python_utils.UNICODE(directory_version))