def repository_factory(session, indices, quiet=False, raise_on_error=False): unavailables = [] full_repository = Repository() _write_and_flush("Fetching indices: ", quiet) with ThreadPoolExecutor(max_workers=4) as executor: tasks = [] for url, store_location in indices: task = executor.submit(_fetch_repository, session, url, store_location, raise_on_error) tasks.append(task) for task in as_completed(tasks): repository_or_none = task.result() if repository_or_none is None: unavailables.append(store_location) else: for package in repository_or_none.iter_packages(): full_repository.add_package(package) _write_and_flush(".", quiet) _write_and_flush("\n\n", quiet) if len(unavailables) > 0 and not quiet: _print_unavailables_warning(unavailables) return full_repository
def repository_factory_from_egg(filenames): repository = Repository() for filename in filenames: path = os.path.join(_EGGINST_COMMON_DATA, filename) package = RemotePackageMetadata.from_egg(path) repository.add_package(package) return repository
def repository_factory(session, repository_infos, quiet=False, raise_on_error=False): unavailables = [] full_repository = Repository() _write_and_flush("Fetching indices: ", quiet) with ThreadPoolExecutor(max_workers=4) as executor: tasks = {} for repository_info in repository_infos: task = executor.submit(_fetch_repository, session, repository_info, raise_on_error) tasks[task] = repository_info for task in as_completed(tasks): repository_or_none = task.result() if repository_or_none is None: unavailables.append(tasks[task]) else: for package in repository_or_none.iter_packages(): full_repository.add_package(package) _write_and_flush(".", quiet) _write_and_flush("\n\n", quiet) if len(unavailables) > 0 and not quiet: _print_unavailables_warning(unavailables) return full_repository
def test_find_packages_sorting(self): # Given entries = [ dummy_installed_package_factory("numpy", "1.6.1", 1), dummy_installed_package_factory("numpy", "1.8.0", 2), dummy_installed_package_factory("numpy", "1.7.1", 1), ] repository = Repository() for entry in entries: repository.add_package(entry) r_versions = [ EnpkgVersion.from_string(v) for v in ("1.6.1-1", "1.7.1-1", "1.8.0-2") ] # When packages = repository.find_packages("numpy") # Then self.assertEqual(len(packages), 3) self.assertEqual([p.version for p in packages], r_versions) with self.assertWarns(DeprecationWarning): deprecated_packages = repository.find_sorted_packages("numpy") self.assertEqual([p.version for p in deprecated_packages], r_versions)
def repository_factory(session, indices): """ Create a repository from legacy indices. Parameters ---------- session : Session A :py:class:`~enstaller.session.Session` instance. indices : list Sequence of (index_url, store_name) pairs, e.g. the indices property of :py:class:`~enstaller.config.Configuration` instances. Note ---- This does not use etag caching nor write anything in sys.stdout. If you want to use etag caching, simply do:: with session.etag(): repository = repository_factory(session, ...) """ repository = Repository() for url, store_location in indices: resp = session.fetch(url) json_data = resp.json() for package in parse_index(json_data, store_location): repository.add_package(package) return repository
def test_simple_installed(self): # Given entries = [ dummy_repository_package_factory("MKL", "10.3", 1), dummy_repository_package_factory("numpy", "1.8.0", 2, dependencies=["MKL 10.3"]), ] repository = repository_factory(entries) installed_repository = Repository() installed_repository.add_package( dummy_installed_package_factory("MKL", "10.3", 1)) expected_actions = [ ('install', "numpy-1.8.0-2.egg"), ] # When request = Request() request.install(Requirement("numpy")) solver = Solver(repository, installed_repository) actions = solver.resolve(request) # Then self.assertListEqual(actions, expected_actions)
def repository_factory(package_names, repository_info, reference_packages): repository = Repository() for package_name in package_names: package = reference_packages[package_name] package = RepositoryPackageMetadata.from_package(package, repository_info) repository.add_package(package) return repository
def test_ctor(self): # When repository = Repository() # Then self.assertEqual(len(repository), 0) # Given eggs = [ "dummy-1.0.1-1.egg", "dummy_with_appinst-1.0.0-1.egg", "dummy_with_entry_points-1.0.0-1.egg", "dummy_with_proxy-1.3.40-3.egg", "dummy_with_proxy_scripts-1.0.0-1.egg", "dummy_with_proxy_softlink-1.0.0-1.egg", "nose-1.2.1-1.egg", "nose-1.3.0-1.egg", "nose-1.3.0-2.egg", ] paths = (os.path.join(_EGGINST_COMMON_DATA, egg) for egg in eggs) packages = [RepositoryPackageMetadata.from_egg(path) for path in paths] # When repository = Repository(packages) # Then self.assertEqual(len(repository), len(eggs))
def test_simple_installed(self): # Given entries = [ dummy_repository_package_factory("MKL", "10.3", 1), dummy_repository_package_factory("numpy", "1.8.0", 2, dependencies=["MKL 10.3"]), ] repository = repository_factory(entries) installed_repository = Repository() installed_repository.add_package( dummy_installed_package_factory("MKL", "10.3", 1) ) expected_actions = [ ('install', "numpy-1.8.0-2.egg"), ] # When request = Request() request.install(Requirement("numpy")) solver = Solver(repository, installed_repository) actions = solver.resolve(request) # Then self.assertListEqual(actions, expected_actions)
def __init__(self, remote_repository, session, prefixes=[sys.prefix], progress_context=None, force=False, max_retries=_DEFAULT_MAX_RETRIES, runtime_info=None): self.prefixes = prefixes self.top_prefix = prefixes[0] self._runtime_info = ( runtime_info or _default_runtime_info(self.top_prefix) ) self._remote_repository = remote_repository self._installed_repository = Repository._from_prefixes(self.prefixes) self._top_installed_repository = \ Repository._from_prefixes([self.top_prefix]) self._session = session self._downloader = _DownloadManager(session, remote_repository) self._progress_context = progress_context or \ ProgressBarContext(dummy_progress_bar_factory) self._force = force self.max_retries = max_retries
def test_simple_scenario(self): egg = DUMMY_EGG r_actions = {1: [], 0: [("remove", os.path.basename(egg))]} config = Configuration() repository = Repository() package = RepositoryPackageMetadata.from_egg(egg) package.python = PY_VER repository.add_package(package) with open(egg, "rb") as fp: responses.add(responses.GET, package.source_url, body=fp.read(), status=200, content_type='application/json') session = Session(DummyAuthenticator(), config.repository_cache) enpkg = Enpkg(repository, session, prefixes=self.prefixes) actions = [("fetch", os.path.basename(egg)), ("install", os.path.basename(egg))] enpkg.execute(actions) name, version = egg_name_to_name_version(egg) enpkg._installed_repository.find_package(name, version) for state in [0, 1]: actions = enpkg.revert_actions(state) self.assertEqual(actions, r_actions[state])
def _downloader_factory(self, paths): repository = Repository() for path in paths: package = RepositoryPackageMetadata.from_egg(path) repository.add_package(package) return (_DownloadManager(Session(DummyAuthenticator(), self.tempdir), repository), repository)
def _create_store_and_repository(self, eggs): repository = Repository() for egg in eggs: path = os.path.join(_EGGINST_COMMON_DATA, egg) package = RepositoryPackageMetadata.from_egg(path) repository.add_package(package) return repository
def test_delete_non_existing(self): # Given path = os.path.join(_EGGINST_COMMON_DATA, "nose-1.3.0-1.egg") to_remove = PackageMetadata.from_egg(path) repository = Repository() # When/Then with self.assertRaises(NoSuchPackage): repository.delete_package(to_remove)
def _install_eggs(self, paths): repository = Repository() for path in paths: package = RepositoryPackageMetadata.from_egg(path) repository.add_package(package) for path in paths: action = InstallAction(path, self.top_prefix, repository, self.top_installed_repository, self.installed_repository, os.path.dirname(path)) action.execute()
def remote_enstaller_available(versions): repository = Repository() for version in versions: package = dummy_repository_package_factory("enstaller", version, build=1) repository.add_package(package) def dec(f): @functools.wraps(f) def wrapper(*a, **kw): with mock.patch("enstaller.main.repository_factory", return_value=repository): return f(*a, **kw) return wrapper return dec
def _retry_common_setup(self): store_url = "http://acme.com" repository_info = OldstyleRepositoryInfo(store_url) filename = "nose-1.3.0-1.egg" path = os.path.join(_EGGINST_COMMON_DATA, filename) repository = Repository() package = RemotePackageMetadata.from_egg(path, repository_info) repository.add_package(package) downloader = _DownloadManager(mocked_session_factory(self.tempdir), repository) return path, downloader, repository
def test_fetch_invalid_md5(self): # Given filename = "nose-1.3.0-1.egg" path = os.path.join(_EGGINST_COMMON_DATA, filename) repository = Repository() package = RepositoryPackageMetadata.from_egg(path) package.md5 = "a" * 32 repository.add_package(package) downloader = _DownloadManager(mocked_session_factory(self.tempdir), repository) with self.assertRaises(InvalidChecksum): downloader.fetch(filename)
def _retry_common_setup(self): store_location = "http://acme.com/" filename = "nose-1.3.0-1.egg" path = os.path.join(_EGGINST_COMMON_DATA, filename) repository = Repository() package = RepositoryPackageMetadata.from_egg(path, store_location=store_location) repository.add_package(package) downloader = _DownloadManager(mocked_session_factory(self.tempdir), repository) return path, downloader, repository
def test_simple_all_installed(self): # Given entries = [ dummy_repository_package_factory("MKL", "10.3", 1), dummy_repository_package_factory("numpy", "1.8.0", 2, dependencies=["MKL 10.3"]), ] repository = repository_factory(entries) installed_entries = [ dummy_installed_package_factory("MKL", "10.3", 1), dummy_installed_package_factory("numpy", "1.8.0", 2) ] installed_repository = Repository() for package in installed_entries: installed_repository.add_package(package) # When request = Request() request.install(Requirement("numpy")) solver = Solver(repository, installed_repository) actions = solver.resolve(request) # Then self.assertListEqual(actions, []) # When solver = Solver(repository, installed_repository, force=True) actions = solver.resolve(request) # Then self.assertListEqual(actions, [("remove", "numpy-1.8.0-2.egg"), ("install", "numpy-1.8.0-2.egg")]) # When solver = Solver(repository, installed_repository, force=True, forceall=True) actions = solver.resolve(request) # Then self.assertListEqual(actions, [("remove", "numpy-1.8.0-2.egg"), ("remove", "MKL-10.3-1.egg"), ("install", "MKL-10.3-1.egg"), ("install", "numpy-1.8.0-2.egg")])
def test_simple_all_installed(self): # Given remote_mkl = dummy_repository_package_factory("MKL", "10.3", 1) remote_numpy = dummy_repository_package_factory( "numpy", "1.8.0", 2, dependencies=["MKL 10.3"] ) entries = [remote_mkl, remote_numpy] repository = repository_factory(entries) installed_mkl = dummy_installed_package_factory("MKL", "10.3", 1) installed_numpy = dummy_installed_package_factory("numpy", "1.8.0", 2) installed_entries = [installed_mkl, installed_numpy] installed_repository = Repository() for package in installed_entries: installed_repository.add_package(package) # When request = Request() request.install(Requirement("numpy")) solver = Solver(repository, installed_repository) actions = solver.resolve(request) # Then self.assertListEqual(actions, []) # When solver = Solver( repository, installed_repository, force=ForceMode.MAIN_ONLY ) actions = solver.resolve(request) # Then self.assertListEqual( actions, [("remove", installed_numpy), ("install", remote_numpy)] ) # When solver = Solver(repository, installed_repository, force=ForceMode.ALL) actions = solver.resolve(request) # Then self.assertListEqual( actions, [("remove", installed_numpy), ("remove", installed_mkl), ("install", remote_mkl), ("install", remote_numpy)] )
def test_print_install_time(self): with mkdtemp(): installed_entries = [dummy_installed_package_factory("dummy", "1.0.1", 1)] installed_repository = Repository() for package in installed_entries: installed_repository.add_package(package) self.assertRegexpMatches(install_time_string(installed_repository, "dummy"), "dummy-1.0.1-1.egg was installed on:") self.assertEqual(install_time_string(installed_repository, "ddummy"), "")
def test_print_install_time(self): with mkdtemp(): installed_entries = [ dummy_installed_package_factory("dummy", "1.0.1", 1) ] installed_repository = Repository() for package in installed_entries: installed_repository.add_package(package) self.assertRegexpMatches( install_time_string(installed_repository, "dummy"), "dummy-1.0.1-1.egg was installed on:") self.assertEqual( install_time_string(installed_repository, "ddummy"), "")
def test_find_packages_invalid_versions(self): # Given entries = [ dummy_installed_package_factory("numpy", "1.6.1", 1), dummy_installed_package_factory("numpy", "1.8k", 2), ] repository = Repository() for entry in entries: repository.add_package(entry) # When packages = repository.find_packages("numpy") # Then self.assertEqual(len(packages), 2) assertCountEqual(self, packages, entries)
def test_iter_most_recent_packages(self): # Given eggs = ["nose-1.3.0-1.egg", "nose-1.2.1-1.egg"] repository = Repository() for egg in eggs: path = os.path.join(_EGGINST_COMMON_DATA, egg) package = RepositoryPackageMetadata.from_egg(path) repository.add_package(package) # When metadata = list(repository.iter_most_recent_packages()) # Then self.assertEqual(len(metadata), 1) self.assertEqual(metadata[0].version, EnpkgVersion.from_string("1.3.0-1"))
def test_sorted_insertion(self): # Given eggs = ["nose-1.3.0-1.egg", "nose-1.2.1-1.egg"] repository = Repository() # When for egg in eggs: path = os.path.join(_EGGINST_COMMON_DATA, egg) package = RemotePackageMetadata.from_egg(path) repository.add_package(package) # Then self.assertEqual([m.version for m in repository._name_to_packages["nose"]], [EnpkgVersion.from_string("1.2.1-1"), EnpkgVersion.from_string("1.3.0-1")])
def setUp(self): eggs = [ "dummy-1.0.1-1.egg", "dummy_with_appinst-1.0.0-1.egg", "dummy_with_entry_points-1.0.0-1.egg", "dummy_with_proxy-1.3.40-3.egg", "dummy_with_proxy_scripts-1.0.0-1.egg", "dummy_with_proxy_softlink-1.0.0-1.egg", "nose-1.2.1-1.egg", "nose-1.3.0-1.egg", "nose-1.3.0-2.egg", ] self.repository = Repository() for egg in eggs: path = os.path.join(_EGGINST_COMMON_DATA, egg) package = RepositoryPackageMetadata.from_egg(path) self.repository.add_package(package)
def test_iter_protocol(self): # Given eggs = ["nose-1.3.0-1.egg", "nose-1.2.1-1.egg"] repository = Repository() for egg in eggs: path = os.path.join(_EGGINST_COMMON_DATA, egg) package = RemotePackageMetadata.from_egg(path) repository.add_package(package) # When metadata = list(iter(repository)) # Then self.assertEqual(len(metadata), 2) self.assertEqual(set(m.version for m in metadata), set([EnpkgVersion.from_string("1.2.1-1"), EnpkgVersion.from_string("1.3.0-1")]))
def test_no_update_enstaller(self): config = Configuration() session = mocked_session_factory(config.repository_cache) opts = mock.Mock() opts.yes = False self.assertFalse(update_enstaller(session, Repository(), opts))
def _create_repositories(self, entries, installed_entries): repository = Repository() for entry in entries: repository.add_package(entry) installed_repository = Repository() for entry in installed_entries: installed_repository.add_package(entry) return repository, installed_repository
def test_sorted_packages_invalid(self): # Given entries = [ dummy_installed_package_factory("numpy", "1.6.1", 1), dummy_installed_package_factory("numpy", "1.8k", 2), ] repository = Repository() for entry in entries: repository.add_package(entry) # When packages = repository.find_sorted_packages("numpy") # Then self.assertEqual(len(packages), 2) assertCountEqual( self, [p.version for p in packages], [EnpkgVersion.from_string(v) for v in ("1.6.1-1", "1.8k-2")])
def test_from_empty_prefix(self): # Given with mkdtemp() as tempdir: # When repository = Repository._from_prefixes([tempdir]) # Then self.assertEqual(len(list(repository.iter_packages())), 0)
def test_sorted_packages_invalid(self): # Given entries = [ dummy_installed_package_factory("numpy", "1.6.1", 1), dummy_installed_package_factory("numpy", "1.8k", 2), ] repository = Repository() for entry in entries: repository.add_package(entry) # When packages = repository.find_sorted_packages("numpy") # Then self.assertEqual(len(packages), 2) assertCountEqual(self, [p.version for p in packages], [EnpkgVersion.from_string(v) for v in ("1.6.1-1", "1.8k-2")])
def test_remove_actions(self): # Given repository = Repository() for egg in [DUMMY_EGG]: egginst = EggInst(egg, self.prefix) egginst.install() solver = Solver(repository, Repository._from_prefixes([self.prefix])) request = Request() request.remove(Requirement("dummy")) # When actions = solver.resolve(request) # Then self.assertEqual(actions, [("remove", os.path.basename(DUMMY_EGG))])
def test_update(self): # Given def repository_factory_from_egg(filenames): repository = Repository() for filename in filenames: path = os.path.join(_EGGINST_COMMON_DATA, filename) package = RemotePackageMetadata.from_egg(path) repository.add_package(package) return repository egg_set1 = ( "dummy-1.0.1-1.egg", "dummy_with_appinst-1.0.0-1.egg", "dummy_with_entry_points-1.0.0-1.egg", "dummy_with_proxy-1.3.40-3.egg", ) egg_set2 = ( "dummy_with_proxy_scripts-1.0.0-1.egg", "dummy_with_proxy_softlink-1.0.0-1.egg", "nose-1.2.1-1.egg", "nose-1.3.0-1.egg", "nose-1.3.0-2.egg", ) repository = Repository() repository1 = repository_factory_from_egg(egg_set1) repository2 = repository_factory_from_egg(egg_set2) # When repository.update(repository1) # Then assertCountEqual(self, iter(repository), iter(repository1)) # When repository.update(repository2) # Then assertCountEqual( self, iter(repository), itertools.chain(iter(repository1), iter(repository2)) )
def __init__(self, remote_repository, session, prefixes=[sys.prefix], progress_context=None, force=False, max_retries=_DEFAULT_MAX_RETRIES): self.prefixes = prefixes self.top_prefix = prefixes[0] self._remote_repository = remote_repository self._installed_repository = Repository._from_prefixes(self.prefixes) self._top_installed_repository = \ Repository._from_prefixes([self.top_prefix]) self._session = session self._downloader = _DownloadManager(session, remote_repository) self._progress_context = progress_context or \ ProgressBarContext(dummy_progress_bar_factory) self._force = force self.max_retries = max_retries
def test_imports_option_sys_only(self): # Given r_output = textwrap.dedent("""\ Name Version Location ============================================================ dummy 1.0.1-1 sys """) repository = Repository() metadata = InstalledPackageMetadata.from_egg(DUMMY_EGG, "random string", sys.prefix) repository.add_package(metadata) # When with mock_print() as m: imports_option(repository) # Then self.assertMultiLineEqual(m.value, r_output)
def test_fetch_unauthorized(self): # Given filename = "nose-1.3.0-1.egg" url = "http://api.enthought.com/eggs/yoyo/" repository = Repository() path = os.path.join(_EGGINST_COMMON_DATA, filename) package = RepositoryPackageMetadata.from_egg(path, url) repository.add_package(package) responses.add(responses.GET, url + filename, body='{"error": "forbidden"}', status=403) downloader = _DownloadManager(mocked_session_factory(self.tempdir), repository) # When/Then with self.assertRaises(requests.exceptions.HTTPError): downloader.fetch(filename)
def unconnected_enpkg_factory(prefixes=None): """ Create an Enpkg instance which does not require an authenticated repository. """ if prefixes is None: prefixes = [sys.prefix] config = Configuration() repository = Repository() return Enpkg(repository, mocked_session_factory(config.repository_cache), prefixes=prefixes)
def create_repositories(remote_entries=None, installed_entries=None): if remote_entries is None: remote_entries = [] if installed_entries is None: installed_entries = [] remote_repository = Repository() for remote_entry in remote_entries: remote_repository.add_package(remote_entry) installed_repository = Repository() for installed_entry in installed_entries: installed_repository.add_package(installed_entry) return remote_repository, installed_repository
def test_install_not_available(self): # Given config = Configuration() config.update(auth=FAKE_AUTH) nose = dummy_repository_package_factory("nose", "1.3.0", 1) nose.available = False repository = Repository() repository.add_package(nose) enpkg = Enpkg(repository, mocked_session_factory(config.repository_cache), [self.prefix]) enpkg.execute = mock.Mock() # When/Then with mock.patch("enstaller.config.subscription_message") as \ subscription_message: with self.assertRaises(SystemExit) as e: install_req(enpkg, config, "nose", FakeOptions()) subscription_message.assert_called() self.assertEqual(e.exception.code, 1)
def _fetch_repository(session, url, store_location, raise_on_error): with session.etag(): resp = session.get(url, stream=True) if resp.status_code != 200: if _should_raise(resp, raise_on_error): resp.raise_for_status() else: return None # failed.append(store_location) else: data = io.BytesIO() for chunk in _ResponseIterator(resp): data.write(chunk) json_data = decode_json_from_buffer(data.getvalue()) return Repository(parse_index(json_data, store_location))
def test_imports_option_empty(self): # Given r_output = textwrap.dedent("""\ Name Version Location ============================================================ """) repository = Repository() # When with mock_print() as m: imports_option(repository) # Then self.assertMultiLineEqual(m.value, r_output)
def get_freeze_list(prefixes): """ Compute the list of eggs installed in the given prefixes. Returns ------- names: seq List of installed eggs, as full names (e.g. 'numpy-1.8.0-1') """ full_names = [ "{0} {1}".format(package.name, package.full_version) for package in Repository._from_prefixes(prefixes).iter_packages() ] return sorted(full_names)
def test_print_installed(self): with mkdtemp() as d: r_out = textwrap.dedent("""\ Name Version Store ============================================================ dummy 1.0.1-1 {0} """.format(disp_store_info(d))) ec = EggInst(DUMMY_EGG, d) ec.install() repository = Repository._from_prefixes([d]) with mock_print() as m: print_installed(repository) self.assertMultiLineEqual(m.value, r_out) r_out = textwrap.dedent("""\ Name Version Store ============================================================ """) repository = Repository._from_prefixes([d]) with mock_print() as m: print_installed(repository, pat=re.compile("no_dummy")) self.assertEqual(m.value, r_out)
def test_from_prefix(self): # Given path = DUMMY_EGG with mkdtemp() as tempdir: create_venv(tempdir) installer = EggInst(path, prefix=tempdir) installer.install() # When repository = Repository._from_prefixes([tempdir]) # Then packages = repository.find_packages("dummy") self.assertEqual(len(packages), 1) self.assertEqual(packages[0].name, "dummy")
def test_remove_non_existing(self): # Given entries = [ dummy_repository_package_factory("numpy", "1.6.1", 1), dummy_repository_package_factory("numpy", "1.8.0", 2), ] repository = repository_factory(entries) solver = Solver(repository, Repository._from_prefixes([self.prefix])) request = Request() request.remove(Requirement("numpy")) # When/Then with self.assertRaises(EnpkgError): solver.resolve(request)
def test_install_no_egg_entry(self): # Given entries = [ dummy_repository_package_factory("numpy", "1.6.1", 1), dummy_repository_package_factory("numpy", "1.8.0", 2), ] repository = repository_factory(entries) installed_repository = Repository._from_prefixes([self.prefix]) solver = Solver(repository, installed_repository) request = Request() request.install(Requirement("scipy")) # When/Then with self.assertRaises(NoPackageFound): solver.resolve(request)