def _run_do_update(app_data, distribution, embed_filename, for_py_version, periodic, search_dirs): from virtualenv.seed.wheels import acquire wheel_filename = None if embed_filename is None else Path(embed_filename) embed_version = None if wheel_filename is None else Wheel( wheel_filename).version_tuple app_data = AppDataDiskFolder(app_data) if isinstance(app_data, str) else app_data search_dirs = [Path(p) if isinstance(p, str) else p for p in search_dirs] wheelhouse = app_data.house embed_update_log = app_data.embed_update_log(distribution, for_py_version) u_log = UpdateLog.from_dict(embed_update_log.read()) now = datetime.now() if wheel_filename is not None: dest = wheelhouse / wheel_filename.name if not dest.exists(): copy2(str(wheel_filename), str(wheelhouse)) last, last_version, versions = None, None, [] while last is None or not last.use(now): download_time = datetime.now() dest = acquire.download_wheel( distribution=distribution, version_spec=None if last_version is None else "<{}".format(last_version), for_py_version=for_py_version, search_dirs=search_dirs, app_data=app_data, to_folder=wheelhouse, env=os.environ, ) if dest is None or (u_log.versions and u_log.versions[0].filename == dest.name): break release_date = release_date_for_wheel_path(dest.path) last = NewVersion(filename=dest.path.name, release_date=release_date, found_date=download_time) logging.info("detected %s in %s", last, datetime.now() - download_time) versions.append(last) last_wheel = Wheel(Path(last.filename)) last_version = last_wheel.version if embed_version is not None: if embed_version >= last_wheel.version_tuple: # stop download if we reach the embed version break u_log.periodic = periodic if not u_log.periodic: u_log.started = now u_log.versions = versions + u_log.versions u_log.completed = datetime.now() embed_update_log.write(u_log.to_dict()) return versions
def test_download_stop_with_embed(tmp_path, mocker, freezer): freezer.move_to(_UP_NOW) wheel = get_embed_wheel("pip", "3.9") app_data_outer = AppDataDiskFolder(str(tmp_path / "app")) pip_version_remote = [wheel_path(wheel, (0, 0, 2)), wheel_path(wheel, (0, 0, 1)), wheel_path(wheel, (-1, 0, 0))] at = {"index": 0} def download(): while True: path = pip_version_remote[at["index"]] at["index"] += 1 yield Wheel(Path(path)) do = download() download_wheel = mocker.patch("virtualenv.seed.wheels.acquire.download_wheel", side_effect=lambda *a, **k: next(do)) url_o = mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen", side_effect=URLError("unavailable")) last_update = _UP_NOW - timedelta(days=14) u_log = UpdateLog(started=last_update, completed=last_update, versions=[], periodic=True) read_dict = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) write = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.write") do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [], True) assert download_wheel.call_count == 3 assert url_o.call_count == 2 assert read_dict.call_count == 1 assert write.call_count == 1
def test_download_manual_stop_after_one_download(tmp_path, mocker, freezer): freezer.move_to(_UP_NOW) wheel = get_embed_wheel("pip", "3.9") app_data_outer = AppDataDiskFolder(str(tmp_path / "app")) pip_version_remote = [wheel_path(wheel, (0, 1, 1))] download_wheel = mock_download(mocker, pip_version_remote) url_o = mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen", side_effect=URLError("unavailable")) last_update = _UP_NOW - timedelta(days=14) u_log = UpdateLog(started=last_update, completed=last_update, versions=[], periodic=True) read_dict = mocker.patch( "virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) write = mocker.patch( "virtualenv.app_data.via_disk_folder.JSONStoreDisk.write") do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [], False) assert download_wheel.call_count == 1 assert url_o.call_count == 2 assert read_dict.call_count == 1 assert write.call_count == 1
def test_do_update_skip_already_done(tmp_path, mocker, freezer): freezer.move_to(_UP_NOW + timedelta(hours=1)) wheel = get_embed_wheel("pip", "3.9") app_data_outer = AppDataDiskFolder(str(tmp_path / "app")) extra = tmp_path / "extra" extra.mkdir() def _download_wheel(distribution, version_spec, for_py_version, search_dirs, app_data, to_folder, env): # noqa return wheel.path download_wheel = mocker.patch( "virtualenv.seed.wheels.acquire.download_wheel", side_effect=_download_wheel) url_o = mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen", side_effect=RuntimeError) released = _UP_NOW - timedelta(days=30) u_log = UpdateLog( started=_UP_NOW - timedelta(days=31), completed=released, versions=[ NewVersion(filename=wheel.path.name, found_date=released, release_date=released) ], periodic=True, ) read_dict = mocker.patch( "virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) write = mocker.patch( "virtualenv.app_data.via_disk_folder.JSONStoreDisk.write") versions = do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [str(extra)], False) assert download_wheel.call_count == 1 assert read_dict.call_count == 1 assert not url_o.call_count assert versions == [] assert write.call_count == 1 wrote_json = write.call_args[0][0] assert wrote_json == { "started": dump_datetime(_UP_NOW + timedelta(hours=1)), "completed": dump_datetime(_UP_NOW + timedelta(hours=1)), "periodic": False, "versions": [ { "filename": wheel.path.name, "release_date": dump_datetime(released), "found_date": dump_datetime(released), }, ], }
def app_data(tmp_path_factory, for_py_version, next_pip_wheel): temp_folder = tmp_path_factory.mktemp("module-app-data") now = dump_datetime(datetime.now()) app_data_ = AppDataDiskFolder(str(temp_folder)) app_data_.embed_update_log("pip", for_py_version).write({ "completed": now, "periodic": True, "started": now, "versions": [{ "filename": next_pip_wheel.name, "found_date": "2000-01-01T00:00:00.000000Z", "release_date": "2000-01-01T00:00:00.000000Z", "source": "periodic", }], }) yield app_data_
def test_download_manual_ignores_pre_release(tmp_path, mocker, freezer): freezer.move_to(_UP_NOW) wheel = get_embed_wheel("pip", "3.9") app_data_outer = AppDataDiskFolder(str(tmp_path / "app")) pip_version_remote = [wheel_path(wheel, (0, 0, 1))] pip_version_pre = NewVersion( Path(wheel_path(wheel, (0, 1, 0), "b1")).name, _UP_NOW, None, "downloaded") download_wheel = mock_download(mocker, pip_version_remote) url_o = mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen", side_effect=URLError("unavailable")) last_update = _UP_NOW - timedelta(days=14) u_log = UpdateLog(started=last_update, completed=last_update, versions=[pip_version_pre], periodic=True) read_dict = mocker.patch( "virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) write = mocker.patch( "virtualenv.app_data.via_disk_folder.JSONStoreDisk.write") do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [], False) assert download_wheel.call_count == 1 assert url_o.call_count == 2 assert read_dict.call_count == 1 assert write.call_count == 1 wrote_json = write.call_args[0][0] assert wrote_json["versions"] == [ { "filename": Path(pip_version_remote[0]).name, "release_date": None, "found_date": dump_datetime(_UP_NOW), "source": "manual", }, pip_version_pre.to_dict(), ]
def test_get_wheel_download_cached(tmp_path, freezer, mocker, for_py_version, downloaded_wheel): from virtualenv.app_data.via_disk_folder import JSONStoreDisk app_data = AppDataDiskFolder(folder=str(tmp_path)) expected = downloaded_wheel[0] write = mocker.spy(JSONStoreDisk, "write") # 1st call, not cached, download is called wheel = get_wheel(expected.distribution, expected.version, for_py_version, [], True, app_data, False, os.environ) assert wheel is not None assert wheel.name == expected.name assert downloaded_wheel[1].call_count == 1 assert write.call_count == 1 # 2nd call, cached, download is not called wheel = get_wheel(expected.distribution, expected.version, for_py_version, [], True, app_data, False, os.environ) assert wheel is not None assert wheel.name == expected.name assert downloaded_wheel[1].call_count == 1 assert write.call_count == 1 wrote_json = write.call_args[0][1] assert wrote_json == { "completed": None, "periodic": None, "started": None, "versions": [ { "filename": expected.name, "release_date": None, "found_date": dump_datetime(datetime.now()), "source": "download", }, ], }
def test_download_periodic_stop_at_first_usable(tmp_path, mocker, freezer): freezer.move_to(_UP_NOW) wheel = get_embed_wheel("pip", "3.9") app_data_outer = AppDataDiskFolder(str(tmp_path / "app")) pip_version_remote = [ wheel_path(wheel, (0, 1, 1)), wheel_path(wheel, (0, 1, 0)) ] rel_date_remote = [ _UP_NOW - timedelta(days=1), _UP_NOW - timedelta(days=30) ] download_wheel = mock_download(mocker, pip_version_remote) rel_date_gen = iter(rel_date_remote) release_date = mocker.patch( "virtualenv.seed.wheels.periodic_update.release_date_for_wheel_path", side_effect=lambda *a, **k: next(rel_date_gen), ) last_update = _UP_NOW - timedelta(days=14) u_log = UpdateLog(started=last_update, completed=last_update, versions=[], periodic=True) read_dict = mocker.patch( "virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) write = mocker.patch( "virtualenv.app_data.via_disk_folder.JSONStoreDisk.write") do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [], True) assert download_wheel.call_count == 2 assert release_date.call_count == 2 assert read_dict.call_count == 1 assert write.call_count == 1
def _run_do_update(app_data, distribution, embed_filename, for_py_version, periodic, search_dirs): from virtualenv.seed.wheels import acquire wheel_filename = None if embed_filename is None else Path(embed_filename) embed_version = None if wheel_filename is None else Wheel( wheel_filename).version_tuple app_data = AppDataDiskFolder(app_data) if isinstance(app_data, str) else app_data search_dirs = [Path(p) if isinstance(p, str) else p for p in search_dirs] wheelhouse = app_data.house embed_update_log = app_data.embed_update_log(distribution, for_py_version) u_log = UpdateLog.from_dict(embed_update_log.read()) now = datetime.now() update_versions, other_versions = [], [] for version in u_log.versions: if version.source in {"periodic", "manual"}: update_versions.append(version) else: other_versions.append(version) if periodic: source = "periodic" else: source = "manual" # mark the most recent one as source "manual" if update_versions: update_versions[0].source = source if wheel_filename is not None: dest = wheelhouse / wheel_filename.name if not dest.exists(): copy2(str(wheel_filename), str(wheelhouse)) last, last_version, versions, filenames = None, None, [], set() while last is None or not last.use(now, ignore_grace_period_ci=True): download_time = datetime.now() dest = acquire.download_wheel( distribution=distribution, version_spec=None if last_version is None else f"<{last_version}", for_py_version=for_py_version, search_dirs=search_dirs, app_data=app_data, to_folder=wheelhouse, env=os.environ, ) if dest is None or (update_versions and update_versions[0].filename == dest.name): break release_date = release_date_for_wheel_path(dest.path) last = NewVersion(filename=dest.path.name, release_date=release_date, found_date=download_time, source=source) logging.info("detected %s in %s", last, datetime.now() - download_time) versions.append(last) filenames.add(last.filename) last_wheel = last.wheel last_version = last_wheel.version if embed_version is not None: if embed_version >= last_wheel.version_tuple: # stop download if we reach the embed version break u_log.periodic = periodic if not u_log.periodic: u_log.started = now # update other_versions by removing version we just found other_versions = [ version for version in other_versions if version.filename not in filenames ] u_log.versions = versions + update_versions + other_versions u_log.completed = datetime.now() embed_update_log.write(u_log.to_dict()) return versions
def session_app_data(tmp_path_factory): app_data = AppDataDiskFolder( folder=str(tmp_path_factory.mktemp("session-app-data"))) with change_env_var(str("VIRTUALENV_OVERRIDE_APP_DATA"), str(app_data.lock.path)): yield app_data
def test_do_update_first(tmp_path, mocker, freezer): freezer.move_to(_UP_NOW) wheel = get_embed_wheel("pip", "3.9") app_data_outer = AppDataDiskFolder(str(tmp_path / "app")) extra = tmp_path / "extra" extra.mkdir() pip_version_remote = [ (wheel_path(wheel, (1, 0, 0)), None), (wheel_path(wheel, (0, 1, 0)), _UP_NOW - timedelta(days=1)), (wheel_path(wheel, (0, 0, 1)), _UP_NOW - timedelta(days=2)), (wheel.path, _UP_NOW - timedelta(days=3)), (wheel_path(wheel, (-1, 0, 0)), _UP_NOW - timedelta(days=30)), ] download_wheels = (Wheel(Path(i[0])) for i in pip_version_remote) def _download_wheel(distribution, version_spec, for_py_version, search_dirs, app_data, to_folder, env): assert distribution == "pip" assert for_py_version == "3.9" assert [str(i) for i in search_dirs] == [str(extra)] assert isinstance(app_data, AppDataDiskFolder) assert to_folder == app_data_outer.house return next(download_wheels) download_wheel = mocker.patch( "virtualenv.seed.wheels.acquire.download_wheel", side_effect=_download_wheel) releases = { Wheel(Path(wheel)).version: [ { "upload_time": datetime.strftime(release_date, "%Y-%m-%dT%H:%M:%S") if release_date is not None else None }, ] for wheel, release_date in pip_version_remote } pypi_release = json.dumps({"releases": releases}) @contextmanager def _release(of, context): assert of == "https://pypi.org/pypi/pip/json" assert context is None yield StringIO(pypi_release) url_o = mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen", side_effect=_release) last_update = _UP_NOW - timedelta(days=14) u_log = UpdateLog(started=last_update, completed=last_update, versions=[], periodic=True) read_dict = mocker.patch( "virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) write = mocker.patch( "virtualenv.app_data.via_disk_folder.JSONStoreDisk.write") copy = mocker.patch("virtualenv.seed.wheels.periodic_update.copy2") versions = do_update("pip", "3.9", str(pip_version_remote[-1][0]), str(app_data_outer), [str(extra)], True) assert download_wheel.call_count == len(pip_version_remote) assert url_o.call_count == 1 assert copy.call_count == 1 expected = [ NewVersion( Path(wheel).name, _UP_NOW, None if release is None else release.replace(microsecond=0)) for wheel, release in pip_version_remote ] assert versions == expected assert read_dict.call_count == 1 assert write.call_count == 1 wrote_json = write.call_args[0][0] assert wrote_json == { "started": dump_datetime(last_update), "completed": dump_datetime(_UP_NOW), "periodic": True, "versions": [e.to_dict() for e in expected], }
def test_download_periodic_stop_at_first_usable_with_previous_minor( tmp_path, mocker, freezer): freezer.move_to(_UP_NOW) wheel = get_embed_wheel("pip", "3.9") app_data_outer = AppDataDiskFolder(str(tmp_path / "app")) pip_version_remote = [ wheel_path(wheel, (0, 1, 1)), wheel_path(wheel, (0, 1, 0)), wheel_path(wheel, (0, -1, 0)) ] rel_date_remote = [ _UP_NOW - timedelta(days=1), _UP_NOW - timedelta(days=30), _UP_NOW - timedelta(days=40) ] downloaded_versions = [ NewVersion( Path(pip_version_remote[2]).name, rel_date_remote[2], None, "download"), NewVersion( Path(pip_version_remote[0]).name, rel_date_remote[0], None, "download"), ] download_wheel = mock_download(mocker, pip_version_remote) rel_date_gen = iter(rel_date_remote) release_date = mocker.patch( "virtualenv.seed.wheels.periodic_update.release_date_for_wheel_path", side_effect=lambda *a, **k: next(rel_date_gen), ) last_update = _UP_NOW - timedelta(days=14) u_log = UpdateLog( started=last_update, completed=last_update, versions=downloaded_versions, periodic=True, ) read_dict = mocker.patch( "virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) write = mocker.patch( "virtualenv.app_data.via_disk_folder.JSONStoreDisk.write") do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [], True) assert download_wheel.call_count == 2 assert release_date.call_count == 2 assert read_dict.call_count == 1 assert write.call_count == 1 wrote_json = write.call_args[0][0] assert wrote_json["versions"] == [ { "filename": Path(pip_version_remote[0]).name, "release_date": dump_datetime(rel_date_remote[0]), "found_date": dump_datetime(_UP_NOW), "source": "periodic", }, { "filename": Path(pip_version_remote[1]).name, "release_date": dump_datetime(rel_date_remote[1]), "found_date": dump_datetime(_UP_NOW), "source": "periodic", }, downloaded_versions[0].to_dict(), ]