def _run_do_update(app_data, distribution, embed_filename, for_py_version,
periodic, search_dirs):
from virtualenv.seed.wheels import acquire
wheel_filename = None if embed_filename is None else Path(embed_filename)
embed_version = None if wheel_filename is None else Wheel(
wheel_filename).version_tuple
app_data = AppDataDiskFolder(app_data) if isinstance(app_data,
str) else app_data
search_dirs = [Path(p) if isinstance(p, str) else p for p in search_dirs]
wheelhouse = app_data.house
embed_update_log = app_data.embed_update_log(distribution, for_py_version)
u_log = UpdateLog.from_dict(embed_update_log.read())
now = datetime.now()
if wheel_filename is not None:
dest = wheelhouse / wheel_filename.name
if not dest.exists():
copy2(str(wheel_filename), str(wheelhouse))
last, last_version, versions = None, None, []
while last is None or not last.use(now):
download_time = datetime.now()
dest = acquire.download_wheel(
distribution=distribution,
version_spec=None
if last_version is None else "<{}".format(last_version),
for_py_version=for_py_version,
search_dirs=search_dirs,
app_data=app_data,
to_folder=wheelhouse,
env=os.environ,
)
if dest is None or (u_log.versions
and u_log.versions[0].filename == dest.name):
break
release_date = release_date_for_wheel_path(dest.path)
last = NewVersion(filename=dest.path.name,
release_date=release_date,
found_date=download_time)
logging.info("detected %s in %s", last, datetime.now() - download_time)
versions.append(last)
last_wheel = Wheel(Path(last.filename))
last_version = last_wheel.version
if embed_version is not None:
if embed_version >= last_wheel.version_tuple: # stop download if we reach the embed version
break
u_log.periodic = periodic
if not u_log.periodic:
u_log.started = now
u_log.versions = versions + u_log.versions
u_log.completed = datetime.now()
embed_update_log.write(u_log.to_dict())
return versions
def test_download_stop_with_embed(tmp_path, mocker, freezer):
freezer.move_to(_UP_NOW)
wheel = get_embed_wheel("pip", "3.9")
app_data_outer = AppDataDiskFolder(str(tmp_path / "app"))
pip_version_remote = [wheel_path(wheel, (0, 0, 2)), wheel_path(wheel, (0, 0, 1)), wheel_path(wheel, (-1, 0, 0))]
at = {"index": 0}
def download():
while True:
path = pip_version_remote[at["index"]]
at["index"] += 1
yield Wheel(Path(path))
do = download()
download_wheel = mocker.patch("virtualenv.seed.wheels.acquire.download_wheel", side_effect=lambda *a, **k: next(do))
url_o = mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen", side_effect=URLError("unavailable"))
last_update = _UP_NOW - timedelta(days=14)
u_log = UpdateLog(started=last_update, completed=last_update, versions=[], periodic=True)
read_dict = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict())
write = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.write")
do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [], True)
assert download_wheel.call_count == 3
assert url_o.call_count == 2
assert read_dict.call_count == 1
assert write.call_count == 1
def test_download_manual_stop_after_one_download(tmp_path, mocker, freezer):
freezer.move_to(_UP_NOW)
wheel = get_embed_wheel("pip", "3.9")
app_data_outer = AppDataDiskFolder(str(tmp_path / "app"))
pip_version_remote = [wheel_path(wheel, (0, 1, 1))]
download_wheel = mock_download(mocker, pip_version_remote)
url_o = mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen",
side_effect=URLError("unavailable"))
last_update = _UP_NOW - timedelta(days=14)
u_log = UpdateLog(started=last_update,
completed=last_update,
versions=[],
periodic=True)
read_dict = mocker.patch(
"virtualenv.app_data.via_disk_folder.JSONStoreDisk.read",
return_value=u_log.to_dict())
write = mocker.patch(
"virtualenv.app_data.via_disk_folder.JSONStoreDisk.write")
do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [], False)
assert download_wheel.call_count == 1
assert url_o.call_count == 2
assert read_dict.call_count == 1
assert write.call_count == 1
def test_do_update_skip_already_done(tmp_path, mocker, freezer):
freezer.move_to(_UP_NOW + timedelta(hours=1))
wheel = get_embed_wheel("pip", "3.9")
app_data_outer = AppDataDiskFolder(str(tmp_path / "app"))
extra = tmp_path / "extra"
extra.mkdir()
def _download_wheel(distribution, version_spec, for_py_version,
search_dirs, app_data, to_folder, env): # noqa
return wheel.path
download_wheel = mocker.patch(
"virtualenv.seed.wheels.acquire.download_wheel",
side_effect=_download_wheel)
url_o = mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen",
side_effect=RuntimeError)
released = _UP_NOW - timedelta(days=30)
u_log = UpdateLog(
started=_UP_NOW - timedelta(days=31),
completed=released,
versions=[
NewVersion(filename=wheel.path.name,
found_date=released,
release_date=released)
],
periodic=True,
)
read_dict = mocker.patch(
"virtualenv.app_data.via_disk_folder.JSONStoreDisk.read",
return_value=u_log.to_dict())
write = mocker.patch(
"virtualenv.app_data.via_disk_folder.JSONStoreDisk.write")
versions = do_update("pip", "3.9", str(wheel.path), str(app_data_outer),
[str(extra)], False)
assert download_wheel.call_count == 1
assert read_dict.call_count == 1
assert not url_o.call_count
assert versions == []
assert write.call_count == 1
wrote_json = write.call_args[0][0]
assert wrote_json == {
"started":
dump_datetime(_UP_NOW + timedelta(hours=1)),
"completed":
dump_datetime(_UP_NOW + timedelta(hours=1)),
"periodic":
False,
"versions": [
{
"filename": wheel.path.name,
"release_date": dump_datetime(released),
"found_date": dump_datetime(released),
},
],
}
def app_data(tmp_path_factory, for_py_version, next_pip_wheel):
temp_folder = tmp_path_factory.mktemp("module-app-data")
now = dump_datetime(datetime.now())
app_data_ = AppDataDiskFolder(str(temp_folder))
app_data_.embed_update_log("pip", for_py_version).write({
"completed":
now,
"periodic":
True,
"started":
now,
"versions": [{
"filename": next_pip_wheel.name,
"found_date": "2000-01-01T00:00:00.000000Z",
"release_date": "2000-01-01T00:00:00.000000Z",
"source": "periodic",
}],
})
yield app_data_
def test_download_manual_ignores_pre_release(tmp_path, mocker, freezer):
freezer.move_to(_UP_NOW)
wheel = get_embed_wheel("pip", "3.9")
app_data_outer = AppDataDiskFolder(str(tmp_path / "app"))
pip_version_remote = [wheel_path(wheel, (0, 0, 1))]
pip_version_pre = NewVersion(
Path(wheel_path(wheel, (0, 1, 0), "b1")).name, _UP_NOW, None,
"downloaded")
download_wheel = mock_download(mocker, pip_version_remote)
url_o = mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen",
side_effect=URLError("unavailable"))
last_update = _UP_NOW - timedelta(days=14)
u_log = UpdateLog(started=last_update,
completed=last_update,
versions=[pip_version_pre],
periodic=True)
read_dict = mocker.patch(
"virtualenv.app_data.via_disk_folder.JSONStoreDisk.read",
return_value=u_log.to_dict())
write = mocker.patch(
"virtualenv.app_data.via_disk_folder.JSONStoreDisk.write")
do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [], False)
assert download_wheel.call_count == 1
assert url_o.call_count == 2
assert read_dict.call_count == 1
assert write.call_count == 1
wrote_json = write.call_args[0][0]
assert wrote_json["versions"] == [
{
"filename": Path(pip_version_remote[0]).name,
"release_date": None,
"found_date": dump_datetime(_UP_NOW),
"source": "manual",
},
pip_version_pre.to_dict(),
]
def test_get_wheel_download_cached(tmp_path, freezer, mocker, for_py_version,
downloaded_wheel):
from virtualenv.app_data.via_disk_folder import JSONStoreDisk
app_data = AppDataDiskFolder(folder=str(tmp_path))
expected = downloaded_wheel[0]
write = mocker.spy(JSONStoreDisk, "write")
# 1st call, not cached, download is called
wheel = get_wheel(expected.distribution, expected.version, for_py_version,
[], True, app_data, False, os.environ)
assert wheel is not None
assert wheel.name == expected.name
assert downloaded_wheel[1].call_count == 1
assert write.call_count == 1
# 2nd call, cached, download is not called
wheel = get_wheel(expected.distribution, expected.version, for_py_version,
[], True, app_data, False, os.environ)
assert wheel is not None
assert wheel.name == expected.name
assert downloaded_wheel[1].call_count == 1
assert write.call_count == 1
wrote_json = write.call_args[0][1]
assert wrote_json == {
"completed":
None,
"periodic":
None,
"started":
None,
"versions": [
{
"filename": expected.name,
"release_date": None,
"found_date": dump_datetime(datetime.now()),
"source": "download",
},
],
}
def test_download_periodic_stop_at_first_usable(tmp_path, mocker, freezer):
freezer.move_to(_UP_NOW)
wheel = get_embed_wheel("pip", "3.9")
app_data_outer = AppDataDiskFolder(str(tmp_path / "app"))
pip_version_remote = [
wheel_path(wheel, (0, 1, 1)),
wheel_path(wheel, (0, 1, 0))
]
rel_date_remote = [
_UP_NOW - timedelta(days=1), _UP_NOW - timedelta(days=30)
]
download_wheel = mock_download(mocker, pip_version_remote)
rel_date_gen = iter(rel_date_remote)
release_date = mocker.patch(
"virtualenv.seed.wheels.periodic_update.release_date_for_wheel_path",
side_effect=lambda *a, **k: next(rel_date_gen),
)
last_update = _UP_NOW - timedelta(days=14)
u_log = UpdateLog(started=last_update,
completed=last_update,
versions=[],
periodic=True)
read_dict = mocker.patch(
"virtualenv.app_data.via_disk_folder.JSONStoreDisk.read",
return_value=u_log.to_dict())
write = mocker.patch(
"virtualenv.app_data.via_disk_folder.JSONStoreDisk.write")
do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [], True)
assert download_wheel.call_count == 2
assert release_date.call_count == 2
assert read_dict.call_count == 1
assert write.call_count == 1
def _run_do_update(app_data, distribution, embed_filename, for_py_version,
periodic, search_dirs):
from virtualenv.seed.wheels import acquire
wheel_filename = None if embed_filename is None else Path(embed_filename)
embed_version = None if wheel_filename is None else Wheel(
wheel_filename).version_tuple
app_data = AppDataDiskFolder(app_data) if isinstance(app_data,
str) else app_data
search_dirs = [Path(p) if isinstance(p, str) else p for p in search_dirs]
wheelhouse = app_data.house
embed_update_log = app_data.embed_update_log(distribution, for_py_version)
u_log = UpdateLog.from_dict(embed_update_log.read())
now = datetime.now()
update_versions, other_versions = [], []
for version in u_log.versions:
if version.source in {"periodic", "manual"}:
update_versions.append(version)
else:
other_versions.append(version)
if periodic:
source = "periodic"
else:
source = "manual"
# mark the most recent one as source "manual"
if update_versions:
update_versions[0].source = source
if wheel_filename is not None:
dest = wheelhouse / wheel_filename.name
if not dest.exists():
copy2(str(wheel_filename), str(wheelhouse))
last, last_version, versions, filenames = None, None, [], set()
while last is None or not last.use(now, ignore_grace_period_ci=True):
download_time = datetime.now()
dest = acquire.download_wheel(
distribution=distribution,
version_spec=None if last_version is None else f"<{last_version}",
for_py_version=for_py_version,
search_dirs=search_dirs,
app_data=app_data,
to_folder=wheelhouse,
env=os.environ,
)
if dest is None or (update_versions
and update_versions[0].filename == dest.name):
break
release_date = release_date_for_wheel_path(dest.path)
last = NewVersion(filename=dest.path.name,
release_date=release_date,
found_date=download_time,
source=source)
logging.info("detected %s in %s", last, datetime.now() - download_time)
versions.append(last)
filenames.add(last.filename)
last_wheel = last.wheel
last_version = last_wheel.version
if embed_version is not None:
if embed_version >= last_wheel.version_tuple: # stop download if we reach the embed version
break
u_log.periodic = periodic
if not u_log.periodic:
u_log.started = now
# update other_versions by removing version we just found
other_versions = [
version for version in other_versions
if version.filename not in filenames
]
u_log.versions = versions + update_versions + other_versions
u_log.completed = datetime.now()
embed_update_log.write(u_log.to_dict())
return versions
def session_app_data(tmp_path_factory):
app_data = AppDataDiskFolder(
folder=str(tmp_path_factory.mktemp("session-app-data")))
with change_env_var(str("VIRTUALENV_OVERRIDE_APP_DATA"),
str(app_data.lock.path)):
yield app_data
def test_do_update_first(tmp_path, mocker, freezer):
freezer.move_to(_UP_NOW)
wheel = get_embed_wheel("pip", "3.9")
app_data_outer = AppDataDiskFolder(str(tmp_path / "app"))
extra = tmp_path / "extra"
extra.mkdir()
pip_version_remote = [
(wheel_path(wheel, (1, 0, 0)), None),
(wheel_path(wheel, (0, 1, 0)), _UP_NOW - timedelta(days=1)),
(wheel_path(wheel, (0, 0, 1)), _UP_NOW - timedelta(days=2)),
(wheel.path, _UP_NOW - timedelta(days=3)),
(wheel_path(wheel, (-1, 0, 0)), _UP_NOW - timedelta(days=30)),
]
download_wheels = (Wheel(Path(i[0])) for i in pip_version_remote)
def _download_wheel(distribution, version_spec, for_py_version,
search_dirs, app_data, to_folder, env):
assert distribution == "pip"
assert for_py_version == "3.9"
assert [str(i) for i in search_dirs] == [str(extra)]
assert isinstance(app_data, AppDataDiskFolder)
assert to_folder == app_data_outer.house
return next(download_wheels)
download_wheel = mocker.patch(
"virtualenv.seed.wheels.acquire.download_wheel",
side_effect=_download_wheel)
releases = {
Wheel(Path(wheel)).version: [
{
"upload_time":
datetime.strftime(release_date, "%Y-%m-%dT%H:%M:%S")
if release_date is not None else None
},
]
for wheel, release_date in pip_version_remote
}
pypi_release = json.dumps({"releases": releases})
@contextmanager
def _release(of, context):
assert of == "https://pypi.org/pypi/pip/json"
assert context is None
yield StringIO(pypi_release)
url_o = mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen",
side_effect=_release)
last_update = _UP_NOW - timedelta(days=14)
u_log = UpdateLog(started=last_update,
completed=last_update,
versions=[],
periodic=True)
read_dict = mocker.patch(
"virtualenv.app_data.via_disk_folder.JSONStoreDisk.read",
return_value=u_log.to_dict())
write = mocker.patch(
"virtualenv.app_data.via_disk_folder.JSONStoreDisk.write")
copy = mocker.patch("virtualenv.seed.wheels.periodic_update.copy2")
versions = do_update("pip", "3.9", str(pip_version_remote[-1][0]),
str(app_data_outer), [str(extra)], True)
assert download_wheel.call_count == len(pip_version_remote)
assert url_o.call_count == 1
assert copy.call_count == 1
expected = [
NewVersion(
Path(wheel).name, _UP_NOW,
None if release is None else release.replace(microsecond=0))
for wheel, release in pip_version_remote
]
assert versions == expected
assert read_dict.call_count == 1
assert write.call_count == 1
wrote_json = write.call_args[0][0]
assert wrote_json == {
"started": dump_datetime(last_update),
"completed": dump_datetime(_UP_NOW),
"periodic": True,
"versions": [e.to_dict() for e in expected],
}
def test_download_periodic_stop_at_first_usable_with_previous_minor(
tmp_path, mocker, freezer):
freezer.move_to(_UP_NOW)
wheel = get_embed_wheel("pip", "3.9")
app_data_outer = AppDataDiskFolder(str(tmp_path / "app"))
pip_version_remote = [
wheel_path(wheel, (0, 1, 1)),
wheel_path(wheel, (0, 1, 0)),
wheel_path(wheel, (0, -1, 0))
]
rel_date_remote = [
_UP_NOW - timedelta(days=1), _UP_NOW - timedelta(days=30),
_UP_NOW - timedelta(days=40)
]
downloaded_versions = [
NewVersion(
Path(pip_version_remote[2]).name, rel_date_remote[2], None,
"download"),
NewVersion(
Path(pip_version_remote[0]).name, rel_date_remote[0], None,
"download"),
]
download_wheel = mock_download(mocker, pip_version_remote)
rel_date_gen = iter(rel_date_remote)
release_date = mocker.patch(
"virtualenv.seed.wheels.periodic_update.release_date_for_wheel_path",
side_effect=lambda *a, **k: next(rel_date_gen),
)
last_update = _UP_NOW - timedelta(days=14)
u_log = UpdateLog(
started=last_update,
completed=last_update,
versions=downloaded_versions,
periodic=True,
)
read_dict = mocker.patch(
"virtualenv.app_data.via_disk_folder.JSONStoreDisk.read",
return_value=u_log.to_dict())
write = mocker.patch(
"virtualenv.app_data.via_disk_folder.JSONStoreDisk.write")
do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [], True)
assert download_wheel.call_count == 2
assert release_date.call_count == 2
assert read_dict.call_count == 1
assert write.call_count == 1
wrote_json = write.call_args[0][0]
assert wrote_json["versions"] == [
{
"filename": Path(pip_version_remote[0]).name,
"release_date": dump_datetime(rel_date_remote[0]),
"found_date": dump_datetime(_UP_NOW),
"source": "periodic",
},
{
"filename": Path(pip_version_remote[1]).name,
"release_date": dump_datetime(rel_date_remote[1]),
"found_date": dump_datetime(_UP_NOW),
"source": "periodic",
},
downloaded_versions[0].to_dict(),
]
