def test_instantiating_package_cache_when_both_tar_bz2_and_conda_exist_read_only(
):
"""
If both .tar.bz2 and .conda packages exist in a read-only package cache, but neither is
unpacked, the .conda package should be preferred and pcrec loaded from that package.
"""
with make_temp_package_cache() as pkgs_dir:
# instantiate to create magic file
PackageCacheData(pkgs_dir)
# copy .tar.bz2 to package cache
cache_action = CacheUrlAction(
"%s/%s/%s" % (CONDA_PKG_REPO, subdir, zlib_tar_bz2_fn),
pkgs_dir,
zlib_tar_bz2_fn,
)
cache_action.verify()
cache_action.execute()
cache_action.cleanup()
# copy .conda to package cache
cache_action = CacheUrlAction(
"%s/%s/%s" % (CONDA_PKG_REPO, subdir, zlib_conda_fn),
pkgs_dir,
zlib_conda_fn,
)
cache_action.verify()
cache_action.execute()
cache_action.cleanup()
make_read_only(join(pkgs_dir, PACKAGE_CACHE_MAGIC_FILE))
PackageCacheData._cache_.clear()
pcd = PackageCacheData(pkgs_dir)
pcrecs = tuple(pcd.iter_records())
assert len(pcrecs) == 1
pcrec = pcrecs[0]
# no repodata_record.json file should be created
assert not isfile(
join(pkgs_dir, zlib_base_fn, "info", "repodata_record.json"))
assert pcrec.fn == zlib_conda_fn
assert pcrec.md5 == "edad165fc3d25636d4f0a61c42873fbc"
assert pcrec.size == 112305
pkgs_dir_files = listdir(pkgs_dir)
assert zlib_base_fn not in pkgs_dir_files
assert zlib_tar_bz2_fn in pkgs_dir_files
assert zlib_conda_fn in pkgs_dir_files
def test_instantiating_package_cache_when_both_tar_bz2_and_conda_exist():
"""
If both .tar.bz2 and .conda packages exist in a writable package cache, but neither is
unpacked, the .conda package should be preferred and unpacked in place.
"""
with make_temp_package_cache() as pkgs_dir:
# copy .tar.bz2 to package cache
cache_action = CacheUrlAction(
"%s/%s/%s" % (CONDA_PKG_REPO, subdir, zlib_tar_bz2_fn),
pkgs_dir,
zlib_tar_bz2_fn,
)
cache_action.verify()
cache_action.execute()
cache_action.cleanup()
# copy .conda to package cache
cache_action = CacheUrlAction(
"%s/%s/%s" % (CONDA_PKG_REPO, subdir, zlib_conda_fn),
pkgs_dir,
zlib_conda_fn,
)
cache_action.verify()
cache_action.execute()
cache_action.cleanup()
PackageCacheData._cache_.clear()
pcd = PackageCacheData(pkgs_dir)
pcrecs = tuple(pcd.iter_records())
assert len(pcrecs) == 1
pcrec = pcrecs[0]
# ensure the package was actually extracted by presence of repodata_record.json
with open(join(pkgs_dir, zlib_base_fn, "info",
"repodata_record.json")) as fh:
repodata_record = json.load(fh)
assert pcrec.fn == zlib_conda_fn == repodata_record["fn"]
assert pcrec.md5 == repodata_record["md5"]
pkgs_dir_files = listdir(pkgs_dir)
assert zlib_base_fn in pkgs_dir_files
assert zlib_tar_bz2_fn in pkgs_dir_files
assert zlib_conda_fn in pkgs_dir_files
def pkg_env(environment_file: Path, coex_path: Path, cache_dir: Path) -> None:
"""Resolve, fetch, and repackage conda env into coex /pkgs directory.
Resolve conda environment file to a specific package list via conda solver,
then fetch and unpack target packages. Repack into .coex package data in
cache_dir or reuse if pre-packed, and assemble into /pkgs under coex_path.
Args:
environment_file: Standard conda env file, can not contain pip deps.
coex_path: Output coex build path.
cache_dir: Coex build cache directory.
"""
# Resolve environment file to dependencies
# Logic culled from conda-env
spec = YamlFileSpec(filename=str(environment_file))
env = spec.environment
logging.info(env.dependencies)
assert set(env.dependencies) == {
"conda"
}, f"coex environments do not support pip dependencies: {env}"
channel_urls = [chan for chan in env.channels if chan != "nodefaults"]
if "nodefaults" not in env.channels:
channel_urls.extend(context.channels)
_channel_priority_map = prioritize_channels(channel_urls)
# Setup an dummpy environment resolution for install into /dev/null
# Execute fetch-and-extract operations for required conda packages
prefix = "/dev/null"
channels = IndexedSet(Channel(url) for url in _channel_priority_map)
subdirs = IndexedSet(
os.path.basename(url) for url in _channel_priority_map)
solver = Solver(prefix,
channels,
subdirs,
specs_to_add=env.dependencies["conda"])
transaction: UnlinkLinkTransaction = solver.solve_for_transaction()
logging.info(transaction)
transaction.download_and_extract()
# Resolve all the, now extracted, target packages in the filesystem
fetcher: ProgressiveFetchExtract = transaction._pfe
target_records: Set[PackageRecord] = set(fetcher.link_precs)
logging.debug("target_records=%s", target_records)
extracted: Set[PackageCacheRecord] = {
next(
(pcrec
for pcrec in chain(*(PackageCacheData(pkgs_dir).query(precord)
for pkgs_dir in context.pkgs_dirs))
if pcrec.is_extracted),
None,
)
for precord in target_records
}
logging.debug("extracted=%s", extracted)
# Repackage into a single-file .zst in the cache, then copy into the output
# package.
output_path = coex_path / "pkgs"
for e in extracted:
extracted_dir = Path(e.extracted_package_dir)
pkgname = extracted_dir.name + ".tar.zst"
cache_dir.mkdir(parents=True, exist_ok=True)
if not (cache_dir / pkgname).exists():
pkg_cmd = (
# tar filtered through zstd
# Seeing errors on macos 10.13 image when using --use-compress-program
# with arguments, consider (a) installing conda-forge tar or (b) using
# a wrapper script if zstd arguments are needed
[
"tar",
"--use-compress-program",
"zstd -T0" if platform.system() != "Darwin" else "zstd",
]
# write to archive file
+ ["-f", str(cache_dir / pkgname)]
# chdir to extracted package directory
+ ["-C", str(extracted_dir)]
# and add all package dirs
+ (["-c"] + [f.name for f in extracted_dir.iterdir()]))
logging.info("packaging: %s", pkg_cmd)
subprocess.check_call(pkg_cmd)
output_path.mkdir(parents=True, exist_ok=True)
shutil.copyfile(cache_dir / pkgname, output_path / pkgname)