def test_clean_and_packages(clear_cache): pkg = "bzip2" with make_temp_package_cache() as pkgs_dir: # pkg doesn't exist ahead of time assert_not_pkg(pkg, _get_pkgs(pkgs_dir)) with make_temp_env(pkg) as prefix: # pkg exists assert_any_pkg(pkg, _get_pkgs(pkgs_dir)) # --json flag is regression test for #5451 stdout, _, _ = run_command(Commands.CLEAN, "", "--packages", "--yes", "--json") json_loads(stdout) # assert valid json # pkg still exists since its in use by temp env assert_any_pkg(pkg, _get_pkgs(pkgs_dir)) run_command(Commands.REMOVE, prefix, pkg, "--yes", "--json") stdout, _, _ = run_command(Commands.CLEAN, "", "--packages", "--yes", "--json") json_loads(stdout) # assert valid json # pkg is removed assert_not_pkg(pkg, _get_pkgs(pkgs_dir)) # pkg is still removed assert_not_pkg(pkg, _get_pkgs(pkgs_dir))
def test_tar_bz2_in_cache_not_extracted(): """ Test that if a .tar.bz2 exists in the package cache (not extracted), and the complementary .conda package is requested, the .tar.bz2 package in the cache is used by default. """ with make_temp_package_cache() as pkgs_dir: copy(join(CHANNEL_DIR, subdir, zlib_tar_bz2_fn), join(pkgs_dir, zlib_tar_bz2_fn)) pfe = ProgressiveFetchExtract((zlib_tar_bz2_prec, )) pfe.prepare() assert len(pfe.cache_actions) == 1 assert len(pfe.extract_actions) == 1 pfe.execute() pkgs_dir_files = listdir(pkgs_dir) assert zlib_base_fn in pkgs_dir_files assert zlib_tar_bz2_fn in pkgs_dir_files # Now ensure download/extract for the complementary .conda package uses the # extracted .tar.bz2 pfe = ProgressiveFetchExtract((zlib_conda_prec, )) pfe.prepare() assert len(pfe.cache_actions) == 0 assert len(pfe.extract_actions) == 0
def test_clean_logfiles(clear_cache): """Logfiles are found in pkgs_dir/.logs. Since these log files are uniquely implemented for the experimental libmamba release we will mock the log files. """ pkg = "bzip2" with make_temp_package_cache() as pkgs_dir: # logfiles don't exist ahead of time assert not _get_logfiles(pkgs_dir) with make_temp_env(pkg): # mimic logfiles being created logs = join(pkgs_dir, CONDA_LOGS_DIR) mkdir_p(logs) path = join(logs, f"{datetime.utcnow():%Y%m%d-%H%M%S-%f}.log") with open(path, "w"): pass # logfiles exist assert path in _get_logfiles(pkgs_dir) # --json flag is regression test for #5451 stdout, _, _ = run_command(Commands.CLEAN, "", "--logfiles", "--yes", "--json") json_loads(stdout) # assert valid json # logfiles removed assert not _get_logfiles(pkgs_dir) # logfiles still removed assert not _get_logfiles(pkgs_dir)
def test_tar_bz2_in_pkg_cache_doesnt_overwrite_conda_pkg(): """ Test that if a .tar.bz2 package is downloaded and extracted in a package cache, the complementary .conda package replaces it if that's what is requested. """ with env_vars({'CONDA_SEPARATE_FORMAT_CACHE': True}, stack_callback=conda_tests_ctxt_mgmt_def_pol): with make_temp_package_cache() as pkgs_dir: # Cache the .tar.bz2 file in the package cache and extract it pfe = ProgressiveFetchExtract((zlib_tar_bz2_prec, )) pfe.prepare() assert len(pfe.cache_actions) == 1 assert len(pfe.extract_actions) == 1 cache_action = pfe.cache_actions[0] extact_action = pfe.extract_actions[0] assert basename(cache_action.target_full_path) == zlib_tar_bz2_fn assert cache_action.target_full_path == extact_action.source_full_path assert basename(extact_action.target_full_path) == zlib_base_fn # Go ahead with executing download and extract now pfe.execute() assert isfile(join(pkgs_dir, zlib_tar_bz2_fn)) assert isfile( join(pkgs_dir, zlib_base_fn, "info", "repodata_record.json")) # Ensure second download/extract is a no-op pfe = ProgressiveFetchExtract((zlib_tar_bz2_prec, )) pfe.prepare() assert len(pfe.cache_actions) == 0 assert len(pfe.extract_actions) == 0 # Now ensure download/extract for the complementary .conda package replaces the # extracted .tar.bz2 pfe = ProgressiveFetchExtract((zlib_conda_prec, )) pfe.prepare() assert len(pfe.cache_actions) == 1 assert len(pfe.extract_actions) == 1 cache_action = pfe.cache_actions[0] extact_action = pfe.extract_actions[0] assert basename(cache_action.target_full_path) == zlib_conda_fn assert cache_action.target_full_path == extact_action.source_full_path assert basename(extact_action.target_full_path) == zlib_base_fn pfe.execute() with open( join(pkgs_dir, zlib_base_fn, "info", "repodata_record.json")) as fh: repodata_record = json.load(fh) assert repodata_record["fn"] == zlib_conda_fn # Now check urls.txt to make sure extensions are included. urls_text = tuple(yield_lines(join(pkgs_dir, "urls.txt"))) assert urls_text[0] == zlib_tar_bz2_prec.url assert urls_text[1] == zlib_conda_prec.url
def test_instantiating_package_cache_when_both_tar_bz2_and_conda_exist_read_only( ): """ If both .tar.bz2 and .conda packages exist in a read-only package cache, but neither is unpacked, the .conda package should be preferred and pcrec loaded from that package. """ with make_temp_package_cache() as pkgs_dir: # instantiate to create magic file PackageCacheData(pkgs_dir) # copy .tar.bz2 to package cache cache_action = CacheUrlAction( "%s/%s/%s" % (CONDA_PKG_REPO, subdir, zlib_tar_bz2_fn), pkgs_dir, zlib_tar_bz2_fn, ) cache_action.verify() cache_action.execute() cache_action.cleanup() # copy .conda to package cache cache_action = CacheUrlAction( "%s/%s/%s" % (CONDA_PKG_REPO, subdir, zlib_conda_fn), pkgs_dir, zlib_conda_fn, ) cache_action.verify() cache_action.execute() cache_action.cleanup() make_read_only(join(pkgs_dir, PACKAGE_CACHE_MAGIC_FILE)) PackageCacheData._cache_.clear() pcd = PackageCacheData(pkgs_dir) pcrecs = tuple(pcd.iter_records()) assert len(pcrecs) == 1 pcrec = pcrecs[0] # no repodata_record.json file should be created assert not isfile( join(pkgs_dir, zlib_base_fn, "info", "repodata_record.json")) assert pcrec.fn == zlib_conda_fn assert pcrec.md5 == "edad165fc3d25636d4f0a61c42873fbc" assert pcrec.size == 112305 pkgs_dir_files = listdir(pkgs_dir) assert zlib_base_fn not in pkgs_dir_files assert zlib_tar_bz2_fn in pkgs_dir_files assert zlib_conda_fn in pkgs_dir_files
def test_clean_all(clear_cache): pkg = "bzip2" with make_temp_package_cache() as pkgs_dir: # pkg, tarball, & index cache doesn't exist ahead of time pkgs, tars, cache = _get_all(pkgs_dir) assert_not_pkg(pkg, pkgs) assert_not_pkg(pkg, tars) assert not cache with make_temp_env(pkg) as prefix: # pkg, tarball, & index cache exists pkgs, tars, cache = _get_all(pkgs_dir) assert_any_pkg(pkg, pkgs) assert_any_pkg(pkg, tars) assert cache stdout, _, _ = run_command(Commands.CLEAN, "", "--all", "--yes", "--json") json_loads(stdout) # assert valid json # pkg still exists since its in use by temp env # tarball is removed # index cache is cleared pkgs, tars, cache = _get_all(pkgs_dir) assert_any_pkg(pkg, pkgs) assert_not_pkg(pkg, tars) assert not cache run_command(Commands.REMOVE, prefix, pkg, "--yes", "--json") stdout, _, _ = run_command(Commands.CLEAN, "", "--packages", "--yes", "--json") json_loads(stdout) # assert valid json # pkg is removed # tarball is still removed # index cache is still cleared pkgs, tars, index_cache = _get_all(pkgs_dir) assert_not_pkg(pkg, pkgs) assert_not_pkg(pkg, tars) assert not cache # pkg is still removed # tarball is still removed # index cache is still cleared pkgs, tars, index_cache = _get_all(pkgs_dir) assert_not_pkg(pkg, pkgs) assert_not_pkg(pkg, tars) assert not cache
def test_clean_force_pkgs_dirs(clear_cache): pkg = "bzip2" with make_temp_package_cache() as pkgs_dir: # pkgs_dir is a directory assert isdir(pkgs_dir) with make_temp_env(pkg): stdout, _, _ = run_command(Commands.CLEAN, "", "--force-pkgs-dirs", "--yes", "--json") json_loads(stdout) # assert valid json # pkgs_dir is removed assert not exists(pkgs_dir) # pkgs_dir is still removed assert not exists(pkgs_dir)
def test_instantiating_package_cache_when_both_tar_bz2_and_conda_exist(): """ If both .tar.bz2 and .conda packages exist in a writable package cache, but neither is unpacked, the .conda package should be preferred and unpacked in place. """ with make_temp_package_cache() as pkgs_dir: # copy .tar.bz2 to package cache cache_action = CacheUrlAction( "%s/%s/%s" % (CONDA_PKG_REPO, subdir, zlib_tar_bz2_fn), pkgs_dir, zlib_tar_bz2_fn, ) cache_action.verify() cache_action.execute() cache_action.cleanup() # copy .conda to package cache cache_action = CacheUrlAction( "%s/%s/%s" % (CONDA_PKG_REPO, subdir, zlib_conda_fn), pkgs_dir, zlib_conda_fn, ) cache_action.verify() cache_action.execute() cache_action.cleanup() PackageCacheData._cache_.clear() pcd = PackageCacheData(pkgs_dir) pcrecs = tuple(pcd.iter_records()) assert len(pcrecs) == 1 pcrec = pcrecs[0] # ensure the package was actually extracted by presence of repodata_record.json with open(join(pkgs_dir, zlib_base_fn, "info", "repodata_record.json")) as fh: repodata_record = json.load(fh) assert pcrec.fn == zlib_conda_fn == repodata_record["fn"] assert pcrec.md5 == repodata_record["md5"] pkgs_dir_files = listdir(pkgs_dir) assert zlib_base_fn in pkgs_dir_files assert zlib_tar_bz2_fn in pkgs_dir_files assert zlib_conda_fn in pkgs_dir_files
def test_clean_index_cache(clear_cache): pkg = "bzip2" with make_temp_package_cache(): # index cache doesn't exist ahead of time assert not _get_index_cache() with make_temp_env(pkg): # index cache exists assert _get_index_cache() stdout, _, _ = run_command(Commands.CLEAN, "", "--index-cache", "--yes", "--json") json_loads(stdout) # assert valid json # index cache is cleared assert not _get_index_cache() # index cache is still cleared assert not _get_index_cache()
def test_clean_tarballs(clear_cache): pkg = "bzip2" with make_temp_package_cache() as pkgs_dir: # tarball doesn't exist ahead of time assert_not_pkg(pkg, _get_tars(pkgs_dir)) with make_temp_env(pkg): # tarball exists assert_any_pkg(pkg, _get_tars(pkgs_dir)) # --json flag is regression test for #5451 stdout, _, _ = run_command(Commands.CLEAN, "", "--tarballs", "--yes", "--json") json_loads(stdout) # assert valid json # tarball is removed assert_not_pkg(pkg, _get_tars(pkgs_dir)) # tarball is still removed assert_not_pkg(pkg, _get_tars(pkgs_dir))
def test_tar_bz2_in_pkg_cache_used_instead_of_conda_pkg(): """ Test that if a .tar.bz2 package is downloaded and extracted in a package cache, the complementary .conda package is not downloaded/extracted """ with make_temp_package_cache() as pkgs_dir: # Cache the .tar.bz2 file in the package cache and extract it pfe = ProgressiveFetchExtract((zlib_tar_bz2_prec, )) pfe.prepare() assert len(pfe.cache_actions) == 1 assert len(pfe.extract_actions) == 1 cache_action = pfe.cache_actions[0] extact_action = pfe.extract_actions[0] assert basename(cache_action.target_full_path) == zlib_tar_bz2_fn assert cache_action.target_full_path == extact_action.source_full_path assert basename(extact_action.target_full_path) == zlib_base_fn # Go ahead with executing download and extract now pfe.execute() assert isfile(join(pkgs_dir, zlib_tar_bz2_fn)) assert isfile( join(pkgs_dir, zlib_base_fn, "info", "repodata_record.json")) # Ensure second download/extract is a no-op pfe = ProgressiveFetchExtract((zlib_tar_bz2_prec, )) pfe.prepare() assert len(pfe.cache_actions) == 0 assert len(pfe.extract_actions) == 0 # Now ensure download/extract for the complementary .conda package uses the cache pfe = ProgressiveFetchExtract((zlib_conda_prec, )) pfe.prepare() assert len(pfe.cache_actions) == 0 assert len(pfe.extract_actions) == 0 # Now check urls.txt to make sure extensions are included. urls_text = tuple(yield_lines(join(pkgs_dir, "urls.txt"))) assert urls_text[0] == zlib_tar_bz2_prec.url
def test_clean_tempfiles(clear_cache): """Tempfiles are either suffixed with .c~ or .trash. .c~ is used to indicate that conda is actively using that file. If the conda process is terminated unexpectedly these .c~ files may remain and hence can be cleaned up after the fact. .trash appears to be a legacy suffix that is no longer used by conda. Since the presence of .c~ and .trash files are dependent upon irregular termination we create our own temporary files to confirm they get cleaned up. """ pkg = "bzip2" with make_temp_package_cache() as pkgs_dir: # tempfiles don't exist ahead of time assert not _get_tempfiles(pkgs_dir) with make_temp_env(pkg): # mimic tempfiles being created path = _get_tars(pkgs_dir)[0] # grab any tarball for ext in CONDA_TEMP_EXTENSIONS: copy(path, f"{path}{ext}") # tempfiles exist assert len(_get_tempfiles(pkgs_dir)) == len(CONDA_TEMP_EXTENSIONS) # --json flag is regression test for #5451 stdout, _, _ = run_command(Commands.CLEAN, "", "--tempfiles", pkgs_dir, "--yes", "--json") json_loads(stdout) # assert valid json # tempfiles removed assert not _get_tempfiles(pkgs_dir) # tempfiles still removed assert not _get_tempfiles(pkgs_dir)