def quetz_harvest(package_version: dict, config, pkgstore: PackageStore,
dao: Dao):
filename: str = package_version["filename"]
channel: str = package_version["channel_name"]
platform = package_version["platform"]
print(f"Harvesting: {filename}, {channel}, {platform}")
# TODO figure out how to handle properly either .conda or .tar.bz2
if not filename.endswith('.tar.bz2'):
return
fh = pkgstore.serve_path(channel, Path(platform) / filename)
print("Harvesting ... ")
try:
result = harvest(fh)
except Exception as e:
print(f"Exception caught in harvesting: {str(e)}")
return
print(f"Uploading harvest result for {channel}/{platform}/{filename}")
pkgstore.add_file(
json.dumps(result, indent=4, sort_keys=True),
channel,
Path("metadata") / platform / filename.replace('.tar.bz2', '.json'),
)
def test_delete_package_version(auth_client, public_channel, package_version,
dao, pkgstore: PackageStore, db):
assert public_channel.size > 0
assert public_channel.size == package_version.size
filename = "test-package-0.1-0.tar.bz2"
platform = "linux-64"
response = auth_client.delete(
f"/api/channels/{public_channel.name}/"
f"packages/{package_version.package_name}/versions/{platform}/{filename}"
)
assert response.status_code == 200
versions = (db.query(PackageVersion).filter(
PackageVersion.package_name == package_version.package_name).all())
assert len(versions) == 0
with pytest.raises(Exception):
pkgstore.serve_path(public_channel.name,
str(Path(platform) / filename))
db.refresh(public_channel)
assert public_channel.size == 0
def transmutation(package_version: dict, config, pkgstore: PackageStore, dao: Dao):
filename: str = package_version["filename"]
channel: str = package_version["channel_name"]
package_format: str = package_version["package_format"]
package_name: str = package_version["package_name"]
platform = package_version["platform"]
version = package_version["version"]
build_number = package_version["build_number"]
build_string = package_version["build_string"]
uploader_id = package_version["uploader_id"]
info = json.loads(package_version["info"])
if package_format == "tarbz2" or not filename.endswith(".tar.bz2"):
return
fh = pkgstore.serve_path(channel, Path(platform) / filename)
with TemporaryDirectory() as tmpdirname:
local_file_name = os.path.join(tmpdirname, filename)
with open(local_file_name, "wb") as local_file:
# chunk size 10MB
shutil.copyfileobj(fh, local_file, 10 * 1024 * 1024)
fn, out_fn, errors = _convert(local_file_name, ".conda", tmpdirname, force=True)
if errors:
logger.error(f"transmutation errors --> {errors}")
return
filename_conda = os.path.basename(filename).replace('.tar.bz2', '.conda')
logger.info(f"Adding file to package store: {Path(platform) / filename_conda}")
with open(out_fn, 'rb') as f:
calculate_file_hashes_and_size(info, f)
f.seek(0)
pkgstore.add_package(f, channel, str(Path(platform) / filename_conda))
version = dao.create_version(
channel,
package_name,
"conda",
platform,
version,
build_number,
build_string,
filename_conda,
json.dumps(info),
uploader_id,
info["size"],
upsert=True,
)
if os.path.exists(out_fn):
os.remove(out_fn)
def test_reindex_package_files(
config,
user,
package_files,
channel,
db,
dao,
pkgstore: PackageStore,
package_filenames,
):
user_id = user.id
reindex_packages_from_store(dao, config, channel.name, user_id)
db.refresh(channel)
assert channel.packages
assert channel.packages[0].name == "test-package"
assert channel.packages[0].members[0].user.username == user.username
assert channel.packages[0].package_versions[0].version == '0.1'
assert channel.packages[0].package_versions[1].version == '0.2'
repodata = pkgstore.serve_path(channel.name, "linux-64/repodata.json")
repodata = json.load(repodata)
assert repodata
assert len(repodata['packages']) == 2
assert set(repodata["packages"].keys()) == set(package_filenames)
def test_validate_packages(
config,
user,
package_files,
channel,
channel_name,
db,
dao,
pkgstore: PackageStore,
package_filenames,
):
user_id = user.id
reindex_packages_from_store(dao, config, channel.name, user_id)
db.refresh(channel)
assert channel.packages
assert channel.packages[0].name == "test-package"
assert channel.packages[0].members[0].user.username == user.username
assert (channel.packages[0].package_versions[0].version == '0.1'
or channel.packages[0].package_versions[0].version == '0.2')
assert (channel.packages[0].package_versions[1].version == '0.1'
or channel.packages[0].package_versions[1].version == '0.2')
repodata = pkgstore.serve_path(channel.name, "linux-64/repodata.json")
repodata = json.load(repodata)
assert repodata
assert len(repodata['packages']) == 2
assert set(repodata["packages"].keys()) == set(package_filenames)
remaining_pkg = channel.packages[0].package_versions[1].filename
pkgstore.delete_file(
channel.name,
f'linux-64/{channel.packages[0].package_versions[0].filename}')
validate_packages(dao, pkgstore, channel_name)
db.refresh(channel)
repodata = pkgstore.serve_path(channel.name, "linux-64/repodata.json")
repodata = json.load(repodata)
assert repodata
assert len(repodata['packages']) == 1
assert set(repodata["packages"].keys()) == set([remaining_pkg])
assert len(channel.packages[0].package_versions) == 1
pkgstore.add_file(b"wrong_size", channel_name, f"linux-64/{remaining_pkg}")
validate_packages(dao, pkgstore, channel_name)
db.refresh(channel)
repodata = pkgstore.serve_path(channel.name, "linux-64/repodata.json")
repodata = json.load(repodata)
assert repodata
assert len(repodata['packages']) == 0
assert set(repodata["packages"].keys()) == set([])
assert len(channel.packages[0].package_versions) == 0
def download_remote_file(repository: RemoteRepository, pkgstore: PackageStore,
channel: str, path: str):
"""Download a file from a remote repository to a package store"""
# Check if a download is already underway for this file
lock = pkgstore.get_download_lock(channel, path)
if lock:
# Wait for the download to complete
lock.acquire()
# Release the lock so that any other clients can also finish
lock.release()
return
# Acquire a lock to prevent multiple concurrent downloads of the same file
with pkgstore.create_download_lock(channel, path):
logger.debug(f"Downloading {path} from {channel} to pkgstore")
remote_file = repository.open(path)
data_stream = remote_file.file
pkgstore.add_package(data_stream, channel, path)
pkgstore.delete_download_lock(channel, path)
def package_files(pkgstore: PackageStore, channel_name, package_filenames):
pkgstore.create_channel(channel_name)
for filename in package_filenames:
with open(filename, 'rb') as fid:
content = fid.read()
pkgstore.add_file(content, channel_name, f"linux-64/{filename}")
def cleanup_temp_files(pkgstore: PackageStore, channel_name: str,
dry_run: bool):
pkgstore.cleanup_temp_files(channel_name, dry_run)