def create_version_from_metadata(
channel_name: str,
user_id: bytes,
package_file_name: str,
package_data: dict,
dao: Dao,
):
package_name = package_data["name"]
package = dao.get_package(channel_name, package_name)
if not package:
package_info = rest_models.Package(
name=package_name,
summary=package_data.get("summary", ""),
description=package_data.get("description", ""),
)
dao.create_package(channel_name, package_info, user_id, "owner")
pkg_format = "tarbz2" if package_file_name.endswith(
".tar.bz2") else ".conda"
version = dao.create_version(
channel_name,
package_name,
pkg_format,
get_subdir_compat(package_data),
package_data["version"],
int(package_data["build_number"]),
package_data["build"],
package_file_name,
json.dumps(package_data),
user_id,
package_data["size"],
)
return version
def package_version(user, mirror_channel, db, dao):
# create package version that will added to local repodata
package_format = "tarbz2"
package_info = (
'{"size": 5000, "sha256": "OLD-SHA", "md5": "OLD-MD5", "subdirs":["noarch"]}'
)
new_package_data = rest_models.Package(name="test-package")
package = dao.create_package(
mirror_channel.name,
new_package_data,
user_id=user.id,
role="owner",
)
version = dao.create_version(
mirror_channel.name,
"test-package",
package_format,
"noarch",
"0.1",
"0",
"",
"test-package-0.1-0.tar.bz2",
package_info,
user.id,
)
yield version
db.delete(package)
db.delete(version)
db.commit()
def create_packages_from_channeldata(channel_name: str, user_id: bytes,
channeldata: dict, dao: Dao):
packages = channeldata.get("packages", {})
for package_name, metadata in packages.items():
description = metadata.get("description", "")
summary = metadata.get("summary", "")
package_data = rest_models.Package(
name=package_name,
summary=summary,
description=description,
)
try:
package = dao.create_package(channel_name,
package_data,
user_id,
role=authorization.OWNER)
except DBError:
# package already exists so skip it so we retrieve and update it
package = dao.get_package(channel_name, package_name)
package.description = description
package.summary = summary
package.url = metadata.get("home", "")
package.platforms = ":".join(metadata.get("subdirs", []))
package.channeldata = json.dumps(metadata)
dao.db.commit()
def package(dao, channel, package_name, user, db):
package_data = rest_models.Package(name=package_name)
package = dao.create_package(channel.name, package_data, user.id, "owner")
yield package
db.delete(package)
db.commit()
def handle_file(channel_name, condainfo, dao, user_id):
"""Add or update conda package info to database"""
filename = os.path.split(condainfo._filename)[-1]
package_name = condainfo.info["name"]
package_format = condainfo.package_format
platform = condainfo.info["subdir"]
version = condainfo.info["version"]
build_number = condainfo.info["build_number"]
build_string = condainfo.info["build"]
size = condainfo.info["size"]
info = json.dumps(condainfo.info)
package = dao.get_package(channel_name, package_name)
if not package:
logger.debug(f"Creating package {package_name}")
dao.create_package(
channel_name,
rest_models.Package(
name=package_name,
summary=condainfo.about.get("summary", "n/a"),
description=condainfo.about.get("description", "n/a"),
),
user_id,
authorization.OWNER,
)
dao.update_package_channeldata(channel_name, package_name,
condainfo.channeldata)
try:
logger.debug(
f"Adding package {channel_name}/{platform}/{package_name}" +
f"-{version}-{build_string}")
dao.create_version(
channel_name=channel_name,
package_name=package_name,
package_format=package_format,
platform=platform,
version=version,
build_number=build_number,
build_string=build_string,
size=size,
filename=filename,
info=info,
uploader_id=user_id,
upsert=True,
)
dao.db.commit()
except IntegrityError:
dao.rollback()
logger.error(f"Duplicate package {channel_name}/{package_name}" +
"-{condainfo.info['version']}")
dao.db.commit()
def handle_file(
channel_name,
filename,
file_buffer,
dao,
user_id,
):
logger.debug(f"adding file '{filename}' to channel '{channel_name}'")
condainfo = CondaInfo(file_buffer, filename)
package_name = condainfo.info["name"]
package = dao.get_package(channel_name, package_name)
if not package:
dao.create_package(
channel_name,
rest_models.Package(
name=package_name,
summary=condainfo.about.get("summary", "n/a"),
description=condainfo.about.get("description", "n/a"),
),
user_id,
authorization.OWNER,
)
# Update channeldata info
dao.update_package_channeldata(channel_name, package_name,
condainfo.channeldata)
filename = os.path.split(filename)[-1]
try:
version = dao.create_version(
channel_name=channel_name,
package_name=package_name,
package_format=condainfo.package_format,
platform=condainfo.info["subdir"],
version=condainfo.info["version"],
build_number=condainfo.info["build_number"],
build_string=condainfo.info["build"],
size=condainfo.info["size"],
filename=filename,
info=json.dumps(condainfo.info),
uploader_id=user_id,
upsert=False,
)
except IntegrityError:
logger.error(
f"duplicate package '{package_name}' in channel '{channel_name}'")
raise
return version
def package(dao, user, channel, db):
new_package_data = rest_models.Package(name="test-package")
package = dao.create_package(
channel.name,
new_package_data,
user_id=user.id,
role="owner",
)
yield package
db.delete(package)
db.commit()
def test_rollback_on_collision(dao: Dao, db, dao_extra, user_with_channel):
"""testing rollback on concurrent writes."""
new_package = rest_models.Package(name=f"new-package-{uuid.uuid4()}")
user_id = user_with_channel
channel_name = "new-test-channel"
dao.create_package(channel_name, new_package, user_id, "owner")
with pytest.raises(errors.DBError,
match="(IntegrityError)|(UniqueViolation)"):
dao_extra.create_package(channel_name, new_package, user_id, "owner")
requested = db.query(Package).filter(
Package.name == new_package.name).one_or_none()
assert requested
# need to clean up because we didn't run the test in a transaction
db.delete(requested)
db.commit()
def handle_package_files(
channel,
files,
dao,
auth,
force,
package=None,
is_mirror_op=False,
):
user_id = auth.assert_user()
# quick fail if not allowed to upload
# note: we're checking later that `parts[0] == conda_info.package_name`
total_size = 0
for file in files:
parts = file.filename.rsplit("-", 2)
if len(parts) != 3:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"package file name has wrong format {file.filename}",
)
else:
package_name = parts[0]
auth.assert_upload_file(channel.name, package_name)
if force:
auth.assert_overwrite_package_version(channel.name, package_name)
# workaround for https://github.com/python/cpython/pull/3249
if type(file.file) is SpooledTemporaryFile and not hasattr(
file, "seekable"):
file.file.seekable = file.file._file.seekable
file.file.seek(0, os.SEEK_END)
size = file.file.tell()
total_size += size
file.file.seek(0)
dao.assert_size_limits(channel.name, total_size)
channel_proxylist = []
if channel.mirror_mode:
if not is_mirror_op:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Cannot upload packages to mirror channel",
)
else:
channel_proxylist = json.loads(channel.channel_metadata).get(
'proxylist', [])
pkgstore.create_channel(channel.name)
nthreads = config.general_package_unpack_threads
with ThreadPoolExecutor(max_workers=nthreads) as executor:
try:
conda_infos = [
ci for ci in executor.map(
_extract_and_upload_package,
files,
(channel.name, ) * len(files),
(channel_proxylist, ) * len(files),
)
]
except exceptions.PackageError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST,
detail=e.detail)
conda_infos = [ci for ci in conda_infos if ci is not None]
for file, condainfo in zip(files, conda_infos):
logger.debug(f"Handling {condainfo.info['name']} -> {file.filename}")
package_type = "tar.bz2" if file.filename.endswith(
".tar.bz2") else "conda"
UPLOAD_COUNT.labels(
channel=channel.name,
platform=condainfo.info["subdir"],
package_name=condainfo.info["name"],
version=condainfo.info["version"],
package_type=package_type,
).inc()
package_name = condainfo.info["name"]
parts = file.filename.rsplit("-", 2)
# check that the filename matches the package name
# TODO also validate version and build string
if parts[0] != condainfo.info["name"]:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="filename does not match package name",
)
if package and (parts[0] != package.name
or package_name != package.name):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=(
f"requested package endpoint '{package.name}'"
f"does not match the uploaded package name '{parts[0]}'"),
)
def _delete_file(condainfo, filename):
dest = os.path.join(condainfo.info["subdir"], file.filename)
pkgstore.delete_file(channel.name, dest)
if not package and not dao.get_package(channel.name, package_name):
try:
if not channel_proxylist or package_name not in channel_proxylist:
pm.hook.validate_new_package(
channel_name=channel.name,
package_name=package_name,
file_handler=file.file,
condainfo=condainfo,
)
# validate uploaded package size and existence
try:
pkgsize, _, _ = pkgstore.get_filemetadata(
channel.name,
f"{condainfo.info['subdir']}/{file.filename}")
if pkgsize != condainfo.info['size']:
raise errors.ValidationError(
f"Uploaded package {file.filename} "
"file size is wrong! Deleting")
except FileNotFoundError:
raise errors.ValidationError(
f"Uploaded package {file.filename} "
"file did not upload correctly!")
package_data = rest_models.Package(
name=package_name,
summary=str(condainfo.about.get("summary", "n/a")),
description=str(condainfo.about.get("description", "n/a")),
)
except pydantic.main.ValidationError as err:
_delete_file(condainfo, file.filename)
raise errors.ValidationError(
"Validation Error for package: " +
f"{channel.name}/{file.filename}: {str(err)}")
except errors.ValidationError as err:
_delete_file(condainfo, file.filename)
logger.error(
f"Validation error in: {channel.name}/{file.filename}: {str(err)}"
)
raise err
dao.create_package(
channel.name,
package_data,
user_id,
authorization.OWNER,
)
# Update channeldata info
dao.update_package_channeldata(channel.name, package_name,
condainfo.channeldata)
try:
version = dao.create_version(
channel_name=channel.name,
package_name=package_name,
package_format=condainfo.package_format,
platform=condainfo.info["subdir"],
version=condainfo.info["version"],
build_number=condainfo.info["build_number"],
build_string=condainfo.info["build"],
size=condainfo.info["size"],
filename=file.filename,
info=json.dumps(condainfo.info),
uploader_id=user_id,
upsert=force,
)
except IntegrityError:
logger.error(
f"duplicate package '{package_name}' in channel '{channel.name}'"
)
raise HTTPException(status_code=status.HTTP_409_CONFLICT,
detail="Duplicate")
pm.hook.post_add_package_version(version=version, condainfo=condainfo)
def handle_package_files(channel_name,
files,
dao,
auth,
force,
background_tasks,
package=None):
for file in files:
condainfo = CondaInfo(file.file, file.filename)
package_name = condainfo.info['name']
if force:
auth.assert_overwrite_package_version(channel_name, package_name)
parts = file.filename.split('-')
if package and (parts[0] != package.name
or package_name != package.name):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST)
if not package and not dao.get_package(channel_name, package_name):
dao.create_package(
channel_name,
rest_models.Package(
name=package_name,
summary=condainfo.about.get('summary', 'n/a'),
description=condainfo.about.get('description', 'n/a'),
),
auth.assert_user(),
authorization.OWNER,
)
# Update channeldata info
dao.update_package_channeldata(channel_name, package_name,
condainfo.channeldata)
auth.assert_upload_file(channel_name, package_name)
user_id = auth.assert_user()
try:
dao.create_version(
channel_name=channel_name,
package_name=package_name,
package_format=condainfo.package_format,
platform=condainfo.info['subdir'],
version=condainfo.info['version'],
build_number=condainfo.info['build_number'],
build_string=condainfo.info['build'],
filename=file.filename,
info=json.dumps(condainfo.info),
uploader_id=user_id,
)
except IntegrityError:
if force:
dao.rollback()
else:
raise HTTPException(status_code=status.HTTP_409_CONFLICT,
detail="Duplicate")
pkgstore.create_channel(channel_name)
dest = os.path.join(condainfo.info["subdir"], file.filename)
file.file._file.seek(0)
pkgstore.add_package(file.file, channel_name, dest)
# Background task to update indexes
background_tasks.add_task(indexing.update_indexes, dao, pkgstore,
channel_name)
def handle_package_files(
channel_name,
files,
dao,
auth,
force,
package=None,
):
for file in files:
logger.debug(
f"adding file '{file.filename}' to channel '{channel_name}'")
try:
condainfo = CondaInfo(file.file, file.filename)
except exceptions.PackageError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST,
detail=e.detail)
package_name = condainfo.info["name"]
if force:
auth.assert_overwrite_package_version(channel_name, package_name)
parts = file.filename.split("-")
if package and (parts[0] != package.name
or package_name != package.name):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST)
if not package and not dao.get_package(channel_name, package_name):
dao.create_package(
channel_name,
rest_models.Package(
name=package_name,
summary=condainfo.about.get("summary", "n/a"),
description=condainfo.about.get("description", "n/a"),
),
auth.assert_user(),
authorization.OWNER,
)
# Update channeldata info
dao.update_package_channeldata(channel_name, package_name,
condainfo.channeldata)
auth.assert_upload_file(channel_name, package_name)
user_id = auth.assert_user()
try:
version = dao.create_version(
channel_name=channel_name,
package_name=package_name,
package_format=condainfo.package_format,
platform=condainfo.info["subdir"],
version=condainfo.info["version"],
build_number=condainfo.info["build_number"],
build_string=condainfo.info["build"],
filename=file.filename,
info=json.dumps(condainfo.info),
uploader_id=user_id,
upsert=force,
)
except IntegrityError:
logger.error(
f"duplicate package '{package_name}' in channel '{channel_name}'"
)
raise HTTPException(status_code=status.HTTP_409_CONFLICT,
detail="Duplicate")
pkgstore.create_channel(channel_name)
dest = os.path.join(condainfo.info["subdir"], file.filename)
file.file._file.seek(0)
logger.debug(
f"uploading file {dest} from channel {channel_name} to package store"
)
pkgstore.add_package(file.file, channel_name, dest)
pm.hook.post_add_package_version(version=version, condainfo=condainfo)
def handle_package_files(
channel_name,
files,
dao,
auth,
force,
package=None,
):
user_id = auth.assert_user()
# quick fail if not allowed to upload
# note: we're checking later that `parts[0] == condainfo.package_name`
for file in files:
logger.info(f"FILE NAME: {file.filename}")
parts = file.filename.rsplit("-", 2)
if len(parts) != 3:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail="package filename wrong"
)
else:
package_name = parts[0]
auth.assert_upload_file(channel_name, package_name)
if force:
auth.assert_overwrite_package_version(channel_name, package_name)
logger.debug(f"adding file '{file.filename}' to channel '{channel_name}'")
pkgstore.create_channel(channel_name)
with TicToc("condainfos"):
with ThreadPoolExecutor(max_workers=10) as executor:
try:
condainfos = [
ci
for ci in executor.map(
lambda file: _upload_package(channel_name, file, pkgstore),
files,
)
]
except exceptions.PackageError as e:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail=e.detail
)
for file, condainfo in zip(files, condainfos):
logger.debug(f"Handling {condainfo.info['name']} -> {file.filename}")
package_name = condainfo.info["name"]
parts = file.filename.rsplit("-", 2)
# check that the filename matches the package name
# TODO also validate version and build string
if parts[0] != condainfo.info["name"]:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST)
if package and (parts[0] != package.name or package_name != package.name):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST)
if not package and not dao.get_package(channel_name, package_name):
dao.create_package(
channel_name,
rest_models.Package(
name=package_name,
summary=condainfo.about.get("summary", "n/a"),
description=condainfo.about.get("description", "n/a"),
),
user_id,
authorization.OWNER,
)
# Update channeldata info
dao.update_package_channeldata(
channel_name, package_name, condainfo.channeldata
)
try:
version = dao.create_version(
channel_name=channel_name,
package_name=package_name,
package_format=condainfo.package_format,
platform=condainfo.info["subdir"],
version=condainfo.info["version"],
build_number=condainfo.info["build_number"],
build_string=condainfo.info["build"],
filename=file.filename,
info=json.dumps(condainfo.info),
uploader_id=user_id,
upsert=force,
)
except IntegrityError:
logger.error(
f"duplicate package '{package_name}' in channel '{channel_name}'"
)
raise HTTPException(
status_code=status.HTTP_409_CONFLICT, detail="Duplicate"
)
with TicToc("Executing post hooks"):
pm.hook.post_add_package_version(version=version, condainfo=condainfo)
