def create_version_from_metadata( channel_name: str, user_id: bytes, package_file_name: str, package_data: dict, dao: Dao, ): package_name = package_data["name"] package = dao.get_package(channel_name, package_name) if not package: package_info = rest_models.Package( name=package_name, summary=package_data.get("summary", ""), description=package_data.get("description", ""), ) dao.create_package(channel_name, package_info, user_id, "owner") pkg_format = "tarbz2" if package_file_name.endswith( ".tar.bz2") else ".conda" version = dao.create_version( channel_name, package_name, pkg_format, get_subdir_compat(package_data), package_data["version"], int(package_data["build_number"]), package_data["build"], package_file_name, json.dumps(package_data), user_id, package_data["size"], ) return version
def package_version(user, mirror_channel, db, dao): # create package version that will added to local repodata package_format = "tarbz2" package_info = ( '{"size": 5000, "sha256": "OLD-SHA", "md5": "OLD-MD5", "subdirs":["noarch"]}' ) new_package_data = rest_models.Package(name="test-package") package = dao.create_package( mirror_channel.name, new_package_data, user_id=user.id, role="owner", ) version = dao.create_version( mirror_channel.name, "test-package", package_format, "noarch", "0.1", "0", "", "test-package-0.1-0.tar.bz2", package_info, user.id, ) yield version db.delete(package) db.delete(version) db.commit()
def create_packages_from_channeldata(channel_name: str, user_id: bytes, channeldata: dict, dao: Dao): packages = channeldata.get("packages", {}) for package_name, metadata in packages.items(): description = metadata.get("description", "") summary = metadata.get("summary", "") package_data = rest_models.Package( name=package_name, summary=summary, description=description, ) try: package = dao.create_package(channel_name, package_data, user_id, role=authorization.OWNER) except DBError: # package already exists so skip it so we retrieve and update it package = dao.get_package(channel_name, package_name) package.description = description package.summary = summary package.url = metadata.get("home", "") package.platforms = ":".join(metadata.get("subdirs", [])) package.channeldata = json.dumps(metadata) dao.db.commit()
def package(dao, channel, package_name, user, db): package_data = rest_models.Package(name=package_name) package = dao.create_package(channel.name, package_data, user.id, "owner") yield package db.delete(package) db.commit()
def handle_file(channel_name, condainfo, dao, user_id): """Add or update conda package info to database""" filename = os.path.split(condainfo._filename)[-1] package_name = condainfo.info["name"] package_format = condainfo.package_format platform = condainfo.info["subdir"] version = condainfo.info["version"] build_number = condainfo.info["build_number"] build_string = condainfo.info["build"] size = condainfo.info["size"] info = json.dumps(condainfo.info) package = dao.get_package(channel_name, package_name) if not package: logger.debug(f"Creating package {package_name}") dao.create_package( channel_name, rest_models.Package( name=package_name, summary=condainfo.about.get("summary", "n/a"), description=condainfo.about.get("description", "n/a"), ), user_id, authorization.OWNER, ) dao.update_package_channeldata(channel_name, package_name, condainfo.channeldata) try: logger.debug( f"Adding package {channel_name}/{platform}/{package_name}" + f"-{version}-{build_string}") dao.create_version( channel_name=channel_name, package_name=package_name, package_format=package_format, platform=platform, version=version, build_number=build_number, build_string=build_string, size=size, filename=filename, info=info, uploader_id=user_id, upsert=True, ) dao.db.commit() except IntegrityError: dao.rollback() logger.error(f"Duplicate package {channel_name}/{package_name}" + "-{condainfo.info['version']}") dao.db.commit()
def handle_file( channel_name, filename, file_buffer, dao, user_id, ): logger.debug(f"adding file '{filename}' to channel '{channel_name}'") condainfo = CondaInfo(file_buffer, filename) package_name = condainfo.info["name"] package = dao.get_package(channel_name, package_name) if not package: dao.create_package( channel_name, rest_models.Package( name=package_name, summary=condainfo.about.get("summary", "n/a"), description=condainfo.about.get("description", "n/a"), ), user_id, authorization.OWNER, ) # Update channeldata info dao.update_package_channeldata(channel_name, package_name, condainfo.channeldata) filename = os.path.split(filename)[-1] try: version = dao.create_version( channel_name=channel_name, package_name=package_name, package_format=condainfo.package_format, platform=condainfo.info["subdir"], version=condainfo.info["version"], build_number=condainfo.info["build_number"], build_string=condainfo.info["build"], size=condainfo.info["size"], filename=filename, info=json.dumps(condainfo.info), uploader_id=user_id, upsert=False, ) except IntegrityError: logger.error( f"duplicate package '{package_name}' in channel '{channel_name}'") raise return version
def package(dao, user, channel, db): new_package_data = rest_models.Package(name="test-package") package = dao.create_package( channel.name, new_package_data, user_id=user.id, role="owner", ) yield package db.delete(package) db.commit()
def test_rollback_on_collision(dao: Dao, db, dao_extra, user_with_channel): """testing rollback on concurrent writes.""" new_package = rest_models.Package(name=f"new-package-{uuid.uuid4()}") user_id = user_with_channel channel_name = "new-test-channel" dao.create_package(channel_name, new_package, user_id, "owner") with pytest.raises(errors.DBError, match="(IntegrityError)|(UniqueViolation)"): dao_extra.create_package(channel_name, new_package, user_id, "owner") requested = db.query(Package).filter( Package.name == new_package.name).one_or_none() assert requested # need to clean up because we didn't run the test in a transaction db.delete(requested) db.commit()
def handle_package_files( channel, files, dao, auth, force, package=None, is_mirror_op=False, ): user_id = auth.assert_user() # quick fail if not allowed to upload # note: we're checking later that `parts[0] == conda_info.package_name` total_size = 0 for file in files: parts = file.filename.rsplit("-", 2) if len(parts) != 3: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=f"package file name has wrong format {file.filename}", ) else: package_name = parts[0] auth.assert_upload_file(channel.name, package_name) if force: auth.assert_overwrite_package_version(channel.name, package_name) # workaround for https://github.com/python/cpython/pull/3249 if type(file.file) is SpooledTemporaryFile and not hasattr( file, "seekable"): file.file.seekable = file.file._file.seekable file.file.seek(0, os.SEEK_END) size = file.file.tell() total_size += size file.file.seek(0) dao.assert_size_limits(channel.name, total_size) channel_proxylist = [] if channel.mirror_mode: if not is_mirror_op: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Cannot upload packages to mirror channel", ) else: channel_proxylist = json.loads(channel.channel_metadata).get( 'proxylist', []) pkgstore.create_channel(channel.name) nthreads = config.general_package_unpack_threads with ThreadPoolExecutor(max_workers=nthreads) as executor: try: conda_infos = [ ci for ci in executor.map( _extract_and_upload_package, files, (channel.name, ) * len(files), (channel_proxylist, ) * len(files), ) ] except exceptions.PackageError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=e.detail) conda_infos = [ci for ci in conda_infos if ci is not None] for file, condainfo in zip(files, conda_infos): logger.debug(f"Handling {condainfo.info['name']} -> {file.filename}") package_type = "tar.bz2" if file.filename.endswith( ".tar.bz2") else "conda" UPLOAD_COUNT.labels( channel=channel.name, platform=condainfo.info["subdir"], package_name=condainfo.info["name"], version=condainfo.info["version"], package_type=package_type, ).inc() package_name = condainfo.info["name"] parts = file.filename.rsplit("-", 2) # check that the filename matches the package name # TODO also validate version and build string if parts[0] != condainfo.info["name"]: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="filename does not match package name", ) if package and (parts[0] != package.name or package_name != package.name): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=( f"requested package endpoint '{package.name}'" f"does not match the uploaded package name '{parts[0]}'"), ) def _delete_file(condainfo, filename): dest = os.path.join(condainfo.info["subdir"], file.filename) pkgstore.delete_file(channel.name, dest) if not package and not dao.get_package(channel.name, package_name): try: if not channel_proxylist or package_name not in channel_proxylist: pm.hook.validate_new_package( channel_name=channel.name, package_name=package_name, file_handler=file.file, condainfo=condainfo, ) # validate uploaded package size and existence try: pkgsize, _, _ = pkgstore.get_filemetadata( channel.name, f"{condainfo.info['subdir']}/{file.filename}") if pkgsize != condainfo.info['size']: raise errors.ValidationError( f"Uploaded package {file.filename} " "file size is wrong! Deleting") except FileNotFoundError: raise errors.ValidationError( f"Uploaded package {file.filename} " "file did not upload correctly!") package_data = rest_models.Package( name=package_name, summary=str(condainfo.about.get("summary", "n/a")), description=str(condainfo.about.get("description", "n/a")), ) except pydantic.main.ValidationError as err: _delete_file(condainfo, file.filename) raise errors.ValidationError( "Validation Error for package: " + f"{channel.name}/{file.filename}: {str(err)}") except errors.ValidationError as err: _delete_file(condainfo, file.filename) logger.error( f"Validation error in: {channel.name}/{file.filename}: {str(err)}" ) raise err dao.create_package( channel.name, package_data, user_id, authorization.OWNER, ) # Update channeldata info dao.update_package_channeldata(channel.name, package_name, condainfo.channeldata) try: version = dao.create_version( channel_name=channel.name, package_name=package_name, package_format=condainfo.package_format, platform=condainfo.info["subdir"], version=condainfo.info["version"], build_number=condainfo.info["build_number"], build_string=condainfo.info["build"], size=condainfo.info["size"], filename=file.filename, info=json.dumps(condainfo.info), uploader_id=user_id, upsert=force, ) except IntegrityError: logger.error( f"duplicate package '{package_name}' in channel '{channel.name}'" ) raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Duplicate") pm.hook.post_add_package_version(version=version, condainfo=condainfo)
def handle_package_files(channel_name, files, dao, auth, force, background_tasks, package=None): for file in files: condainfo = CondaInfo(file.file, file.filename) package_name = condainfo.info['name'] if force: auth.assert_overwrite_package_version(channel_name, package_name) parts = file.filename.split('-') if package and (parts[0] != package.name or package_name != package.name): raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST) if not package and not dao.get_package(channel_name, package_name): dao.create_package( channel_name, rest_models.Package( name=package_name, summary=condainfo.about.get('summary', 'n/a'), description=condainfo.about.get('description', 'n/a'), ), auth.assert_user(), authorization.OWNER, ) # Update channeldata info dao.update_package_channeldata(channel_name, package_name, condainfo.channeldata) auth.assert_upload_file(channel_name, package_name) user_id = auth.assert_user() try: dao.create_version( channel_name=channel_name, package_name=package_name, package_format=condainfo.package_format, platform=condainfo.info['subdir'], version=condainfo.info['version'], build_number=condainfo.info['build_number'], build_string=condainfo.info['build'], filename=file.filename, info=json.dumps(condainfo.info), uploader_id=user_id, ) except IntegrityError: if force: dao.rollback() else: raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Duplicate") pkgstore.create_channel(channel_name) dest = os.path.join(condainfo.info["subdir"], file.filename) file.file._file.seek(0) pkgstore.add_package(file.file, channel_name, dest) # Background task to update indexes background_tasks.add_task(indexing.update_indexes, dao, pkgstore, channel_name)
def handle_package_files( channel_name, files, dao, auth, force, package=None, ): for file in files: logger.debug( f"adding file '{file.filename}' to channel '{channel_name}'") try: condainfo = CondaInfo(file.file, file.filename) except exceptions.PackageError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=e.detail) package_name = condainfo.info["name"] if force: auth.assert_overwrite_package_version(channel_name, package_name) parts = file.filename.split("-") if package and (parts[0] != package.name or package_name != package.name): raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST) if not package and not dao.get_package(channel_name, package_name): dao.create_package( channel_name, rest_models.Package( name=package_name, summary=condainfo.about.get("summary", "n/a"), description=condainfo.about.get("description", "n/a"), ), auth.assert_user(), authorization.OWNER, ) # Update channeldata info dao.update_package_channeldata(channel_name, package_name, condainfo.channeldata) auth.assert_upload_file(channel_name, package_name) user_id = auth.assert_user() try: version = dao.create_version( channel_name=channel_name, package_name=package_name, package_format=condainfo.package_format, platform=condainfo.info["subdir"], version=condainfo.info["version"], build_number=condainfo.info["build_number"], build_string=condainfo.info["build"], filename=file.filename, info=json.dumps(condainfo.info), uploader_id=user_id, upsert=force, ) except IntegrityError: logger.error( f"duplicate package '{package_name}' in channel '{channel_name}'" ) raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Duplicate") pkgstore.create_channel(channel_name) dest = os.path.join(condainfo.info["subdir"], file.filename) file.file._file.seek(0) logger.debug( f"uploading file {dest} from channel {channel_name} to package store" ) pkgstore.add_package(file.file, channel_name, dest) pm.hook.post_add_package_version(version=version, condainfo=condainfo)
def handle_package_files( channel_name, files, dao, auth, force, package=None, ): user_id = auth.assert_user() # quick fail if not allowed to upload # note: we're checking later that `parts[0] == condainfo.package_name` for file in files: logger.info(f"FILE NAME: {file.filename}") parts = file.filename.rsplit("-", 2) if len(parts) != 3: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="package filename wrong" ) else: package_name = parts[0] auth.assert_upload_file(channel_name, package_name) if force: auth.assert_overwrite_package_version(channel_name, package_name) logger.debug(f"adding file '{file.filename}' to channel '{channel_name}'") pkgstore.create_channel(channel_name) with TicToc("condainfos"): with ThreadPoolExecutor(max_workers=10) as executor: try: condainfos = [ ci for ci in executor.map( lambda file: _upload_package(channel_name, file, pkgstore), files, ) ] except exceptions.PackageError as e: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=e.detail ) for file, condainfo in zip(files, condainfos): logger.debug(f"Handling {condainfo.info['name']} -> {file.filename}") package_name = condainfo.info["name"] parts = file.filename.rsplit("-", 2) # check that the filename matches the package name # TODO also validate version and build string if parts[0] != condainfo.info["name"]: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST) if package and (parts[0] != package.name or package_name != package.name): raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST) if not package and not dao.get_package(channel_name, package_name): dao.create_package( channel_name, rest_models.Package( name=package_name, summary=condainfo.about.get("summary", "n/a"), description=condainfo.about.get("description", "n/a"), ), user_id, authorization.OWNER, ) # Update channeldata info dao.update_package_channeldata( channel_name, package_name, condainfo.channeldata ) try: version = dao.create_version( channel_name=channel_name, package_name=package_name, package_format=condainfo.package_format, platform=condainfo.info["subdir"], version=condainfo.info["version"], build_number=condainfo.info["build_number"], build_string=condainfo.info["build"], filename=file.filename, info=json.dumps(condainfo.info), uploader_id=user_id, upsert=force, ) except IntegrityError: logger.error( f"duplicate package '{package_name}' in channel '{channel_name}'" ) raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail="Duplicate" ) with TicToc("Executing post hooks"): pm.hook.post_add_package_version(version=version, condainfo=condainfo)