def reindex_packages_from_store( dao: Dao, config: Config, channel_name: str, user_id: bytes, ): """Reindex packages from files in the package store""" db = dao.db pkgstore = config.get_package_store() all_files = pkgstore.list_files(channel_name) pkg_files = [f for f in all_files if f.endswith(".tar.bz2")] channel = dao.get_channel(channel_name) if channel: for package in channel.packages: db.delete(package) db.commit() else: data = rest_models.Channel(name=channel_name, description="re-indexed from files", private=True) channel = dao.create_channel(data, user_id, authorization.OWNER) for fname in pkg_files: fid = pkgstore.serve_path(channel_name, fname) handle_file(channel_name, fname, fid, dao, user_id) update_indexes(dao, pkgstore, channel_name)
def test_channel_with_huge_size_limit(dao, user, db): channel_data = rest_models.Channel(name="my-channel", private=False, size_limit=1000000000000000000) channel = dao.create_channel(channel_data, user.id, "owner", 1000000000000000000) del channel
def channel(dao, db, user, channel_name): channel_data = rest_models.Channel(name=channel_name, private=False) channel = dao.create_channel(channel_data, user.id, "owner") yield channel db.delete(channel) db.commit()
def channel(dao, db, user, channel_name): channel_data = rest_models.Channel(name=channel_name, private=False) channel = dao.create_channel(channel_data, user.id, "owner") yield channel try: db.delete(channel) db.commit() except ObjectDeletedError: pass
def user_with_channel(dao, db): channel_data = rest_models.Channel(name="new-test-channel", private=False) user = dao.create_user_with_role("new-user") user_id = user.id channel = dao.create_channel(channel_data, user_id, "owner") db.commit() yield user_id db.delete(channel) db.delete(user) db.commit()
def channel(dao, user, db): channel_data = rest_models.Channel( name="test_channel", private=False, ) channel = dao.create_channel(channel_data, user.id, "owner") yield channel db.delete(channel) db.commit()
def channel(dao, db, user, channel_role): channel_data = rest_models.Channel( name="test-channel", private=False, ) if channel_role: c = dao.create_channel(channel_data, user_id=user.id, role=channel_role) else: c = dao.create_channel(channel_data) yield c
def mirror_channel(dao, user, db): channel_data = rest_models.Channel( name="test_mirror_channel", private=False, mirror_channel_url="http://host", mirror_mode="mirror", ) channel = dao.create_channel(channel_data, user.id, "owner") yield channel db.delete(channel) db.commit()
def create_user_with_identity( dao: Dao, provider: str, profile: 'base.UserProfile', default_role: Optional[str], default_channels: Optional[List[str]], ) -> User: username = profile["login"] user = dao.create_user_with_profile( username=username, provider=provider, emails=profile.get("emails", []), identity_id=profile["id"], name=profile["name"], avatar_url=profile["avatar_url"], role=default_role, exist_ok=False, ) if default_channels is not None: for channel_name in default_channels: i = 0 while (dao.db.query(Channel).filter( Channel.name == channel_name).one_or_none()): channel_name = f"{username}-{i}" i += 1 channel_meta = rest_models.Channel( name=channel_name, description=f"{username}'s default channel", private=True, ) dao.create_channel(channel_meta, user.id, OWNER) return user
def test_create_channel_with_invalid_name(dao, user, db): with pytest.raises(errors.ValidationError): channel_data = rest_models.Channel(name="my_channel", private=False) channel = dao.create_channel(channel_data, user.id, "owner") del channel
def reindex_packages_from_store(dao: Dao, config: Config, channel_name: str, user_id, sync: bool = True): """Reindex packages from files in the package store""" logger.debug(f"Re-indexing channel {channel_name}") channel = dao.get_channel(channel_name) pkg_db = [] if channel: if not sync: for package in channel.packages: dao.db.delete(package) dao.db.commit() else: dao.cleanup_channel_db(channel_name) for package in channel.packages: for pv in package.package_versions: # type: ignore pkg_db.append(f"{pv.platform}/{pv.filename}") dao.db.commit() else: data = rest_models.Channel(name=channel_name, description="re-indexed from store", private=True) channel = dao.create_channel(data, user_id, authorization.OWNER) logger.debug(f"Reading package list for channel {channel_name}") user_id = uuid_to_bytes(user_id) pkgstore = config.get_package_store() all_files = pkgstore.list_files(channel_name) pkg_files = [f for f in all_files if f.endswith(".tar.bz2")] nthreads = config.general_package_unpack_threads logger.debug( f"Found {len(pkg_db)} packages for channel {channel_name} in database") logger.debug( f"Found {len(pkg_files)} packages for channel {channel_name} in pkgstore" ) pkg_files = list(set(pkg_files) - set(pkg_db)) logger.debug( f"Importing {len(pkg_files)} packages for channel {channel_name}" + " from pkgstore") for pkg_group in chunks(pkg_files, nthreads * 8): tic = time.perf_counter() with ThreadPoolExecutor(max_workers=nthreads) as executor: results = [] for fname in pkg_group: results.append( executor.submit(handle_condainfo, pkgstore, channel_name, fname)) for future in as_completed(results): condainfo = future.result() if condainfo: handle_file(channel_name, condainfo, dao, user_id) toc = time.perf_counter() logger.debug( f"Imported files {pkg_group[0]} to {pkg_group[-1]} " + f"for channel {channel_name} in {toc - tic:0.4f} seconds " + f"using {nthreads} threads") try: update_indexes(dao, pkgstore, channel_name) dao.db.commit() except IntegrityError: dao.rollback() logger.error(f"Update index {channel_name} failed") dao.cleanup_channel_db(channel_name) dao.db.commit()