def reindex_packages_from_store(dao: Dao,
config: Config,
channel_name: str,
user_id,
sync: bool = True):
"""Reindex packages from files in the package store"""
logger.debug(f"Re-indexing channel {channel_name}")
channel = dao.get_channel(channel_name)
pkg_db = []
if channel:
if not sync:
for package in channel.packages:
dao.db.delete(package)
dao.db.commit()
else:
dao.cleanup_channel_db(channel_name)
for package in channel.packages:
for pv in package.package_versions: # type: ignore
pkg_db.append(f"{pv.platform}/{pv.filename}")
dao.db.commit()
else:
data = rest_models.Channel(name=channel_name,
description="re-indexed from store",
private=True)
channel = dao.create_channel(data, user_id, authorization.OWNER)
logger.debug(f"Reading package list for channel {channel_name}")
user_id = uuid_to_bytes(user_id)
pkgstore = config.get_package_store()
all_files = pkgstore.list_files(channel_name)
pkg_files = [f for f in all_files if f.endswith(".tar.bz2")]
nthreads = config.general_package_unpack_threads
logger.debug(
f"Found {len(pkg_db)} packages for channel {channel_name} in database")
logger.debug(
f"Found {len(pkg_files)} packages for channel {channel_name} in pkgstore"
)
pkg_files = list(set(pkg_files) - set(pkg_db))
logger.debug(
f"Importing {len(pkg_files)} packages for channel {channel_name}" +
" from pkgstore")
for pkg_group in chunks(pkg_files, nthreads * 8):
tic = time.perf_counter()
with ThreadPoolExecutor(max_workers=nthreads) as executor:
results = []
for fname in pkg_group:
results.append(
executor.submit(handle_condainfo, pkgstore, channel_name,
fname))
for future in as_completed(results):
condainfo = future.result()
if condainfo:
handle_file(channel_name, condainfo, dao, user_id)
toc = time.perf_counter()
logger.debug(
f"Imported files {pkg_group[0]} to {pkg_group[-1]} " +
f"for channel {channel_name} in {toc - tic:0.4f} seconds " +
f"using {nthreads} threads")
try:
update_indexes(dao, pkgstore, channel_name)
dao.db.commit()
except IntegrityError:
dao.rollback()
logger.error(f"Update index {channel_name} failed")
dao.cleanup_channel_db(channel_name)
dao.db.commit()
def cleanup_channel_db(dao: Dao, channel_name: str, dry_run: bool):
dao.cleanup_channel_db(channel_name, dry_run)