def post_package_indexing(tempdir, channel_name, subdirs, files, packages):
for subdir in subdirs:
fname = f"{channel_name}.{subdir}.map"
try:
f = pkgstore.serve_path(channel_name, f"{subdir}/{fname}")
data_bytes = f.read()
add_entry_for_index(files, subdir, fname, data_bytes)
except FileNotFoundError:
pass
def _addfile(content, path):
subdir_path = f"{subdir}/{path}"
if isinstance(content, str):
content = content.encode("utf-8")
compressed_content = bz2.compress(content)
pkgstore.add_file(content, channel_name, subdir_path)
pkgstore.add_file(compressed_content, channel_name,
subdir_path + ".bz2")
add_entry_for_index(files, subdir, path, content)
add_entry_for_index(files, subdir, f"{path}.bz2",
compressed_content)
def post_package_indexing(tempdir: Path, channel_name, subdirs, files, packages):
for subdir in subdirs:
path1 = tempdir / channel_name / subdir / "repodata.json"
path2 = tempdir / channel_name / subdir / "repodata.json.zck"
try:
subprocess.check_call(['zck', path1, '-o', path2])
except FileNotFoundError:
raise RuntimeError(
'zchunk does not seem to be installed, '
'you can install it with:\n'
'mamba install zchunk -c conda-forge'
)
except subprocess.CalledProcessError:
raise RuntimeError('Error calling zck on repodata.')
with open(path2, 'rb') as fi:
add_entry_for_index(files, subdir, "repodata.json.zck", fi.read())
def post_package_indexing(pkgstore: "quetz.pkgstores.PackageStore",
channel_name, subdirs, files, packages):
fname = "current_repodata.json"
pins = {}
for subdir in subdirs:
path = f"{subdir}/{fname}"
f = pkgstore.serve_path(channel_name, f"{subdir}/repodata.json")
repodata = json.load(f)
current_repodata = _build_current_repodata(subdir, repodata, pins)
raw_current_repodata = json.dumps(current_repodata,
indent=2,
sort_keys=True).encode("utf-8")
compressed_current_repodata = bz2.compress(raw_current_repodata)
pkgstore.add_file(raw_current_repodata, channel_name, path)
pkgstore.add_file(compressed_current_repodata, channel_name,
path + ".bz2")
add_entry_for_index(files, subdir, fname, raw_current_repodata)
add_entry_for_index(files, subdir, f"{fname}.bz2",
compressed_current_repodata)
def post_package_indexing(pkgstore: "quetz.pkgstores.PackageStore",
channel_name, subdirs, files, packages):
for subdir in subdirs:
fname = "repodata"
fs = pkgstore.serve_path(channel_name, f"{subdir}/{fname}.json")
repodata_str = fs.read()
repodata = json.loads(repodata_str)
_, path1 = tempfile.mkstemp()
_, path2 = tempfile.mkstemp()
with open(path1, 'wb') as f:
f.write(repodata_str)
try:
subprocess.check_call(['zck', path1, '-o', path2])
except FileNotFoundError:
raise RuntimeError('zchunk does not seem to be installed, '
'you can install it with:\n'
'mamba install zchunk -c conda-forge')
except subprocess.CalledProcessError:
raise RuntimeError('Error calling zck on repodata.')
finally:
os.remove(path1)
with open(path2, 'rb') as f:
repodata_zck = f.read()
os.remove(path2)
pkgstore.add_file(repodata_zck, channel_name,
f'{subdir}/{fname}.json.zck')
add_entry_for_index(files, subdir, f'{fname}.json.zck', repodata_zck)
pkgstore.add_file(repodata_str, channel_name, f'{subdir}/{fname}.json')
add_entry_for_index(files, subdir, f'{fname}.json', repodata_str)
repodata_bz2 = bz2.compress(repodata_str)
pkgstore.add_file(repodata_bz2, channel_name,
f'{subdir}/{fname}.json.bz2')
add_entry_for_index(files, subdir, f'{fname}.json.bz2', repodata_bz2)
packages[subdir] = repodata["packages"]
packages[subdir].update(repodata["packages.conda"])
def update_indexes(dao, pkgstore, channel_name, subdirs=None):
jinjaenv = _jinjaenv()
channeldata = channel_data.export(dao, channel_name)
if subdirs is None:
subdirs = sorted(channeldata["subdirs"], key=_subdir_key)
# Generate channeldata.json and its compressed version
chandata_json = json.dumps(channeldata, indent=2, sort_keys=True)
pkgstore.add_file(
bz2.compress(chandata_json.encode("utf-8")),
channel_name,
"channeldata.json.bz2",
)
pkgstore.add_file(chandata_json, channel_name, "channeldata.json")
# Generate index.html for the "root" directory
channel_template = jinjaenv.get_template("channeldata-index.html.j2")
pkgstore.add_file(
channel_template.render(
title=channel_name,
packages=channeldata["packages"],
subdirs=subdirs,
current_time=datetime.now(timezone.utc),
),
channel_name,
"index.html",
)
# NB. No rss.xml is being generated here
files = {}
packages = {}
subdir_template = jinjaenv.get_template("subdir-index.html.j2")
for dir in subdirs:
logger.debug(
f"creating indexes for subdir {dir} of channel {channel_name}")
raw_repodata = repo_data.export(dao, channel_name, dir)
repodata = json.dumps(raw_repodata, indent=2,
sort_keys=True).encode("utf-8")
compressed_repodata = bz2.compress(repodata)
files[dir] = []
packages[dir] = raw_repodata["packages"]
fname = "repodata.json"
pkgstore.add_file(compressed_repodata, channel_name,
f"{dir}/{fname}.bz2")
pkgstore.add_file(repodata, channel_name, f"{dir}/{fname}")
add_entry_for_index(files, dir, fname, repodata)
add_entry_for_index(files, dir, f"{fname}.bz2", compressed_repodata)
pm = quetz.config.get_plugin_manager()
pm.hook.post_package_indexing(
pkgstore=pkgstore,
channel_name=channel_name,
subdirs=subdirs,
files=files,
packages=packages,
)
for dir in subdirs:
# Generate subdir index.html
pkgstore.add_file(
subdir_template.render(
title=f"{channel_name}/{dir}",
packages=packages[dir],
current_time=datetime.now(timezone.utc),
add_files=files[dir],
),
channel_name,
f"{dir}/index.html",
)