def post_package_indexing(tempdir: Path, channel_name, subdirs, files, packages): with get_db_manager() as db: # the most recent created key is fetched since we # cannot get `user_id` outside a request / API call. query = (db.query(db_models.RepodataSigningKey).filter( db_models.RepodataSigningKey.channel_name == channel_name, ).order_by(desc('time_created')).first()) if query: for subdir in subdirs: repodata_folderpath = tempdir / channel_name / subdir RepoSigner(repodata_folderpath, query.private_key) with open(tempdir / channel_name / subdir / "repodata_signed.json") as f: repodata_signed = f.read() add_temp_static_file( repodata_signed, channel_name, subdir, "repodata_signed.json", tempdir, files, )
def post_package_indexing(tempdir: Path, channel_name, subdirs, files, packages): with get_db_manager() as db: query = ( db.query(PackageVersion).filter( PackageVersion.channel_name == channel_name, PackageVersion.package_name == f"{channel_name}-repodata-patches", PackageVersion.version_order == 0, # newest patch package ).order_by(PackageVersion.version.desc())) patches_pkg = query.one_or_none() if patches_pkg: filename = patches_pkg.filename fs = pkgstore.serve_path(channel_name, "noarch/" + filename) package_format = patches_pkg.package_format if package_format == PackageFormatEnum.tarbz2: extract_ = extract_from_tarfile else: extract_ = extract_from_conda with extract_(fs) as tar: for subdir in subdirs: packages[subdir] = {} path = f"{subdir}/patch_instructions.json" patch_instructions = _load_instructions(tar, path) with open(tempdir / channel_name / subdir / "repodata.json") as fs: repodata_str = fs.read() repodata = json.loads(repodata_str) add_temp_static_file( repodata_str, channel_name, subdir, "repodata_from_packages.json", tempdir, files, ) patch_repodata(repodata, patch_instructions) packages[subdir].update(repodata["packages"]) packages[subdir].update(repodata["packages.conda"]) patched_repodata_str = json.dumps(repodata) add_temp_static_file( patched_repodata_str, channel_name, subdir, "repodata.json", tempdir, files, )
def post_package_indexing(tempdir: Path, channel_name, subdirs, files, packages): pins = {} for subdir in subdirs: with open(tempdir / channel_name / subdir / "repodata.json") as f: repodata = json.load(f) current_repodata = _build_current_repodata(subdir, repodata, pins) current_repodata_string = json.dumps(current_repodata, indent=2, sort_keys=True) add_temp_static_file( current_repodata_string, channel_name, subdir, "current_repodata.json", tempdir, files, )
def update_indexes(dao, pkgstore, channel_name, subdirs=None): jinjaenv = _jinjaenv() channeldata = channel_data.export(dao, channel_name) if subdirs is None: subdirs = sorted(channeldata["subdirs"], key=_subdir_key) # Generate channeldata.json and its compressed version chandata_json = json.dumps(channeldata, indent=2, sort_keys=False) add_static_file(chandata_json, channel_name, None, "channeldata.json", pkgstore) # Generate index.html for the "root" directory channel_index = jinjaenv.get_template("channeldata-index.html.j2").render( title=channel_name, packages=channeldata["packages"], subdirs=subdirs, current_time=datetime.now(timezone.utc), ) add_static_file(channel_index, channel_name, None, "index.html", pkgstore) # NB. No rss.xml is being generated here files = {} packages = {} subdir_template = jinjaenv.get_template("subdir-index.html.j2") tempdir = tempfile.TemporaryDirectory() tempdir_path = Path(tempdir.name) pm = quetz.config.get_plugin_manager() for sdir in subdirs: logger.debug( f"creating indexes for subdir {sdir} of channel {channel_name}") raw_repodata = repo_data.export(dao, channel_name, sdir) pm.hook.post_index_creation( raw_repodata=raw_repodata, channel_name=channel_name, subdir=sdir, ) files[sdir] = [] packages[sdir] = raw_repodata["packages"] repodata = json.dumps(raw_repodata, indent=2, sort_keys=False) add_temp_static_file(repodata, channel_name, sdir, "repodata.json", tempdir_path, files) pm.hook.post_package_indexing( tempdir=tempdir_path, channel_name=channel_name, subdirs=subdirs, files=files, packages=packages, ) for sdir in subdirs: # Generate subdir index.html subdir_index_html = subdir_template.render( title=f"{channel_name}/{sdir}", packages=packages[sdir], current_time=datetime.now(timezone.utc), add_files=files[sdir], ) add_static_file(subdir_index_html, channel_name, sdir, "index.html", pkgstore) # recursively walk through the tree tmp_suffix = uuid.uuid4().hex after_upload_move = [] for path in tempdir_path.rglob('*.*'): rel_path = path.relative_to(tempdir_path) to_upload = open(path, 'rb') if len(rel_path.parts) == 2: channel_name, filename = rel_path.parts dest = f"{filename}{tmp_suffix}" elif len(rel_path.parts) == 3: channel_name, sdir, filename = rel_path.parts dest = f"{sdir}/{filename}{tmp_suffix}" else: raise NotImplementedError( "We can only handle channel_name/subdir/file.xyz OR" "channel_name/file.xyz") logger.debug(f"Uploading {path} -> {channel_name} {dest}") if type(pkgstore).__name__ == "S3Store": with pkgstore._get_fs() as fs: bucket = pkgstore._bucket_map(channel_name) fs.put_file(path, f"{bucket}/{dest}") else: pkgstore.add_file(to_upload.read(), channel_name, dest) after_upload_move.append(dest) for f_to_move in after_upload_move: logger.debug("Moving to final destination: " f"{f_to_move} -> {f_to_move[:-len(tmp_suffix)]}") pkgstore.move_file(channel_name, f_to_move, f_to_move[:-len(tmp_suffix)]) tempdir.cleanup()