def _process_doc(self, pages, path, root, block, doc): # Add Houdini-specific fields to each document path = paths.basepath(path) attrs = block.get("attrs", {}) doctype = attrs.get("type", "").strip() context = attrs.get("context", "").strip().replace(",", "") or None if doctype == "node": if context in ("pop", "part") or context.endswith("_state"): return internal = attrs.get("internal") if internal: doc["grams"] += " %s" % internal if doc.get("category") == "_": if path.startswith("/shelf/"): doc["category"] = "tool" elif path.startswith("/ref/util/"): doc["category"] = "utility" elif path.startswith("/gallery/shop/"): doc["category"] = "gallery/shop" elif doctype == "node": doc["category"] = "%s/%s" % (doctype, context) doc["grams"] += " %s" % context elif doctype in ("hscript", "expression", "example", "homclass", "hommodule", "vex"): doc["category"] = doctype replaces = attrs.get("replaces") rsection = functions.subblock_by_id(block, "replaces") if rsection: rlist = " ".join(link.get("fullpath", "") for link in functions.find_links(rsection)) if replaces: replaces = replaces + " " + rlist else: replaces = rlist doc.update({ "context": context, "bestbet": attrs.get("bestbet"), "helpid": attrs.get("helpid"), "superclass": attrs.get("superclass"), "version": attrs.get("version"), "replaces": replaces or None, "examplefor": root.get("examplefor"), "examplefile": root.get("examplefile"), "group": attrs.get("group"), }) # Add example file info if root is block and "examplefile" in root: otlpath = root["examplefile"] # Usages file should be in the same location with .usages ext usagespath = paths.basepath(otlpath) + ".usages" if pages.exists(usagespath): usagescontent = pages.content(usagespath) usages = ws_exp.split(usagescontent) doc["uses"] = " ".join(usages)
def missing(images=False, links=False, unused=False, prefix="/", verbose=False): if not (images or links or unused): images = links = unused = True pages = flaskapp.get_wikipages(manager.app) all_images = set() used_images = set() for path in get_prefixed_paths(pages, prefix): if paths.extension(path) in (".png", ".jpg", ".jpeg", ".gif"): all_images.add(path) if not pages.is_wiki_source(path): continue if verbose: print(path) printed = False json = pages.json(path) for link in functions.find_links(json): value = link["value"] scheme = link.get("scheme") if value.startswith("#") or scheme in ("Icon", "Smallicon", "Largeicon"): continue fullpath = link.get("fullpath") if not fullpath: continue if pages.is_wiki(fullpath): fullpath = pages.source_path(fullpath) exists = pages.exists(fullpath) isimage = scheme in ("Image", "Anim") if isimage: used_images.add(fullpath) if exists: continue if (images and isimage) or (links and not isimage): if not verbose and not printed: print(path) printed = True print(" ", value, " ", fullpath) if unused: unused_images = all_images - used_images bytes = 0 for imgpath in sorted(unused_images): size = pages.size(imgpath) bytes += size print("Unused image:", imgpath, size) print(len(unused_images), "unused images (", bytes, ") out of", len(all_images))
def _make_doc(self, pages, path, root, block, text): attrs = block.get("attrs", {}) blocktype = block.get("type") body = block.get("body") is_root = blocktype == "root" # If a title was not passed in: if this is the root, look for a title # block, otherwise use the block text title = self._get_title(block) or paths.basename(path) container = False path = paths.basepath(path) if is_root: # Store a boolean if this page has subtopics subtopics = functions.subblock_by_id(block, "subtopics") container = subtopics and bool(subtopics.get("body")) else: blockid = functions.block_id(block) path = "%s#%s" % (path, blockid) # Look for a summary block summary = self._get_block_text(body, "summary") # Look for tags in the page attributes tags = attrs.get("tags", "").strip().replace(",", "") # Find outgoing links outgoing = [] for link in functions.find_links(block): val = link.get("value") if val: outgoing.append(pages.full_path(path, val)) outgoing = " ".join(outgoing) doctype = attrs.get("type") d = { "path": path, "status": attrs.get("status"), "category": "_", "content": functions.string(text), "title": title, "sortkey": attrs.get("sortkey") or title.lower(), "summary": summary, "grams": title, "type": doctype, "tags": tags, "icon": attrs.get("icon"), "links": outgoing, "container": container, "parent": self._get_path_attr(block, path, attrs, "parent"), "bestbet": attrs.get("bestbet"), } return d