def _process_doc(self, pages, path, root, block, doc):
# Add Houdini-specific fields to each document
path = paths.basepath(path)
attrs = block.get("attrs", {})
doctype = attrs.get("type", "").strip()
context = attrs.get("context", "").strip().replace(",", "") or None
if doctype == "node":
if context in ("pop", "part") or context.endswith("_state"):
return
internal = attrs.get("internal")
if internal:
doc["grams"] += " %s" % internal
if doc.get("category") == "_":
if path.startswith("/shelf/"):
doc["category"] = "tool"
elif path.startswith("/ref/util/"):
doc["category"] = "utility"
elif path.startswith("/gallery/shop/"):
doc["category"] = "gallery/shop"
elif doctype == "node":
doc["category"] = "%s/%s" % (doctype, context)
doc["grams"] += " %s" % context
elif doctype in ("hscript", "expression", "example", "homclass",
"hommodule", "vex"):
doc["category"] = doctype
replaces = attrs.get("replaces")
rsection = functions.subblock_by_id(block, "replaces")
if rsection:
rlist = " ".join(link.get("fullpath", "") for link
in functions.find_links(rsection))
if replaces:
replaces = replaces + " " + rlist
else:
replaces = rlist
doc.update({
"context": context,
"bestbet": attrs.get("bestbet"),
"helpid": attrs.get("helpid"),
"superclass": attrs.get("superclass"),
"version": attrs.get("version"),
"replaces": replaces or None,
"examplefor": root.get("examplefor"),
"examplefile": root.get("examplefile"),
"group": attrs.get("group"),
})
# Add example file info
if root is block and "examplefile" in root:
otlpath = root["examplefile"]
# Usages file should be in the same location with .usages ext
usagespath = paths.basepath(otlpath) + ".usages"
if pages.exists(usagespath):
usagescontent = pages.content(usagespath)
usages = ws_exp.split(usagescontent)
doc["uses"] = " ".join(usages)
def missing(images=False, links=False, unused=False, prefix="/", verbose=False):
if not (images or links or unused):
images = links = unused = True
pages = flaskapp.get_wikipages(manager.app)
all_images = set()
used_images = set()
for path in get_prefixed_paths(pages, prefix):
if paths.extension(path) in (".png", ".jpg", ".jpeg", ".gif"):
all_images.add(path)
if not pages.is_wiki_source(path):
continue
if verbose:
print(path)
printed = False
json = pages.json(path)
for link in functions.find_links(json):
value = link["value"]
scheme = link.get("scheme")
if value.startswith("#") or scheme in ("Icon", "Smallicon", "Largeicon"):
continue
fullpath = link.get("fullpath")
if not fullpath:
continue
if pages.is_wiki(fullpath):
fullpath = pages.source_path(fullpath)
exists = pages.exists(fullpath)
isimage = scheme in ("Image", "Anim")
if isimage:
used_images.add(fullpath)
if exists:
continue
if (images and isimage) or (links and not isimage):
if not verbose and not printed:
print(path)
printed = True
print(" ", value, " ", fullpath)
if unused:
unused_images = all_images - used_images
bytes = 0
for imgpath in sorted(unused_images):
size = pages.size(imgpath)
bytes += size
print("Unused image:", imgpath, size)
print(len(unused_images), "unused images (", bytes, ") out of", len(all_images))
def _make_doc(self, pages, path, root, block, text):
attrs = block.get("attrs", {})
blocktype = block.get("type")
body = block.get("body")
is_root = blocktype == "root"
# If a title was not passed in: if this is the root, look for a title
# block, otherwise use the block text
title = self._get_title(block) or paths.basename(path)
container = False
path = paths.basepath(path)
if is_root:
# Store a boolean if this page has subtopics
subtopics = functions.subblock_by_id(block, "subtopics")
container = subtopics and bool(subtopics.get("body"))
else:
blockid = functions.block_id(block)
path = "%s#%s" % (path, blockid)
# Look for a summary block
summary = self._get_block_text(body, "summary")
# Look for tags in the page attributes
tags = attrs.get("tags", "").strip().replace(",", "")
# Find outgoing links
outgoing = []
for link in functions.find_links(block):
val = link.get("value")
if val:
outgoing.append(pages.full_path(path, val))
outgoing = " ".join(outgoing)
doctype = attrs.get("type")
d = {
"path": path,
"status": attrs.get("status"),
"category": "_",
"content": functions.string(text),
"title": title,
"sortkey": attrs.get("sortkey") or title.lower(),
"summary": summary,
"grams": title,
"type": doctype,
"tags": tags,
"icon": attrs.get("icon"),
"links": outgoing,
"container": container,
"parent": self._get_path_attr(block, path, attrs, "parent"),
"bestbet": attrs.get("bestbet"),
}
return d
