def valid_sectionname(db, title):
"""
Checks if the ``sectionname`` property of given title is valid, i.e. if a
corresponding section exists on a page with given title.
.. note::
Validation is limited to pages in the Main namespace for easier access
to the cache; anchors on other pages are considered to be always valid.
:param ws.db.database.Database db: database object
:param title: parsed title of the wikilink to be checked
:type title: ws.parser_helpers.title.Title
:returns: ``True`` if the anchor corresponds to an existing section
"""
# we can't check interwiki links
if title.iwprefix:
return True
# empty sectionname is always valid
if title.sectionname == "":
return True
# get list of valid anchors
result = db.query(titles=title.fullpagename, prop="sections", secprop={"anchor"})
page = next(result)
anchors = [section["anchor"] for section in page.get("sections", [])]
# encode the given anchor and validate
return dotencode(title.sectionname) in anchors
def valid_sectionname(db, title):
"""
Checks if the ``sectionname`` property of given title is valid, i.e. if a
corresponding section exists on a page with given title.
.. note::
Validation is limited to pages in the Main namespace for easier access
to the cache; anchors on other pages are considered to be always valid.
:param ws.db.database.Database db: database object
:param title: parsed title of the wikilink to be checked
:type title: ws.parser_helpers.title.Title
:returns: ``True`` if the anchor corresponds to an existing section
"""
# we can't check interwiki links
if title.iwprefix:
return True
# empty sectionname is always valid
if title.sectionname == "":
return True
# get list of valid anchors
result = db.query(titles=title.fullpagename,
prop="sections",
secprop={"anchor"})
page = next(result)
anchors = [section["anchor"] for section in page.get("sections", [])]
# encode the given anchor and validate
return dotencode(title.sectionname) in anchors
def valid_sectionname(title, pages, wrapped_titles):
"""
Checks if the ``sectionname`` property of given title is valid, i.e. if a
corresponding section exists on a page with given title.
.. note::
Validation is limited to pages in the Main namespace for easier access
to the cache; anchors on other pages are considered to be always valid.
:param title: parsed title of the wikilink to be checked
:type title: ws.parser_helpers.title.Title
:returns: ``True`` if the anchor corresponds to an existing section
"""
# we can't check interwiki links
if title.iwprefix:
return True
# TODO: limitation of the cache, we can easily check only the main namespace
if title.namespace != "":
return True
# empty sectionname is always valid
if title.sectionname == "":
return True
page = ws.utils.bisect_find(pages, title.fullpagename, index_list=wrapped_titles)
text = page["revisions"][0]["*"]
# get list of valid anchors
anchors = get_anchors(get_section_headings(text))
# encode the given anchor and validate
return dotencode(title.sectionname) in anchors
def check_anchor(self, src_title, wikilink, title):
"""
:returns:
``True`` if the anchor is correct or has been corrected, ``False``
if it is definitely broken, ``None`` if it can't be checked at all
or the check was indecisive and a warning/error has been printed to
the log.
"""
# TODO: beware of https://phabricator.wikimedia.org/T20431
# we can't check interwiki links
if title.iwprefix:
return None
# empty sectionname is always valid
if title.sectionname == "":
return None
# determine target page
_target_title = title.make_absolute(src_title)
# skip links to special pages (e.g. [[Special:Preferences#mw-prefsection-rc]])
if _target_title.namespacenumber < 0:
return None
# resolve redirects
anchor_on_redirect_to_section = False
if _target_title.fullpagename in self.api.redirects.map:
_target_title = self.api.Title(
self.api.redirects.resolve(_target_title.fullpagename))
if _target_title.sectionname:
logger.warning(
"warning: section fragment placed on a redirect to possibly different section: {}"
.format(wikilink))
anchor_on_redirect_to_section = True
# get lists of section headings and anchors
_result = self.db.query(titles=_target_title.fullpagename,
prop="sections",
secprop={"title", "anchor"})
_result = list(_result)
assert len(_result) == 1
if "missing" in _result[0]:
logger.error(
"could not find content of page: '{}' (wikilink {})".format(
_target_title.fullpagename, wikilink))
return None
headings = [
section["title"] for section in _result[0].get("sections", [])
]
anchors = [
section["anchor"] for section in _result[0].get("sections", [])
]
if len(headings) == 0:
logger.warning(
"wikilink with broken section fragment: {}".format(wikilink))
return False
anchor = dotencode(title.sectionname)
needs_fix = True
# handle double-anchor redirects first
if anchor_on_redirect_to_section is True:
if anchor in anchors:
return True
else:
return False
# try exact match first
if anchor in anchors:
needs_fix = False
# otherwise try case-insensitive match to detect differences in capitalization
elif self.interactive is True:
# FIXME: first detect section renaming properly, fuzzy search should be only the last resort to deal with typos and such
ranks = get_ranks(anchor, anchors)
ranks = list(filter(lambda rank: rank[1] >= 0.8, ranks))
if len(ranks) == 1 or (len(ranks) >= 2
and ranks[0][1] - ranks[1][1] > 0.2):
logger.debug(
"wikilink {}: replacing anchor '{}' with '{}' on similarity level {}"
.format(wikilink, anchor, ranks[0][0], ranks[0][1]))
anchor = ranks[0][0]
elif len(ranks) > 1:
logger.debug(
"skipping {}: multiple feasible anchors per similarity ratio: {}"
.format(wikilink, ranks))
return False
else:
logger.warning(
"wikilink with broken section fragment: {}".format(
wikilink))
return False
else:
# FIXME: anchors with encoded characters like '[' or ']' are not handled properly in non-interactive mode - links get flagged as broken, although they work
# (e.g. [[Systemd-networkd#%5BNetDev%5D section|systemd-networkd]] - linked from [[systemd-timesyncd]])
logger.warning(
"wikilink with broken section fragment: {}".format(wikilink))
return False
# assemble new section fragment
# try to preserve the character separating base anchor and numeric suffix
dupl_match = re.match(r"(.+)([_ ])(\d+)$", str(wikilink.title))
if dupl_match:
suffix_sep = dupl_match.group(2)
else:
suffix_sep = " "
# get_anchors makes sure to strip markup and handle duplicate section names
new_fragment = get_anchors(
headings, pretty=True,
suffix_sep=suffix_sep)[anchors.index(anchor)]
# Avoid beautification if there is alternative text and the link
# actually works.
if wikilink.text is None or needs_fix is True:
# preserve title set in check_displaytitle()
# TODO: simplify (see #25)
t, _ = wikilink.title.split("#", maxsplit=1)
wikilink.title = t + "#" + new_fragment
title.parse(wikilink.title)
return True
def check_anchor(self, src_title, wikilink, title):
"""
:returns:
``True`` if the anchor is correct or has been corrected, ``False``
if it is definitely broken, ``None`` if it can't be checked at all
or the check was indecisive and a warning/error has been printed to
the log.
"""
# TODO: beware of https://phabricator.wikimedia.org/T20431
# we can't check interwiki links
if title.iwprefix:
return None
# empty sectionname is always valid
if title.sectionname == "":
return None
# determine target page
_target_title = title.make_absolute(src_title)
# skip links to special pages (e.g. [[Special:Preferences#mw-prefsection-rc]])
if _target_title.namespacenumber < 0:
return None
# resolve redirects
anchor_on_redirect_to_section = False
if _target_title.fullpagename in self.api.redirects.map:
_target_title = self.api.Title(self.api.redirects.resolve(_target_title.fullpagename))
if _target_title.sectionname:
logger.warning("warning: section fragment placed on a redirect to possibly different section: {}".format(wikilink))
anchor_on_redirect_to_section = True
# get lists of section headings and anchors
_result = self.db.query(titles=_target_title.fullpagename, prop="sections", secprop={"title", "anchor"})
_result = list(_result)
assert len(_result) == 1
if "missing" in _result[0]:
logger.error("could not find content of page: '{}' (wikilink {})".format(_target_title.fullpagename, wikilink))
return None
headings = [section["title"] for section in _result[0].get("sections", [])]
anchors = [section["anchor"] for section in _result[0].get("sections", [])]
if len(headings) == 0:
logger.warning("wikilink with broken section fragment: {}".format(wikilink))
return False
anchor = dotencode(title.sectionname)
needs_fix = True
# handle double-anchor redirects first
if anchor_on_redirect_to_section is True:
if anchor in anchors:
return True
else:
return False
# try exact match first
if anchor in anchors:
needs_fix = False
# otherwise try case-insensitive match to detect differences in capitalization
elif self.interactive is True:
# FIXME: first detect section renaming properly, fuzzy search should be only the last resort to deal with typos and such
ranks = get_ranks(anchor, anchors)
ranks = list(filter(lambda rank: rank[1] >= 0.8, ranks))
if len(ranks) == 1 or ( len(ranks) >= 2 and ranks[0][1] - ranks[1][1] > 0.2 ):
logger.debug("wikilink {}: replacing anchor '{}' with '{}' on similarity level {}".format(wikilink, anchor, ranks[0][0], ranks[0][1]))
anchor = ranks[0][0]
elif len(ranks) > 1:
logger.debug("skipping {}: multiple feasible anchors per similarity ratio: {}".format(wikilink, ranks))
return False
else:
logger.warning("wikilink with broken section fragment: {}".format(wikilink))
return False
else:
logger.warning("wikilink with broken section fragment: {}".format(wikilink))
return False
# assemble new section fragment
# try to preserve the character separating base anchor and numeric suffix
dupl_match = re.match("(.+)([_ ])(\d+)$", str(wikilink.title))
if dupl_match:
suffix_sep = dupl_match.group(2)
else:
suffix_sep = " "
# get_anchors makes sure to strip markup and handle duplicate section names
new_fragment = get_anchors(headings, pretty=True, suffix_sep=suffix_sep)[anchors.index(anchor)]
# Avoid beautification if there is alternative text and the link
# actually works.
if wikilink.text is None or needs_fix is True:
# preserve title set in check_displaytitle()
# TODO: simplify (see #25)
t, _ = wikilink.title.split("#", maxsplit=1)
wikilink.title = t + "#" + new_fragment
title.parse(wikilink.title)
return True
def check_anchor(self, wikilink, title, srcpage):
# TODO: beware of https://phabricator.wikimedia.org/T20431
# - mark with {{Broken fragment}} instead of reporting?
# - someday maybe: check N older revisions, section might have been renamed (must be interactive!) or moved to other page (just report)
# we can't check interwiki links
if title.iwprefix:
return True
# empty sectionname is always valid
if title.sectionname == "":
return True
# lookup target page content
# TODO: pulling revisions from cache does not expand templates
# (transclusions like on List of applications)
if title.fullpagename:
_target_ns = title.namespacenumber
_target_title = title.fullpagename
else:
src_title = Title(self.api, srcpage)
_target_ns = src_title.namespacenumber
_target_title = src_title.fullpagename
# skip links to special pages (e.g. [[Special:Preferences#mw-prefsection-rc]])
if _target_ns < 0:
return
if _target_title in self.redirects:
_new = self.redirects.get(_target_title)
if "#" not in _new:
_target_title = _new
else:
logger.warning("skipping {} (section fragment placed on a redirect to possibly different section)".format(wikilink))
return
pages = self.db_copy[str(_target_ns)]
wrapped_titles = ws.utils.ListOfDictsAttrWrapper(pages, "title")
try:
page = ws.utils.bisect_find(pages, _target_title, index_list=wrapped_titles)
except IndexError:
logger.error("could not find content of page: '{}' (wikilink {})".format(_target_title, wikilink))
return
text = page["revisions"][0]["*"]
# get lists of section headings and anchors
headings = get_section_headings(text)
if len(headings) == 0:
logger.warning("wikilink with broken section fragment: {}".format(wikilink))
return
anchors = get_anchors(headings)
anchor = dotencode(title.sectionname)
needs_fix = True
# try exact match first
if anchor in anchors:
needs_fix = False
# otherwise try case-insensitive match to detect differences in capitalization
elif self.interactive is True:
# FIXME: first detect section renaming properly, fuzzy search should be only the last resort to deal with typos and such
ranks = get_ranks(anchor, anchors)
ranks = list(filter(lambda rank: rank[1] >= 0.8, ranks))
if len(ranks) == 1 or ( len(ranks) >= 2 and ranks[0][1] - ranks[1][1] > 0.2 ):
logger.debug("wikilink {}: replacing anchor '{}' with '{}' on similarity level {}".format(wikilink, anchor, ranks[0][0], ranks[0][1]))
anchor = ranks[0][0]
elif len(ranks) > 1:
logger.debug("skipping {}: multiple feasible anchors per similarity ratio: {}".format(wikilink, ranks))
return
else:
logger.warning("wikilink with broken section fragment: {}".format(wikilink))
return
else:
logger.warning("wikilink with broken section fragment: {}".format(wikilink))
return
# assemble new section fragment
new_fragment = strip_markup(headings[anchors.index(anchor)])
# anchors can't contain '[', '|' and ']', encode them manually
new_fragment = new_fragment.replace("[", ".5B").replace("|", ".7C").replace("]", ".5D")
# point to the right duplicated sectionname
# NOTE: the dupl. section number might get changed in fuzzy match
dupl_match = re.match("(.+)_(\d+)$", anchor)
if dupl_match:
base = dupl_match.group(1)
suffix = dupl_match.group(2)
if base in anchors:
if wikilink.title.endswith("_" + suffix):
new_fragment += "_" + suffix
else:
new_fragment += " " + suffix
# Avoid beautification if there is alternative text and the link
# actually works.
if wikilink.text is None or needs_fix is True:
# preserve title set in check_displaytitle()
# TODO: simplify (see #25)
t, _ = wikilink.title.split("#", maxsplit=1)
wikilink.title = t + "#" + new_fragment
title.parse(wikilink.title)