def standardizePageFooter(self, text):
"""
Standardize page footer.
Makes sure that interwiki links and categories are put to the correct
position and into the right order. This combines the old instances
standardizeInterwiki and standardizeCategories.
The page footer has the following section in that sequence:
1. categories
2. ## TODO: template beyond categories ##
3. additional information depending on local site policy
4. interwiki links
"""
categories = None
interwikiLinks = None
# Pywikibot is no longer allowed to touch categories on the
# German Wikipedia. See
# https://de.wikipedia.org/wiki/Hilfe_Diskussion:Personendaten/Archiv/1#Position_der_Personendaten_am_.22Artikelende.22
# ignoring nn-wiki of cause of the comment line above iw section
if not self.template and '{{Personendaten' not in text and \
'{{SORTIERUNG' not in text and '{{DEFAULTSORT' not in text and \
self.site.code not in ('et', 'it', 'bg', 'ru'):
categories = textlib.getCategoryLinks(text, site=self.site)
if not self.talkpage:
subpage = False
if self.template:
try:
tmpl, loc = moved_links[self.site.code]
del tmpl
except KeyError:
loc = None
if loc is not None and loc in self.title:
subpage = True
interwikiLinks = textlib.getLanguageLinks(text,
insite=self.site,
template_subpage=subpage)
# Removing the interwiki
text = textlib.removeLanguageLinks(text, site=self.site)
# Adding categories
if categories:
# TODO: Sorting categories in alphabetic order.
# e.g. using categories.sort()
# TODO: Taking main cats to top
text = textlib.replaceCategoryLinks(text,
categories,
site=self.site)
# Adding the interwiki
if interwikiLinks:
text = textlib.replaceLanguageLinks(text,
interwikiLinks,
site=self.site,
template=self.template,
template_subpage=subpage)
return text
def add_template(self, source, dest, task, fromsite):
"""Place or remove the Link_GA/FA template on/from a page."""
def compile_link(site, templates):
"""compile one link template list."""
findtemplate = '(%s)' % '|'.join(templates)
return re.compile(
r"\{\{%s\|%s\}\}" %
(findtemplate.replace(u' ', u'[ _]'), site.code),
re.IGNORECASE)
tosite = dest.site
add_tl, remove_tl = self.getTemplateList(tosite.code, task)
re_Link_add = compile_link(fromsite, add_tl)
re_Link_remove = compile_link(fromsite, remove_tl)
text = dest.text
m1 = add_tl and re_Link_add.search(text)
m2 = remove_tl and re_Link_remove.search(text)
changed = False
interactive = self.getOption('interactive')
if add_tl:
if m1:
pywikibot.output(u"(already added)")
else:
# insert just before interwiki
if (not interactive or pywikibot.input(
u'Connecting %s -> %s. Proceed? [Y/N]' %
(source.title(), dest.title())) in ['Y', 'y']):
if self.getOption('side'):
# Placing {{Link FA|xx}} right next to
# corresponding interwiki
text = (text[:m1.end()] + u" {{%s|%s}}" %
(add_tl[0], fromsite.code) + text[m1.end():])
else:
# Moving {{Link FA|xx}} to top of interwikis
iw = textlib.getLanguageLinks(text, tosite)
text = textlib.removeLanguageLinks(text, tosite)
text += u"%s{{%s|%s}}%s" % (LS, add_tl[0],
fromsite.code, LS)
text = textlib.replaceLanguageLinks(text, iw, tosite)
changed = True
if remove_tl:
if m2:
if (changed or # Don't force the user to say "Y" twice
not interactive or pywikibot.input(
u'Connecting %s -> %s. Proceed? [Y/N]' %
(source.title(), dest.title())) in ['Y', 'y']):
text = re.sub(re_Link_remove, '', text)
changed = True
elif task == 'former':
pywikibot.output(u"(already removed)")
if changed:
comment = i18n.twtranslate(tosite, 'featured-' + task,
{'page': unicode(source)})
try:
dest.put(text, comment)
except pywikibot.LockedPage:
pywikibot.output(u'Page %s is locked!' % dest.title())
except pywikibot.PageNotSaved:
pywikibot.output(u"Page not saved")
def main(*args):
"""Process command line arguments and run the script."""
start = '!'
# Load the default parameters and start
for arg in pywikibot.handle_args():
if arg.startswith('-start'):
if len(arg) == 6:
start = pywikibot.input(
u'From what page do you want to start?')
else:
start = arg[7:]
site = pywikibot.Site()
comm = pywikibot.translate(site, comment)
for pl in site.allpages(start):
plname = pl.title()
pywikibot.output(u'\nLoading %s...' % plname)
try:
oldtext = pl.get()
except pywikibot.IsRedirectPage:
pywikibot.output(u"%s is a redirect!" % plname)
continue
old = pl.interwiki()
new = {}
for pl2 in old:
new[pl2.site] = pywikibot.Page(pl2)
newtext = textlib.replaceLanguageLinks(oldtext, new, site=site)
if new:
if oldtext != newtext:
pywikibot.showDiff(oldtext, newtext)
# Submit changes
try:
pl.put(newtext, comment=comm)
except pywikibot.LockedPage:
pywikibot.output(u"%s is locked" % plname)
continue
else:
pywikibot.output(u'No changes needed.')
continue
else:
pywikibot.output(u'No interwiki found.')
continue
def main(*args):
"""Process command line arguments and run the script."""
start = '!'
# Load the default parameters and start
for arg in pywikibot.handle_args():
if arg.startswith('-start'):
if len(arg) == 6:
start = pywikibot.input('From what page do you want to start?')
else:
start = arg[7:]
site = pywikibot.Site()
comm = pywikibot.translate(site, comment)
for pl in site.allpages(start):
plname = pl.title()
pywikibot.output(u'\nLoading %s...' % plname)
try:
oldtext = pl.get()
except pywikibot.IsRedirectPage:
pywikibot.output(u"%s is a redirect!" % plname)
continue
old = pl.interwiki()
new = {}
for pl2 in old:
new[pl2.site] = pywikibot.Page(pl2)
newtext = textlib.replaceLanguageLinks(oldtext, new, site=site)
if new:
if oldtext != newtext:
pywikibot.showDiff(oldtext, newtext)
# Submit changes
try:
pl.put(newtext, comment=comm)
except pywikibot.LockedPage:
pywikibot.output(u"%s is locked" % plname)
continue
else:
pywikibot.output(u'No changes needed.')
continue
else:
pywikibot.output(u'No interwiki found.')
continue
def apply(self, page, summaries=[], callbacks=[]):
iw_links = textlib.getLanguageLinks(page.text, page.site)
if not iw_links:
return
try:
item = page.data_item()
item.get()
except pywikibot.NoPage:
return
sitelinks = item.iterlinks(family=page.site.family)
new_sites = set(iw_links.keys()) - {page.site for page in sitelinks}
if len(new_sites) == len(iw_links):
return
new_links = {site: iw_links[site] for site in new_sites}
page.text = textlib.replaceLanguageLinks(page.text, new_links,
page.site)
summaries.append('odstranění interwiki')
def add_text(
page,
addText: str,
summary: Optional[str] = None,
regexSkip: Optional[str] = None,
regexSkipUrl: Optional[str] = None,
always: bool = False,
up: bool = False,
putText: bool = True,
oldTextGiven: Optional[str] = None,
reorderEnabled: bool = True,
create: bool = False
) -> Union[Tuple[bool, bool, bool], Tuple[str, str, bool]]:
"""
Add text to a page.
@param page: The page to add text to
@type page: pywikibot.page.BasePage
@param addText: Text to add
@param summary: Summary of changes. If None, beginning of addText is used.
@param regexSkip: Abort if text on page matches
@param regexSkipUrl: Abort if full url matches
@param always: Always add text without user confirmation
@param up: If True, add text to top of page, else add at bottom.
@param putText: If True, save changes to the page, else return
(_, newtext, _)
@param oldTextGiven: If None fetch page text, else use this text
@param reorderEnabled: If True place text above categories and
interwiki, else place at page bottom. No effect if up = False.
@param create: Create page if it does not exist
@return: If putText=True: (success, success, always)
else: (_, newtext, _)
"""
site = page.site
if not summary:
summary = i18n.twtranslate(site, 'add_text-adding',
{'adding': addText[:200]})
if putText:
pywikibot.output('Loading {}...'.format(page.title()))
text = get_text(page, oldTextGiven, create)
if text is None:
return (False, False, always)
# Understand if the bot has to skip the page or not
# In this way you can use both -except and -excepturl
if regexSkipUrl is not None:
url = page.full_url()
result = re.findall(regexSkipUrl, site.getUrl(url))
if result != []:
pywikibot.output('Exception! regex (or word) used with -exceptUrl '
'is in the page. Skip!\n'
'Match was: {}'.format(result))
return (False, False, always)
if regexSkip is not None:
result = re.findall(regexSkip, text)
if result != []:
pywikibot.output('Exception! regex (or word) used with -except '
'is in the page. Skip!\n'
'Match was: {}'.format(result))
return (False, False, always)
# If not up, text put below
if not up:
newtext = text
# Translating the \\n into binary \n
addText = addText.replace('\\n', '\n')
if reorderEnabled:
# Getting the categories
categoriesInside = textlib.getCategoryLinks(newtext, site)
# Deleting the categories
newtext = textlib.removeCategoryLinks(newtext, site)
# Getting the interwiki
interwikiInside = textlib.getLanguageLinks(newtext, site)
# Removing the interwiki
newtext = textlib.removeLanguageLinks(newtext, site)
# Adding the text
newtext += '\n' + addText
# Reputting the categories
newtext = textlib.replaceCategoryLinks(newtext, categoriesInside,
site, True)
# Adding the interwiki
newtext = textlib.replaceLanguageLinks(newtext, interwikiInside,
site)
else:
newtext += '\n' + addText
else:
newtext = addText + '\n' + text
if not putText:
# If someone load it as module, maybe it's not so useful to put the
# text in the page
return (text, newtext, always)
if text != newtext:
pywikibot.output(
color_format('\n\n>>> {lightpurple}{0}{default} <<<',
page.title()))
pywikibot.showDiff(text, newtext)
# Let's put the changes.
error_count = 0
while True:
if not always:
try:
choice = pywikibot.input_choice(
'Do you want to accept these changes?',
[('Yes', 'y'), ('No', 'n'), ('All', 'a'),
('open in Browser', 'b')], 'n')
except QuitKeyboardInterrupt:
sys.exit('User quit bot run.')
if choice == 'a':
always = True
elif choice == 'n':
return (False, False, always)
elif choice == 'b':
pywikibot.bot.open_webbrowser(page)
continue
# either always or choice == 'y' is selected
result = put_text(page,
newtext,
summary,
error_count,
asynchronous=not always)
if result is not None:
return (result, result, always)
error_count += 1
def add_text(page,
addText,
summary=None,
regexSkip=None,
regexSkipUrl=None,
always=False,
up=False,
putText=True,
oldTextGiven=None,
reorderEnabled=True,
create=False):
"""
Add text to a page.
@rtype: tuple of (text, newtext, always)
"""
site = page.site
if not summary:
summary = i18n.twtranslate(site, 'add_text-adding',
{'adding': addText[:200]})
# When a page is tagged as "really well written" it has a star in the
# interwiki links. This is a list of all the templates used (in regex
# format) to make the stars appear.
errorCount = 0
if putText:
pywikibot.output(u'Loading %s...' % page.title())
if oldTextGiven is None:
try:
text = page.get()
except pywikibot.NoPage:
if create:
pywikibot.output(u"%s doesn't exist, creating it!" %
page.title())
text = u''
else:
pywikibot.output(u"%s doesn't exist, skip!" % page.title())
return (False, False, always)
except pywikibot.IsRedirectPage:
pywikibot.output(u"%s is a redirect, skip!" % page.title())
return (False, False, always)
else:
text = oldTextGiven
# Understand if the bot has to skip the page or not
# In this way you can use both -except and -excepturl
if regexSkipUrl is not None:
url = page.full_url()
result = re.findall(regexSkipUrl, site.getUrl(url))
if result != []:
pywikibot.output('Exception! regex (or word) used with -exceptUrl '
'is in the page. Skip!\n'
'Match was: %s' % result)
return (False, False, always)
if regexSkip is not None:
result = re.findall(regexSkip, text)
if result != []:
pywikibot.output('Exception! regex (or word) used with -except '
'is in the page. Skip!\n'
'Match was: %s' % result)
return (False, False, always)
# If not up, text put below
if not up:
newtext = text
# Translating the \\n into binary \n
addText = addText.replace('\\n', config.line_separator)
if (reorderEnabled):
# Getting the categories
categoriesInside = textlib.getCategoryLinks(newtext, site)
# Deleting the categories
newtext = textlib.removeCategoryLinks(newtext, site)
# Getting the interwiki
interwikiInside = textlib.getLanguageLinks(newtext, site)
# Removing the interwiki
newtext = textlib.removeLanguageLinks(newtext, site)
# Adding the text
newtext += u"%s%s" % (config.line_separator, addText)
# Reputting the categories
newtext = textlib.replaceCategoryLinks(newtext, categoriesInside,
site, True)
# Dealing the stars' issue
allstars = []
starstext = textlib.removeDisabledParts(text)
for star in starsList:
regex = re.compile(
'(\{\{(?:template:|)%s\|.*?\}\}[\s]*)' % star, re.I)
found = regex.findall(starstext)
if found != []:
newtext = regex.sub('', newtext)
allstars += found
if allstars != []:
newtext = newtext.strip() + config.line_separator * 2
allstars.sort()
for element in allstars:
newtext += '%s%s' % (element.strip(), config.LS)
# Adding the interwiki
newtext = textlib.replaceLanguageLinks(newtext, interwikiInside,
site)
else:
newtext += u"%s%s" % (config.line_separator, addText)
else:
newtext = addText + config.line_separator + text
if putText and text != newtext:
pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<" %
page.title())
pywikibot.showDiff(text, newtext)
# Let's put the changes.
while True:
# If someone load it as module, maybe it's not so useful to put the
# text in the page
if putText:
if not always:
choice = pywikibot.input_choice(
u'Do you want to accept these changes?',
[('Yes', 'y'), ('No', 'n'), ('All', 'a'),
('open in Browser', 'b')],
'n',
automatic_quit=False)
if choice == 'a':
always = True
elif choice == 'n':
return (False, False, always)
elif choice == 'b':
pywikibot.bot.open_webbrowser(page)
if always or choice == 'y':
try:
if always:
page.put(newtext,
summary,
minorEdit=page.namespace() != 3)
else:
page.put_async(newtext,
summary,
minorEdit=page.namespace() != 3)
except pywikibot.EditConflict:
pywikibot.output(u'Edit conflict! skip!')
return (False, False, always)
except pywikibot.ServerError:
errorCount += 1
if errorCount < config.max_retries:
pywikibot.output(u'Server Error! Wait..')
time.sleep(config.retry_wait)
continue
else:
raise pywikibot.ServerError(u'Fifth Server Error!')
except pywikibot.SpamfilterError as e:
pywikibot.output(
u'Cannot change %s because of blacklist entry %s' %
(page.title(), e.url))
return (False, False, always)
except pywikibot.LockedPage:
pywikibot.output(u'Skipping %s (locked page)' %
page.title())
return (False, False, always)
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Error putting page: %s' % error.args)
return (False, False, always)
else:
# Break only if the errors are one after the other...
errorCount = 0
return (True, True, always)
else:
return (text, newtext, always)
def standardizePageFooter(self, text):
"""
Standardize page footer.
Makes sure that interwiki links and categories are put
into the correct position and into the right order. This
combines the old instances of standardizeInterwiki
and standardizeCategories.
The page footer consists of the following parts
in that sequence:
1. categories
2. additional information depending on the local site policy
3. interwiki
"""
categories = []
interwiki_links = []
# get categories
if not self.template:
categories = textlib.getCategoryLinks(text, site=self.site)
if not self.talkpage:
subpage = False
if self.template:
try:
tmpl, loc = moved_links[self.site.code]
del tmpl
except KeyError:
loc = None
if loc is not None and loc in self.title:
subpage = True
# get interwiki
interwiki_links = textlib.getLanguageLinks(
text, insite=self.site, template_subpage=subpage)
# remove interwiki
text = textlib.removeLanguageLinks(text, site=self.site)
# add categories, main to top
if categories:
# TODO: Sort categories in alphabetic order, e.g. using
# categories.sort()? (T100265)
# TODO: Get main categories from Wikidata?
main = pywikibot.Category(self.site,
'Category:' + self.title,
sort_key=' ')
if main in categories:
categories.pop(categories.index(main))
categories.insert(0, main)
text = textlib.replaceCategoryLinks(text,
categories,
site=self.site)
# add interwiki
if interwiki_links:
text = textlib.replaceLanguageLinks(text,
interwiki_links,
site=self.site,
template=self.template,
template_subpage=subpage)
return text
# What follows is the main part of the code.
try:
for pl in site.allpages(start):
plname = pl.title()
pywikibot.output(u'\nLoading %s...' % plname)
try:
oldtext = pl.get()
except pywikibot.IsRedirectPage:
pywikibot.output(u"%s is a redirect!" % plname)
continue
old = pl.interwiki()
new = {}
for pl2 in old:
new[pl2.site] = pywikibot.Page(pl2)
newtext = textlib.replaceLanguageLinks(oldtext, new, site=site)
if new:
if oldtext != newtext:
pywikibot.showDiff(oldtext, newtext)
# Submit changes
try:
pl.put(newtext, comment=comm)
except pywikibot.LockedPage:
pywikibot.output(u"%s is locked" % plname)
continue
else:
pywikibot.output(u'No changes needed.')
continue
else:
pywikibot.output(u'No interwiki found.')
continue
# What follows is the main part of the code.
try:
for pl in site.allpages(start):
plname = pl.title()
pywikibot.output(u'\nLoading {0!s}...'.format(plname))
try:
oldtext = pl.get()
except pywikibot.IsRedirectPage:
pywikibot.output(u"{0!s} is a redirect!".format(plname))
continue
old = pl.interwiki()
new = {}
for pl2 in old:
new[pl2.site] = pywikibot.Page(pl2)
newtext = textlib.replaceLanguageLinks(oldtext, new, site=site)
if new:
if oldtext != newtext:
pywikibot.showDiff(oldtext, newtext)
# Submit changes
try:
pl.put(newtext, comment=comm)
except pywikibot.LockedPage:
pywikibot.output(u"{0!s} is locked".format(plname))
continue
else:
pywikibot.output(u'No changes needed.')
continue
else:
pywikibot.output(u'No interwiki found.')
continue
def standardizePageFooter(self, text):
"""
Standardize page footer.
Makes sure that interwiki links, categories and star templates are
put to the correct position and into the right order. This combines the
old instances standardizeInterwiki and standardizeCategories
The page footer has the following section in that sequence:
1. categories
2. ## TODO: template beyond categories ##
3. additional information depending on local site policy
4. stars templates for featured and good articles
5. interwiki links
"""
starsList = [
u'bueno',
u'bom interwiki',
u'cyswllt[ _]erthygl[ _]ddethol', u'dolen[ _]ed',
u'destacado', u'destaca[tu]',
u'enllaç[ _]ad',
u'enllaz[ _]ad',
u'leam[ _]vdc',
u'legătură[ _]a[bcf]',
u'liamm[ _]pub',
u'lien[ _]adq',
u'lien[ _]ba',
u'liên[ _]kết[ _]bài[ _]chất[ _]lượng[ _]tốt',
u'liên[ _]kết[ _]chọn[ _]lọc',
u'ligam[ _]adq',
u'ligazón[ _]a[bd]',
u'ligoelstara',
u'ligoleginda',
u'link[ _][afgu]a', u'link[ _]adq', u'link[ _]f[lm]', u'link[ _]km',
u'link[ _]sm', u'linkfa',
u'na[ _]lotura',
u'nasc[ _]ar',
u'tengill[ _][úg]g',
u'ua',
u'yüm yg',
u'רא',
u'وصلة مقالة جيدة',
u'وصلة مقالة مختارة',
]
categories = None
interwikiLinks = None
allstars = []
# The PyWikipediaBot is no longer allowed to touch categories on the
# German Wikipedia. See
# https://de.wikipedia.org/wiki/Hilfe_Diskussion:Personendaten/Archiv/1#Position_der_Personendaten_am_.22Artikelende.22
# ignoring nn-wiki of cause of the comment line above iw section
if not self.template and '{{Personendaten' not in text and \
'{{SORTIERUNG' not in text and '{{DEFAULTSORT' not in text and \
self.site.code not in ('et', 'it', 'bg', 'ru'):
categories = textlib.getCategoryLinks(text, site=self.site)
if not self.talkpage: # and pywikibot.calledModuleName() <> 'interwiki':
subpage = False
if self.template:
loc = None
try:
tmpl, loc = moved_links[self.site.code]
del tmpl
except KeyError:
pass
if loc is not None and loc in self.title:
subpage = True
interwikiLinks = textlib.getLanguageLinks(
text, insite=self.site, template_subpage=subpage)
# Removing the interwiki
text = textlib.removeLanguageLinks(text, site=self.site)
# Removing the stars' issue
starstext = textlib.removeDisabledParts(text)
for star in starsList:
regex = re.compile(r'(\{\{(?:template:|)%s\|.*?\}\}[\s]*)'
% star, re.I)
found = regex.findall(starstext)
if found != []:
text = regex.sub('', text)
allstars += found
# Adding categories
if categories:
# TODO: Sorting categories in alphabetic order.
# e.g. using categories.sort()
# TODO: Taking main cats to top
# for name in categories:
# if re.search(u"(.+?)\|(.{,1}?)",name.title()) or name.title()==name.title().split(":")[0]+title:
# categories.remove(name)
# categories.insert(0, name)
text = textlib.replaceCategoryLinks(text, categories,
site=self.site)
# Adding stars templates
if allstars:
text = text.strip() + self.site.family.interwiki_text_separator
allstars.sort()
for element in allstars:
text += '%s%s' % (element.strip(), config.line_separator)
pywikibot.log(u'%s' % element.strip())
# Adding the interwiki
if interwikiLinks:
text = textlib.replaceLanguageLinks(text, interwikiLinks,
site=self.site,
template=self.template,
template_subpage=subpage)
return text
def add_template(self, source, dest, task, fromsite):
"""Place or remove the Link_GA/FA template on/from a page."""
def compile_link(site, templates):
"""Compile one link template list."""
findtemplate = '(%s)' % '|'.join(templates)
return re.compile(r"\{\{%s\|%s\}\}"
% (findtemplate.replace(u' ', u'[ _]'),
site.code), re.IGNORECASE)
tosite = dest.site
add_tl, remove_tl = self.getTemplateList(tosite.code, task)
re_Link_add = compile_link(fromsite, add_tl)
re_Link_remove = compile_link(fromsite, remove_tl)
text = dest.text
m1 = add_tl and re_Link_add.search(text)
m2 = remove_tl and re_Link_remove.search(text)
changed = False
interactive = self.getOption('interactive')
if add_tl:
if m1:
pywikibot.output(u"(already added)")
else:
# insert just before interwiki
if (not interactive or
pywikibot.input_yn(
u'Connecting %s -> %s. Proceed?'
% (source.title(), dest.title()),
default=False, automatic_quit=False)):
if self.getOption('side'):
# Placing {{Link FA|xx}} right next to
# corresponding interwiki
text = (text[:m1.end()] +
u" {{%s|%s}}" % (add_tl[0], fromsite.code) +
text[m1.end():])
else:
# Moving {{Link FA|xx}} to top of interwikis
iw = textlib.getLanguageLinks(text, tosite)
text = textlib.removeLanguageLinks(text, tosite)
text += u"%s{{%s|%s}}%s" % (config.LS, add_tl[0],
fromsite.code, config.LS)
text = textlib.replaceLanguageLinks(text,
iw, tosite)
changed = True
if remove_tl:
if m2:
if (changed or # Don't force the user to say "Y" twice
not interactive or
pywikibot.input_yn(
u'Connecting %s -> %s. Proceed?'
% (source.title(), dest.title()),
default=False, automatic_quit=False)):
text = re.sub(re_Link_remove, '', text)
changed = True
elif task == 'former':
pywikibot.output(u"(already removed)")
if changed:
comment = i18n.twtranslate(tosite, 'featured-' + task,
{'page': source})
try:
dest.put(text, comment)
self._save_counter += 1
except pywikibot.LockedPage:
pywikibot.output(u'Page %s is locked!'
% dest.title())
except pywikibot.PageNotSaved:
pywikibot.output(u"Page not saved")
def standardizePageFooter(self, text):
"""
Standardize page footer.
Makes sure that interwiki links, categories and star templates are
put to the correct position and into the right order. This combines the
old instances standardizeInterwiki and standardizeCategories
The page footer has the following section in that sequence:
1. categories
2. ## TODO: template beyond categories ##
3. additional information depending on local site policy
4. stars templates for featured and good articles
5. interwiki links
"""
starsList = [
u'bueno',
u'bom interwiki',
u'cyswllt[ _]erthygl[ _]ddethol', u'dolen[ _]ed',
u'destacado', u'destaca[tu]',
u'enllaç[ _]ad',
u'enllaz[ _]ad',
u'leam[ _]vdc',
u'legătură[ _]a[bcf]',
u'liamm[ _]pub',
u'lien[ _]adq',
u'lien[ _]ba',
u'liên[ _]kết[ _]bài[ _]chất[ _]lượng[ _]tốt',
u'liên[ _]kết[ _]chọn[ _]lọc',
u'ligam[ _]adq',
u'ligazón[ _]a[bd]',
u'ligoelstara',
u'ligoleginda',
u'link[ _][afgu]a', u'link[ _]adq', u'link[ _]f[lm]', u'link[ _]km',
u'link[ _]sm', u'linkfa',
u'na[ _]lotura',
u'nasc[ _]ar',
u'tengill[ _][úg]g',
u'ua',
u'yüm yg',
u'רא',
u'وصلة مقالة جيدة',
u'وصلة مقالة مختارة',
]
categories = None
interwikiLinks = None
allstars = []
# The PyWikipediaBot is no longer allowed to touch categories on the
# German Wikipedia. See
# https://de.wikipedia.org/wiki/Hilfe_Diskussion:Personendaten/Archiv/1#Position_der_Personendaten_am_.22Artikelende.22
# ignoring nn-wiki of cause of the comment line above iw section
if not self.template and '{{Personendaten' not in text and \
'{{SORTIERUNG' not in text and '{{DEFAULTSORT' not in text and \
self.site.code not in ('et', 'it', 'bg', 'ru'):
categories = textlib.getCategoryLinks(text, site=self.site)
if not self.talkpage: # and pywikibot.calledModuleName() <> 'interwiki':
subpage = False
if self.template:
loc = None
try:
tmpl, loc = moved_links[self.site.code]
del tmpl
except KeyError:
pass
if loc is not None and loc in self.title:
subpage = True
interwikiLinks = textlib.getLanguageLinks(
text, insite=self.site, template_subpage=subpage)
# Removing the interwiki
text = textlib.removeLanguageLinks(text, site=self.site)
# Removing the stars' issue
starstext = textlib.removeDisabledParts(text)
for star in starsList:
regex = re.compile(r'(\{\{(?:template:|)%s\|.*?\}\}[\s]*)'
% star, re.I)
found = regex.findall(starstext)
if found != []:
text = regex.sub('', text)
allstars += found
# Adding categories
if categories:
# TODO: Sorting categories in alphabetic order.
# e.g. using categories.sort()
# TODO: Taking main cats to top
# for name in categories:
# if re.search(u"(.+?)\|(.{,1}?)",name.title()) or name.title()==name.title().split(":")[0]+title:
# categories.remove(name)
# categories.insert(0, name)
text = textlib.replaceCategoryLinks(text, categories,
site=self.site)
# Adding stars templates
if allstars:
text = text.strip() + self.site.family.interwiki_text_separator
allstars.sort()
for element in allstars:
text += '%s%s' % (element.strip(), config.line_separator)
pywikibot.log(u'%s' % element.strip())
# Adding the interwiki
if interwikiLinks:
text = textlib.replaceLanguageLinks(text, interwikiLinks,
site=self.site,
template=self.template,
template_subpage=subpage)
return text
def add_template(self, source, dest, task, fromsite):
"""Place or remove the Link_GA/FA template on/from a page."""
def compile_link(site, templates):
"""Compile one link template list."""
findtemplate = '(%s)' % '|'.join(templates)
return re.compile(
r'\{\{%s\|%s\}\}' %
(findtemplate.replace(' ', '[ _]'), site.code), re.IGNORECASE)
tosite = dest.site
add_tl, remove_tl = self.getTemplateList(tosite.code, task)
re_link_add = compile_link(fromsite, add_tl)
re_link_remove = compile_link(fromsite, remove_tl)
text = dest.text
m1 = add_tl and re_link_add.search(text)
m2 = remove_tl and re_link_remove.search(text)
changed = False
interactive = self.getOption('interactive')
if add_tl:
if m1:
pywikibot.output('(already added)')
else:
# insert just before interwiki
if (not interactive
or pywikibot.input_yn('Connecting %s -> %s. Proceed?' %
(source.title(), dest.title()),
default=False,
automatic_quit=False)):
if self.getOption('side'):
# Placing {{Link FA|xx}} right next to
# corresponding interwiki
text = (text[:m1.end()] + ' {{%s|%s}}' %
(add_tl[0], fromsite.code) + text[m1.end():])
else:
# Moving {{Link FA|xx}} to top of interwikis
iw = textlib.getLanguageLinks(text, tosite)
text = textlib.removeLanguageLinks(text, tosite)
text += '%s{{%s|%s}}%s' % (config.LS, add_tl[0],
fromsite.code, config.LS)
text = textlib.replaceLanguageLinks(text, iw, tosite)
changed = True
if remove_tl:
if m2:
if (changed # Don't force the user to say "Y" twice
or not interactive or
pywikibot.input_yn('Connecting %s -> %s. Proceed?' %
(source.title(), dest.title()),
default=False,
automatic_quit=False)):
text = re.sub(re_link_remove, '', text)
changed = True
elif task == 'former':
pywikibot.output('(already removed)')
if changed:
comment = i18n.twtranslate(tosite, 'featured-' + task,
{'page': source})
try:
dest.put(text, comment)
self._save_counter += 1
except pywikibot.LockedPage:
pywikibot.output('Page %s is locked!' % dest.title())
except pywikibot.PageSaveRelatedError:
pywikibot.output('Page not saved')
def standardizePageFooter(self, text):
"""
Standardize page footer.
Makes sure that interwiki links and categories are put to the correct
position and into the right order. This combines the old instances
standardizeInterwiki and standardizeCategories.
The page footer has the following section in that sequence:
1. categories
2. ## TODO: template beyond categories ##
3. additional information depending on local site policy
4. interwiki links
"""
categories = None
interwikiLinks = None
# Pywikibot is no longer allowed to touch categories on the
# German Wikipedia. See
# https://de.wikipedia.org/wiki/Hilfe_Diskussion:Personendaten/Archiv/1#Position_der_Personendaten_am_.22Artikelende.22
# ignoring nn-wiki of cause of the comment line above iw section
if not self.template and '{{Personendaten' not in text and \
'{{SORTIERUNG' not in text and '{{DEFAULTSORT' not in text and \
self.site.code not in ('et', 'it', 'bg', 'ru'):
categories = textlib.getCategoryLinks(text, site=self.site)
if not self.talkpage: # and pywikibot.calledModuleName() <> 'interwiki':
subpage = False
if self.template:
loc = None
try:
tmpl, loc = moved_links[self.site.code]
del tmpl
except KeyError:
pass
if loc is not None and loc in self.title:
subpage = True
interwikiLinks = textlib.getLanguageLinks(
text, insite=self.site, template_subpage=subpage)
# Removing the interwiki
text = textlib.removeLanguageLinks(text, site=self.site)
# Adding categories
if categories:
# TODO: Sorting categories in alphabetic order.
# e.g. using categories.sort()
# TODO: Taking main cats to top
# for name in categories:
# if (re.search(u"(.+?)\|(.{,1}?)",name.title()) or
# name.title() == name.title().split(":")[0] + title):
# categories.remove(name)
# categories.insert(0, name)
text = textlib.replaceCategoryLinks(text, categories,
site=self.site)
# Adding the interwiki
if interwikiLinks:
text = textlib.replaceLanguageLinks(text, interwikiLinks,
site=self.site,
template=self.template,
template_subpage=subpage)
return text
def add_text(page,
addText,
summary=None,
regexSkip=None,
regexSkipUrl=None,
always=False,
up=False,
putText=True,
oldTextGiven=None,
reorderEnabled=True,
create=False):
"""
Add text to a page.
@rtype: tuple of (text, newtext, always)
"""
site = page.site
if not summary:
summary = i18n.twtranslate(site, 'add_text-adding',
{'adding': addText[:200]})
if putText:
pywikibot.output('Loading {}...'.format(page.title()))
text = get_text(page, oldTextGiven, create)
if text is None:
return (False, False, always)
# Understand if the bot has to skip the page or not
# In this way you can use both -except and -excepturl
if regexSkipUrl is not None:
url = page.full_url()
result = re.findall(regexSkipUrl, site.getUrl(url))
if result != []:
pywikibot.output('Exception! regex (or word) used with -exceptUrl '
'is in the page. Skip!\n'
'Match was: {}'.format(result))
return (False, False, always)
if regexSkip is not None:
result = re.findall(regexSkip, text)
if result != []:
pywikibot.output('Exception! regex (or word) used with -except '
'is in the page. Skip!\n'
'Match was: {}'.format(result))
return (False, False, always)
# If not up, text put below
if not up:
newtext = text
# Translating the \\n into binary \n
addText = addText.replace('\\n', config.line_separator)
if (reorderEnabled):
# Getting the categories
categoriesInside = textlib.getCategoryLinks(newtext, site)
# Deleting the categories
newtext = textlib.removeCategoryLinks(newtext, site)
# Getting the interwiki
interwikiInside = textlib.getLanguageLinks(newtext, site)
# Removing the interwiki
newtext = textlib.removeLanguageLinks(newtext, site)
# Adding the text
newtext += '{}{}'.format(config.line_separator, addText)
# Reputting the categories
newtext = textlib.replaceCategoryLinks(newtext, categoriesInside,
site, True)
# Adding the interwiki
newtext = textlib.replaceLanguageLinks(newtext, interwikiInside,
site)
else:
newtext += '{}{}'.format(config.line_separator, addText)
else:
newtext = addText + config.line_separator + text
if putText and text != newtext:
pywikibot.output(
color_format('\n\n>>> {lightpurple}{0}{default} <<<',
page.title()))
pywikibot.showDiff(text, newtext)
# Let's put the changes.
error_count = 0
while True:
# If someone load it as module, maybe it's not so useful to put the
# text in the page
if not putText:
return (text, newtext, always)
if not always:
try:
choice = pywikibot.input_choice(
'Do you want to accept these changes?',
[('Yes', 'y'), ('No', 'n'), ('All', 'a'),
('open in Browser', 'b')], 'n')
except QuitKeyboardInterrupt:
sys.exit('User quit bot run.')
if choice == 'a':
always = True
elif choice == 'n':
return (False, False, always)
elif choice == 'b':
pywikibot.bot.open_webbrowser(page)
if always or choice == 'y':
result = put_text(page,
newtext,
summary,
error_count,
asynchronous=not always)
if result is not None:
return (result, result, always)
error_count += 1
def assertFailedRoundtripInterwiki(self, text):
old_interwikis = textlib.getLanguageLinks(text, self.site)
new_text = textlib.replaceLanguageLinks(text, old_interwikis, site = self.site)
self.assertNotEqual(text, new_text)
def assertRoundtripInterwiki(self, text, count):
old_interwikis = textlib.getLanguageLinks(text, self.site)
new_text = textlib.replaceLanguageLinks(text, old_interwikis, site = self.site)
self.assertEqual(len(old_interwikis), count)
self.assertEqual(text, new_text)
def add_text(page, addText, summary=None, regexSkip=None,
regexSkipUrl=None, always=False, up=False, putText=True,
oldTextGiven=None, reorderEnabled=True, create=False):
"""
Add text to a page.
@rtype: tuple of (text, newtext, always)
"""
site = page.site
if not summary:
summary = i18n.twtranslate(site, 'add_text-adding',
{'adding': addText[:200]})
# When a page is tagged as "really well written" it has a star in the
# interwiki links. This is a list of all the templates used (in regex
# format) to make the stars appear.
errorCount = 0
if putText:
pywikibot.output(u'Loading %s...' % page.title())
if oldTextGiven is None:
try:
text = page.get()
except pywikibot.NoPage:
if create:
pywikibot.output(u"%s doesn't exist, creating it!"
% page.title())
text = u''
else:
pywikibot.output(u"%s doesn't exist, skip!" % page.title())
return (False, False, always)
except pywikibot.IsRedirectPage:
pywikibot.output(u"%s is a redirect, skip!" % page.title())
return (False, False, always)
else:
text = oldTextGiven
# Understand if the bot has to skip the page or not
# In this way you can use both -except and -excepturl
if regexSkipUrl is not None:
url = page.full_url()
result = re.findall(regexSkipUrl, site.getUrl(url))
if result != []:
pywikibot.output(
'Exception! regex (or word) used with -exceptUrl '
'is in the page. Skip!\n'
'Match was: %s' % result)
return (False, False, always)
if regexSkip is not None:
result = re.findall(regexSkip, text)
if result != []:
pywikibot.output(
'Exception! regex (or word) used with -except '
'is in the page. Skip!\n'
'Match was: %s' % result)
return (False, False, always)
# If not up, text put below
if not up:
newtext = text
# Translating the \\n into binary \n
addText = addText.replace('\\n', config.line_separator)
if (reorderEnabled):
# Getting the categories
categoriesInside = textlib.getCategoryLinks(newtext, site)
# Deleting the categories
newtext = textlib.removeCategoryLinks(newtext, site)
# Getting the interwiki
interwikiInside = textlib.getLanguageLinks(newtext, site)
# Removing the interwiki
newtext = textlib.removeLanguageLinks(newtext, site)
# Adding the text
newtext += u"%s%s" % (config.line_separator, addText)
# Reputting the categories
newtext = textlib.replaceCategoryLinks(newtext,
categoriesInside, site,
True)
# Dealing the stars' issue
allstars = []
starstext = textlib.removeDisabledParts(text)
for star in starsList:
regex = re.compile('(\{\{(?:template:|)%s\|.*?\}\}[\s]*)'
% star, re.I)
found = regex.findall(starstext)
if found != []:
newtext = regex.sub('', newtext)
allstars += found
if allstars != []:
newtext = newtext.strip() + config.line_separator * 2
allstars.sort()
for element in allstars:
newtext += '%s%s' % (element.strip(), config.LS)
# Adding the interwiki
newtext = textlib.replaceLanguageLinks(newtext, interwikiInside,
site)
else:
newtext += u"%s%s" % (config.line_separator, addText)
else:
newtext = addText + config.line_separator + text
if putText and text != newtext:
pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
% page.title())
pywikibot.showDiff(text, newtext)
# Let's put the changes.
while True:
# If someone load it as module, maybe it's not so useful to put the
# text in the page
if putText:
if not always:
choice = pywikibot.input_choice(
u'Do you want to accept these changes?',
[('Yes', 'y'), ('No', 'n'), ('All', 'a'),
('open in Browser', 'b')], 'n', automatic_quit=False)
if choice == 'a':
always = True
elif choice == 'n':
return (False, False, always)
elif choice == 'b':
pywikibot.bot.open_webbrowser(page)
if always or choice == 'y':
try:
if always:
page.put(newtext, summary,
minorEdit=page.namespace() != 3)
else:
page.put_async(newtext, summary,
minorEdit=page.namespace() != 3)
except pywikibot.EditConflict:
pywikibot.output(u'Edit conflict! skip!')
return (False, False, always)
except pywikibot.ServerError:
errorCount += 1
if errorCount < config.max_retries:
pywikibot.output(u'Server Error! Wait..')
time.sleep(config.retry_wait)
continue
else:
raise pywikibot.ServerError(u'Fifth Server Error!')
except pywikibot.SpamfilterError as e:
pywikibot.output(
u'Cannot change %s because of blacklist entry %s'
% (page.title(), e.url))
return (False, False, always)
except pywikibot.LockedPage:
pywikibot.output(u'Skipping %s (locked page)'
% page.title())
return (False, False, always)
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Error putting page: %s' % error.args)
return (False, False, always)
else:
# Break only if the errors are one after the other...
errorCount = 0
return (True, True, always)
else:
return (text, newtext, always)
def standardizePageFooter(self, text):
"""
Standardize page footer.
Makes sure that interwiki links and categories are put
into the correct position and into the right order. This
combines the old instances of standardizeInterwiki
and standardizeCategories.
The page footer consists of the following parts
in that sequence:
1. categories
2. additional information depending on the local site policy
3. interwiki
"""
categories = []
interwiki_links = []
# get categories
if not self.template:
categories = textlib.getCategoryLinks(text, site=self.site)
if not self.talkpage:
subpage = False
if self.template:
try:
tmpl, loc = moved_links[self.site.code]
del tmpl
except KeyError:
loc = None
if loc is not None and loc in self.title:
subpage = True
# get interwiki
interwiki_links = textlib.getLanguageLinks(
text, insite=self.site, template_subpage=subpage)
# remove interwiki
text = textlib.removeLanguageLinks(text, site=self.site)
# add categories, main to top
if categories:
# TODO: Sort categories in alphabetic order, e.g. using
# categories.sort()? (T100265)
# TODO: Get main categories from Wikidata?
main = pywikibot.Category(self.site, 'Category:' + self.title,
sort_key=' ')
if main in categories:
categories.pop(categories.index(main))
categories.insert(0, main)
text = textlib.replaceCategoryLinks(text, categories,
site=self.site)
# add interwiki
if interwiki_links:
text = textlib.replaceLanguageLinks(text, interwiki_links,
site=self.site,
template=self.template,
template_subpage=subpage)
return text
