def __init__(self, generator, **kwargs):
self.availableOptions.update({
'enablePage': None, # Check if someone set an enablePage or not
'disambigPage': None, # If no disambigPage given, not use it.
})
super(LonelyPagesBot, self).__init__(**kwargs)
self.generator = generator
# Take the configurations according to our project
self.site = pywikibot.Site()
if self.getOption('enablePage'):
self.options['enablePage'] = pywikibot.Page(
self.site, self.getOption('enablePage'))
self.comment = i18n.twtranslate(
self.site, 'lonelypages-comment-add-template')
self.commentdisambig = i18n.twtranslate(
self.site, 'lonelypages-comment-add-disambig-template')
self.template = i18n.translate(self.site, template)
self.exception = i18n.translate(self.site, exception_regex)
if self.template is None or self.exception is None:
pywikibot.showHelp()
sys.exit(u'Missing configuration for site %s' % self.site)
# DisambigPage part
if self.getOption('disambigPage') is not None:
self.disambigpage = pywikibot.Page(self.site, self.getOption('disambigPage'))
try:
self.disambigtext = self.disambigpage.get()
except pywikibot.NoPage:
pywikibot.output(u"%s doesn't esist, skip!" % self.disambigpage.title())
self.disambigtext = ''
except pywikibot.IsRedirectPage:
pywikibot.output(u"%s is a redirect, don't use it!"
% self.disambigpage.title())
self.options['disambigPage'] = None
def delete_redirect(self, page, summary_key):
"""Delete the redirect page."""
assert page.site == self.site, (
'target page is on different site {0}'.format(page.site))
reason = i18n.twtranslate(self.site, summary_key)
if page.site.logged_in(sysop=True):
page.delete(reason, prompt=False)
elif i18n.twhas_key(page.site,
'redirect-broken-redirect-template'):
pywikibot.output(u"No sysop in user-config.py, "
u"put page to speedy deletion.")
try:
content = page.get(get_redirect=True)
except pywikibot.SectionError:
content_page = pywikibot.Page(page.site,
page.title(withSection=False))
content = content_page.get(get_redirect=True)
# TODO: Add bot's signature if needed (Bug: T131517)
content = i18n.twtranslate(
page.site,
'redirect-broken-redirect-template') + '\n' + content
try:
page.put(content, reason)
except pywikibot.PageSaveRelatedError as e:
pywikibot.error(e)
else:
pywikibot.output(
u'No speedy deletion template available')
def testSemiLocalized(self):
self.assertEqual(i18n.twtranslate('en', 'test-semi-localized'),
u'test-semi-localized EN')
self.assertEqual(i18n.twtranslate('nl', 'test-semi-localized'),
u'test-semi-localized NL')
self.assertEqual(i18n.twtranslate('fy', 'test-semi-localized'),
u'test-semi-localized NL')
def save(self, title, contents):
"""Upload page content."""
mysite = pywikibot.Site()
page = pywikibot.Page(mysite, title)
self.current_page = page
if self.getOption('summary'):
comment = self.getOption('summary')
else:
comment = i18n.twtranslate(mysite, 'pagefromfile-msg')
comment_top = comment + " - " + i18n.twtranslate(
mysite, 'pagefromfile-msg_top')
comment_bottom = comment + " - " + i18n.twtranslate(
mysite, 'pagefromfile-msg_bottom')
comment_force = "%s *** %s ***" % (
comment, i18n.twtranslate(mysite, 'pagefromfile-msg_force'))
# Remove trailing newlines (cause troubles when creating redirects)
contents = re.sub('^[\r\n]*', '', contents)
if page.exists():
if not self.getOption('redirect') and page.isRedirectPage():
pywikibot.output(u"Page %s is redirect, skipping!" % title)
return
pagecontents = page.get(get_redirect=True)
nocontent = self.getOption('nocontent')
if nocontent and (
nocontent in pagecontents or
nocontent.lower() in pagecontents):
pywikibot.output('Page has %s so it is skipped' % nocontent)
return
if self.getOption('append') == 'top':
pywikibot.output(u"Page %s already exists, appending on top!"
% title)
contents = contents + pagecontents
comment = comment_top
elif self.getOption('append') == 'bottom':
pywikibot.output(u"Page %s already exists, appending on bottom!"
% title)
contents = pagecontents + contents
comment = comment_bottom
elif self.getOption('force'):
pywikibot.output(u"Page %s already exists, ***overwriting!"
% title)
comment = comment_force
else:
pywikibot.output(u"Page %s already exists, not adding!" % title)
return
else:
if self.getOption('autosummary'):
comment = ''
config.default_edit_summary = ''
self.userPut(page, page.text, contents,
summary=comment,
minor=self.getOption('minor'),
show_diff=False,
ignore_save_related_errors=True)
def run(self):
newCat = pywikibot.Category(self.site, self.newCatTitle)
reason = i18n.twtranslate(self.site, 'category-was-moved') \
% {'newcat': self.newCatTitle, 'title': self.newCatTitle}
self.editSummary = i18n.twtranslate(site, 'category-changing') \
% {'oldcat':self.oldCat.title(),
'newcat':newCat.title()}
copied = False
oldMovedTalk = None
if self.oldCat.exists():
copied = copyAndKeep(self.oldCat, self.newCatTitle)
if copied:
oldTalk = self.oldCat.toggleTalkPage()
if oldTalk.exists():
newTalkTitle = newCat.toggleTalkPage().title()
try:
talkMoved = oldTalk.move(newTalkTitle, reason)
except (pywikibot.NoPage, pywikibot.PageNotSaved), e:
#in order :
#Source talk does not exist, or
#Target talk already exists
pywikibot.output(e.message)
else:
if talkMoved:
oldMovedTalk = oldTalk
def main():
genFactory = pagegenerators.GeneratorFactory()
pageName = ''
summary = None
always = False
undelete = False
generator = None
# read command line parameters
local_args = pywikibot.handleArgs()
mysite = pywikibot.Site()
for arg in local_args:
if arg == '-always':
always = True
elif arg.startswith('-summary'):
if len(arg) == len('-summary'):
summary = pywikibot.input(u'Enter a reason for the deletion:')
else:
summary = arg[len('-summary:'):]
elif arg.startswith('-images'):
pywikibot.output('\n\03{lightred}-image option is deprecated. '
'Please use -imageused instead.\03{default}\n')
local_args.append('-imageused' + arg[7:])
elif arg.startswith('-undelete'):
undelete = True
else:
genFactory.handleArg(arg)
found = arg.find(':') + 1
if found:
pageName = arg[found:]
if not summary:
if pageName:
if arg.startswith('-cat') or arg.startswith('-subcats'):
summary = i18n.twtranslate(mysite, 'delete-from-category',
{'page': pageName})
elif arg.startswith('-links'):
summary = i18n.twtranslate(mysite, 'delete-linked-pages',
{'page': pageName})
elif arg.startswith('-ref'):
summary = i18n.twtranslate(mysite, 'delete-referring-pages',
{'page': pageName})
elif arg.startswith('-imageused'):
summary = i18n.twtranslate(mysite, 'delete-images',
{'page': pageName})
elif arg.startswith('-file'):
summary = i18n.twtranslate(mysite, 'delete-from-file')
generator = genFactory.getCombinedGenerator()
# We are just deleting pages, so we have no need of using a preloading
# page generator to actually get the text of those pages.
if generator:
if summary is None:
summary = pywikibot.input(u'Enter a reason for the %sdeletion:'
% ['', 'un'][undelete])
bot = DeletionRobot(generator, summary, always, undelete)
bot.run()
else:
# Show help text from the top of this file
pywikibot.showHelp()
def run(self):
site = pywikibot.getSite()
newCat = catlib.Category(pywikibot.Link('Category:' + self.newCatTitle))
# set edit summary message
if not self.editSummary:
self.editSummary = i18n.twtranslate(site, 'category-replacing',\
{'oldcat':self.oldCat.title(),
'newcat':newCat.title()})
# Copy the category contents to the new category page
copied = False
oldMovedTalk = None
if self.oldCat.exists() and self.moveCatPage:
copied = self.oldCat.copyAndKeep(
self.newCatTitle,
pywikibot.translate(site, cfd_templates),
i18n.twtranslate(site, 'category-renamed')
)
# Also move the talk page
if copied:
reason = i18n.twtranslate(site, 'category-was-moved', \
{'newcat':self.newCatTitle, 'title':self.newCatTitle})
oldTalk = self.oldCat.toggleTalkPage()
if oldTalk.exists():
newTalkTitle = newCat.toggleTalkPage().title()
try:
talkMoved = oldTalk.move(newTalkTitle, reason)
except (pywikibot.NoPage, pywikibot.PageNotSaved), e:
#in order :
#Source talk does not exist, or
#Target talk already exists
pywikibot.output(e.message)
else:
if talkMoved:
oldMovedTalk = oldTalk
def run(self, sim=False, msg=None, EditFlags=bot_config['EditFlags']):
"""Run SubsterBot()."""
pywikibot.output(
u'\03{lightgreen}* Processing Template Backlink List:\03{default}')
if sim:
self.pagegen = ['dummy']
for page in self.pagegen:
# setup source to get data from
if sim:
content = sim['content']
params = [sim]
else:
pywikibot.output(u'Getting page "%s" via API from %s...'
% (page.title(asLink=True), self.site))
# get page content and operating mode
content = self.load(page)
params = self.loadTemplates(
page,
self._bot_config['TemplateName'],
default=self._bot_config['param_default'])
if not params:
continue
(substed_content, substed_tags) = self.subContent(content, params)
# output result to page or return directly
if sim:
return substed_content
else:
# if changed, write!
if (substed_content != content):
#if substed_tags:
self.outputContentDiff(content, substed_content)
head = i18n.twtranslate(self.site.lang,
'thirdparty-drtrigonbot-sum_disc-summary-head')
if msg is None:
msg = i18n.twtranslate(self.site.lang,
'thirdparty-drtrigonbot-subster-summary-mod')
flags = copy.deepcopy(EditFlags)
if page.title() in self._flagenable:
flags.update(self._flagenable[page.title()])
pywikibot.output(u'Flags used for writing: %s' % flags)
self.save(page, substed_content,
(head + u' ' + msg)
% {'tags': ", ".join(substed_tags)},
**flags)
# DRTRIGON-130: data repository (wikidata) output to items
if self.site.is_data_repository():
data = self.data_convertContent(substed_content)
self.data_save(page, data)
else:
pywikibot.output(u'NOTHING TO DO!')
def treat_page(self):
"""Upload page content."""
page = self.current_page
title = page.title()
# save the content retrieved from generator
contents = page.text
# delete page's text to get it from live wiki
del page.text
if self.getOption('summary'):
comment = self.getOption('summary')
else:
comment = i18n.twtranslate(self.site, 'pagefromfile-msg')
comment_top = comment + " - " + i18n.twtranslate(
self.site, 'pagefromfile-msg_top')
comment_bottom = comment + " - " + i18n.twtranslate(
self.site, 'pagefromfile-msg_bottom')
comment_force = "%s *** %s ***" % (
comment, i18n.twtranslate(self.site, 'pagefromfile-msg_force'))
if page.exists():
if not self.getOption('redirect') and page.isRedirectPage():
pywikibot.output(u"Page %s is redirect, skipping!" % title)
return
pagecontents = page.text
nocontent = self.getOption('nocontent')
if nocontent and (
nocontent in pagecontents or
nocontent.lower() in pagecontents):
pywikibot.output('Page has %s so it is skipped' % nocontent)
return
if self.getOption('append'):
separator = self.getOption('append')[1]
if separator == r'\n':
separator = '\n'
if self.getOption('append')[0] == 'top':
above, below = contents, pagecontents
comment = comment_top
else:
above, below = pagecontents, contents
comment = comment_bottom
pywikibot.output('Page {0} already exists, appending on {1}!'.format(
title, self.getOption('append')[0]))
contents = above + separator + below
elif self.getOption('force'):
pywikibot.output(u"Page %s already exists, ***overwriting!"
% title)
comment = comment_force
else:
pywikibot.output(u"Page %s already exists, not adding!" % title)
return
else:
if self.getOption('autosummary'):
comment = config.default_edit_summary = ''
self.put_current(contents, summary=comment,
minor=self.getOption('minor'),
show_diff=self.getOption('showdiff'))
def testSemiLocalized(self):
"""Test translating with fallback to alternative language."""
self.assertEqual(i18n.twtranslate('en', 'test-semi-localized'),
u'test-semi-localized EN')
self.assertEqual(i18n.twtranslate('nl', 'test-semi-localized'),
u'test-semi-localized NL')
self.assertEqual(i18n.twtranslate('fy', 'test-semi-localized'),
u'test-semi-localized NL')
def testLocalized(self):
"""Test fully localized entry."""
self.assertEqual(i18n.twtranslate('en', 'test-localized'),
u'test-localized EN')
self.assertEqual(i18n.twtranslate('nl', 'test-localized'),
u'test-localized NL')
self.assertEqual(i18n.twtranslate('fy', 'test-localized'),
u'test-localized FY')
def __init__(self, **kwargs):
"""Constructor."""
self.availableOptions.update({
'tiny': False, # use Non-empty category redirects only
'delay': 7, # cool down delay in days
})
super(CategoryRedirectBot, self).__init__(**kwargs)
self.cooldown = self.getOption('delay')
self.site = pywikibot.Site()
self.catprefix = self.site.namespace(14) + ":"
self.log_text = []
self.edit_requests = []
self.problems = []
self.template_list = []
self.cat = None
self.log_page = pywikibot.Page(self.site,
u"User:%(user)s/category redirect log"
% {'user': self.site.username()})
# Localization:
# Category that contains all redirected category pages
self.cat_redirect_cat = {
'commons': "Category:Category redirects",
'meta': 'Category:Maintenance of categories/Soft redirected categories',
'ar': u"تصنيف:تحويلات تصنيفات ويكيبيديا",
'cs': 'Kategorie:Údržba:Zastaralé kategorie',
'da': "Kategori:Omdirigeringskategorier",
'en': "Category:Wikipedia soft redirected categories",
'es': "Categoría:Wikipedia:Categorías redirigidas",
'fa': u"رده:ردههای منتقلشده",
'hi': 'श्रेणी:विकिपीडिया श्रेणी अनुप्रेषित',
'hu': "Kategória:Kategóriaátirányítások",
'ja': "Category:移行中のカテゴリ",
'no': "Kategori:Wikipedia omdirigertekategorier",
'pl': "Kategoria:Przekierowania kategorii",
'pt': "Categoria:!Redirecionamentos de categorias",
'ru': "Категория:Википедия:Категории-дубликаты",
'sco': "Category:Wikipaedia soft redirectit categories",
'simple': "Category:Category redirects",
'sh': u"Kategorija:Preusmjerene kategorije Wikipedije",
'sr': 'Категорија:Википедијине меко преусмерене категорије',
'vi': u"Thể loại:Thể loại đổi hướng",
'zh': u"Category:已重定向的分类",
'ro': 'Categorie:Categorii de redirecționare',
}
# Category that contains non-empty redirected category pages
self.tiny_cat_redirect_cat = 'Q8099903'
self.move_comment = 'category_redirect-change-category'
self.redir_comment = 'category_redirect-add-template'
self.dbl_redir_comment = 'category_redirect-fix-double'
self.maint_comment = 'category_redirect-comment'
self.edit_request_text = i18n.twtranslate(
self.site, 'category_redirect-edit-request') + u'\n~~~~'
self.edit_request_item = i18n.twtranslate(
self.site, 'category_redirect-edit-request-item')
def testNonLocalized(self):
self.assertEqual(i18n.twtranslate('en', 'test-non-localized'),
u'test-non-localized EN')
self.assertEqual(i18n.twtranslate('fy', 'test-non-localized'),
u'test-non-localized EN')
self.assertEqual(i18n.twtranslate('nl', 'test-non-localized'),
u'test-non-localized EN')
self.assertEqual(i18n.twtranslate('ru', 'test-non-localized'),
u'test-non-localized EN')
def __init__(self, oldcat, newcat=None, batch=False, comment='',
inplace=False, move_oldcat=True, delete_oldcat=True,
title_regex=None, history=False, pagesonly=False):
"""Stores all given parameters in the objects attributes.
@param oldcat: The move source.
@param newcat: The move target.
@param batch: If True the user has not to confirm the deletion.
@param comment: The edit summary for all pages where the
category is changed.
@param inplace: If True the categories are not reordered.
@param move_oldcat: If True the category page (and talkpage) is
copied to the new category.
@param delete_oldcat: If True the oldcat page and talkpage are
deleted (or nominated for deletion) if it is empty.
@param title_regex: Only pages (and subcats) with a title that
matches the regex are moved.
@param history: If True the history of the oldcat is posted on
the talkpage of newcat.
@param pagesonly: If True only move pages, not subcategories.
"""
self.site = pywikibot.Site()
# Create attributes for the categories and their talk pages.
self.oldcat = self._makecat(oldcat)
self.oldtalk = self.oldcat.toggleTalkPage()
if newcat:
self.newcat = self._makecat(newcat)
self.newtalk = self.newcat.toggleTalkPage()
else:
self.newcat = None
self.newtalk = None
# Set boolean settings.
self.inplace = inplace
self.move_oldcat = move_oldcat
self.delete_oldcat = delete_oldcat
self.batch = batch
self.title_regex = title_regex
self.history = history
self.pagesonly = pagesonly
# Set edit summary for changed pages.
self.comment = comment
if not self.comment:
if self.newcat:
template_vars = { \
'oldcat': self.oldcat.title(
withNamespace=False),
'newcat': self.newcat.title(
withNamespace=False)}
self.comment = i18n.twtranslate(self.site,
'category-replacing',
template_vars)
else:
template_vars = {'oldcat': self.oldcat.title( \
withNamespace=False)}
self.comment = i18n.twtranslate(self.site,
'category-removing',
template_vars)
def testNonLocalized(self):
"""Test translating non localized entries."""
self.assertEqual(i18n.twtranslate('en', 'test-non-localized'),
u'test-non-localized EN')
self.assertEqual(i18n.twtranslate('fy', 'test-non-localized'),
u'test-non-localized EN')
self.assertEqual(i18n.twtranslate('nl', 'test-non-localized'),
u'test-non-localized EN')
self.assertEqual(i18n.twtranslate('ru', 'test-non-localized'),
u'test-non-localized EN')
def _hist(self):
"""Private function to copy the history of the to-be-deleted category.
Do not use this function from outside the class. It adds a table with
the history of the old category on the new talk page.
"""
history = self.oldcat.getVersionHistoryTable()
title = i18n.twtranslate(self.site, "category-section-title", {"oldcat": self.oldcat.title()})
self.newtalk.text = "%s\n== %s ==\n%s" % (self.newtalk.text, title, history)
comment = i18n.twtranslate(self.site, "category-version-history", {"oldcat": self.oldcat.title()})
self.newtalk.save(comment)
def _hist(self):
"""Private function to create a history table with the history
of the old category on the new talk page.
Do not use this function from outside the class.
"""
history = self.oldcat.getVersionHistoryTable()
title = i18n.twtranslate(self.site, 'category-section-title',
{'oldcat': self.oldcat.title()})
self.newtalk.text = "%s\n== %s ==\n%s" % (self.newtalk.text,
title, history)
comment = i18n.twtranslate(self.site, 'category-version-history',
{'oldcat': self.oldcat.title()})
self.newtalk.save(comment)
def __init__(self):
self.cooldown = 7 # days
self.site = pywikibot.getSite()
self.site.login()
self.catprefix = self.site.namespace(14)+":"
self.log_text = []
self.edit_requests = []
self.log_page = pywikibot.Page(self.site,
u"User:%(user)s/category redirect log" %
{'user': self.site.user()})
# Localization:
# Category that contains all redirected category pages
self.cat_redirect_cat = {
'wikipedia': {
'ar': u"تصنيف:تحويلات تصنيفات ويكيبيديا",
'cs': u"Kategorie:Zastaralé kategorie",
'da': "Kategori:Omdirigeringskategorier",
'en': "Category:Wikipedia soft redirected categories",
'es': "Categoría:Wikipedia:Categorías redirigidas",
'fa': u"رده:ردههای منتقل شده",
'hu': "Kategória:Kategóriaátirányítások",
'ja': "Category:移行中のカテゴリ",
'no': "Kategori:Wikipedia omdirigertekategorier",
'pl': "Kategoria:Przekierowania kategorii",
'pt': "Categoria:!Redirecionamentos de categorias",
'ru': "Категория:Википедия:Категории-дубликаты",
'simple': "Category:Category redirects",
'vi': u"Thể loại:Thể loại đổi hướng",
'zh': u"Category:已重定向的分类",
},
'commons': {
'commons': "Category:Category redirects"
}
}
self.move_comment = 'category_redirect-change-category'
self.redir_comment = 'category_redirect-add-template'
self.dbl_redir_comment = 'category_redirect-fix-double'
self.maint_comment = 'category_redirect-comment'
self.edit_request_text = i18n.twtranslate(
self.site.lang,
'category_redirect-edit-request') + \
u'\n~~~~'
self.edit_request_item = i18n.twtranslate(
self.site.lang,
'category_redirect-edit-request-item')
def __init__(self, generator, img, info, imdb):
"""
Constructor. Parameters:
@param generator: The page generator that determines on which pages
to work.
@type generator: generator.
"""
self.generator = generator
# Set the edit summary message
self.summary = i18n.twtranslate(pywikibot.getSite(), 'basic-changing')
self.chrome = filmsettings.getChrome()
self.img = img
self.info = info
self.imdb = imdb
self.imdbNum = 0
self.templateRegex = re.compile("{{.*}}") #This is how templates are in wikipedia
self.referenceRegex = re.compile("(<ref.*?/(ref)?>)+")
self.commentRegex = re.compile("<!--.*?-->")
self.wikilinkRegex = re.compile("\[\[.*\|.*\]\]")
infoTemp = pywikibot.Page(pywikibot.getSite(), "Template:Infobox_film/doc").get()
infoTempStart = infoTemp.find("{{Infobox film") + 2
bracketCount = 2
infoTempEnd = infoTempStart
while bracketCount != 0 :
infoTempEnd += 1
if infoTemp[infoTempEnd:infoTempEnd+1] == "{":
bracketCount += 1
elif infoTemp[infoTempEnd:infoTempEnd+1] == "}":
bracketCount -= 1
self.infoboxTemplate = re.sub(self.commentRegex, "", infoTemp[infoTempStart - 2:infoTempEnd+1])
def changeCommonscat(self, page=None, oldtemplate=u'', oldcat=u'',
newtemplate=u'', newcat=u'', linktitle=u'',
description=NotImplemented): # pylint: disable=unused-argument
"""Change the current commonscat template and target."""
if oldcat == '3=S' or linktitle == '3=S':
return # TODO: handle additional param on de-wiki
if not linktitle and (page.title().lower() in oldcat.lower() or
oldcat.lower() in page.title().lower()):
linktitle = oldcat
if linktitle and newcat != page.title(withNamespace=False):
newtext = re.sub(u'(?i)\{\{%s\|?[^{}]*(?:\{\{.*\}\})?\}\}'
% oldtemplate,
u'{{%s|%s|%s}}' % (newtemplate, newcat, linktitle),
page.get())
elif newcat == page.title(withNamespace=False):
newtext = re.sub(u'(?i)\{\{%s\|?[^{}]*(?:\{\{.*\}\})?\}\}'
% oldtemplate,
u'{{%s}}' % newtemplate,
page.get())
elif oldcat.strip() != newcat: # strip trailing white space
newtext = re.sub(u'(?i)\{\{%s\|?[^{}]*(?:\{\{.*\}\})?\}\}'
% oldtemplate,
u'{{%s|%s}}' % (newtemplate, newcat),
page.get())
else: # nothing left to do
return
if self.getOption('summary'):
comment = self.getOption('summary')
else:
comment = i18n.twtranslate(page.site,
'commonscat-msg_change',
{'oldcat': oldcat, 'newcat': newcat})
self.userPut(page, page.text, newtext, summary=comment,
ignore_save_related_errors=True)
def __init__(self, source, archiver, params=None):
super(DiscussionPage, self).__init__(source)
self.threads = []
self.full = False
self.archiver = archiver
# for testing purposes we allow archiver to be None and we are able
# to create the a DiscussionPage in this way:
# >>> import pywikibot as py
# >>> from scripts.archivebot import DiscussionPage
# >>> d = DiscussionPage(py.Page(py.Site(), <talk page name>), None)
if archiver is None:
self.timestripper = TimeStripper(self.site)
else:
self.timestripper = self.archiver.timestripper
self.params = params
self.now = datetime.datetime.utcnow().replace(tzinfo=TZoneUTC())
try:
self.load_page()
except pywikibot.NoPage:
self.header = archiver.get_attr('archiveheader',
i18n.twtranslate(
self.site.code,
'archivebot-archiveheader'))
if self.params:
self.header = self.header % self.params
def __init__(self, generator, to13=False, format=False, always=False):
self.generator = generator
self.to13 = to13
self.format = format
self.always = always
self.isbnR = re.compile(r'(?<=ISBN )(?P<code>[\d\-]+[Xx]?)')
self.comment = i18n.twtranslate(pywikibot.getSite(), 'isbn-formatting')
def generate_summary(self, applied_replacements):
"""Generate a summary message for the replacements."""
# all replacements which are merged into the default message
default_summaries = set()
# all message parts
summary_messages = set()
for replacement in applied_replacements:
if replacement.edit_summary:
summary_messages.add(replacement.edit_summary)
elif replacement.default_summary:
default_summaries.add((replacement.old, replacement.new))
summary_messages = sorted(summary_messages)
if default_summaries:
if self.summary:
summary_messages.insert(0, self.summary)
else:
comma = self.site.mediawiki_message("comma-separator")
default_summary = comma.join(
"-{0} +{1}".format(*default_summary) for default_summary in default_summaries
)
summary_messages.insert(
0,
i18n.twtranslate(self.site, "replace-replacing", {"description": " ({0})".format(default_summary)}),
)
semicolon = self.site.mediawiki_message("semicolon-separator")
return semicolon.join(summary_messages)
def __init__(self, link, name):
self.refname = name
self.link = link
self.site = pywikibot.Site()
self.linkComment = i18n.twtranslate(self.site, 'reflinks-comment')
self.url = re.sub(u'#.*', '', self.link)
self.title = None
def __init__(self, generator, **kwargs):
"""Constructor."""
self.availableOptions.update({
'verbose': True,
})
super(NoReferencesBot, self).__init__(**kwargs)
self.generator = pagegenerators.PreloadingGenerator(generator)
self.site = pywikibot.Site()
self.comment = i18n.twtranslate(self.site, 'noreferences-add-tag')
self.refR = re.compile('</ref>', re.IGNORECASE)
self.referencesR = re.compile('<references.*?/>', re.IGNORECASE)
self.referencesTagR = re.compile('<references>.*?</references>',
re.IGNORECASE | re.DOTALL)
try:
self.referencesTemplates = referencesTemplates[
self.site.family.name][self.site.code]
except KeyError:
self.referencesTemplates = []
try:
self.referencesText = referencesSubstitute[
self.site.family.name][self.site.code]
except KeyError:
self.referencesText = u'<references />'
def tagNowcommons(self, imagepage, filename):
""" Tagged the imag which has been moved to Commons for deletion. """
if pywikibot.Page(pywikibot.getSite('commons', 'commons'),
u'File:' + filename).exists():
# Get a fresh copy, force to get the page so we dont run into edit
# conflicts
imtxt = imagepage.get(force=True)
# Remove the move to commons templates
if imagepage.site().language() in moveToCommonsTemplate:
for moveTemplate in moveToCommonsTemplate[imagepage.site().language()]:
imtxt = re.sub(u'(?i)\{\{' + moveTemplate +
u'[^\}]*\}\}', u'', imtxt)
# add {{NowCommons}}
if imagepage.site().language() in nowCommonsTemplate:
addTemplate = nowCommonsTemplate[
imagepage.site().language()] % filename
else:
addTemplate = nowCommonsTemplate['_default'] % filename
commentText = i18n.twtranslate(
imagepage.site(), 'commons-file-now-available',
{'localfile': imagepage.title(withNamespace=False),
'commonsfile': filename})
pywikibot.showDiff(imagepage.get(), imtxt + addTemplate)
imagepage.put(imtxt + addTemplate, comment=commentText)
def run(self):
setOfArticles = set(self.cat.articles(recurse=self.recurse))
if self.subCats:
setOfArticles = setOfArticles.union(set(self.cat.subcategories()))
if not self.editSummary:
self.editSummary = i18n.twtranslate(self.site,
'category-listifying',
{'fromcat': self.cat.title(),
'num': len(setOfArticles)})
listString = ""
for article in setOfArticles:
if (not article.isImage() or self.showImages) and not article.isCategory():
if self.talkPages and not article.isTalkPage():
listString = listString + "*[[%s]] -- [[%s|talk]]\n" % (article.title(), article.toggleTalkPage().title())
else:
listString = listString + "*[[%s]]\n" % article.title()
else:
if self.talkPages and not article.isTalkPage():
listString = listString + "*[[:%s]] -- [[%s|talk]]\n" % (article.title(), article.toggleTalkPage().title())
else:
listString = listString + "*[[:%s]]\n" % article.title()
if self.list.exists() and not self.overwrite:
pywikibot.output(u'Page %s already exists, aborting.' % self.list.title())
else:
self.list.put(listString, comment=self.editSummary)
def revert(self, item):
history = pywikibot.Page(self.site, item['title']).fullVersionHistory(
total=2, rollback=self.rollback)
if len(history) > 1:
rev = history[1]
else:
return False
comment = i18n.twtranslate(pywikibot.Site(), 'revertbot-revert', {'revid': rev[0], 'author': rev[2], 'timestamp': rev[1]})
if self.comment:
comment += ': ' + self.comment
page = pywikibot.Page(self.site, item['title'])
pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
% page.title(asLink=True, forceInterwiki=True,
textlink=True))
if not self.rollback:
old = page.text
page.text = rev[3]
pywikibot.showDiff(old, page.text)
page.save(comment)
return comment
try:
pywikibot.data.api.Request(action="rollback", title=page.title(), user=self.user,
token=rev[4], markbot=1).submit()
except pywikibot.data.api.APIError as e:
if e == "badtoken: Invalid token":
pywikibot.out("There is an issue for rollbacking the edit, Giving up")
return False
return u"The edit(s) made in %s by %s was rollbacked" % (page.title(), self.user)
def delete_broken_redirects(self):
# get reason for deletion text
reason = i18n.twtranslate(self.site, 'redirect-remove-broken')
for redir_name in self.generator.retrieve_broken_redirects():
self.delete_1_broken_redirect(redir_name, reason)
if self.exiting:
break
def run(self):
articles = set(self.cat.articles())
if len(articles) == 0:
pywikibot.output(u'There are no articles in category %s' % self.cat.title())
else:
for article in articles:
if not self.titleRegex or re.search(self.titleRegex, article.title()):
catlib.change_category(article, self.cat, None, comment=self.editSummary, inPlace=self.inPlace)
if self.pagesonly:
return
# Also removes the category tag from subcategories' pages
subcategories = set(self.cat.subcategories())
if len(subcategories) == 0:
pywikibot.output(u'There are no subcategories in category %s' % self.cat.title())
else:
for subcategory in subcategories:
catlib.change_category(subcategory, self.cat, None, comment=self.editSummary, inPlace=self.inPlace)
# Deletes the category page
if self.cat.exists() and self.cat.isEmptyCategory():
if self.useSummaryForDeletion and self.editSummary:
reason = self.editSummary
else:
reason = i18n.twtranslate(self.site, 'category-was-disbanded')
talkPage = self.cat.toggleTalkPage()
try:
self.cat.delete(reason, not self.batchMode)
except pywikibot.NoUsername:
pywikibot.output(u'You\'re not setup sysop info, category will not delete.' % self.cat.site())
return
if (talkPage.exists()):
talkPage.delete(reason=reason, prompt=not self.batchMode)
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
add_cat = None
gen = None
# summary message
edit_summary = u""
# Array which will collect commandline parameters.
# First element is original text, second element is replacement text.
commandline_replacements = []
# A list of 2-tuples of original text and replacement text.
replacements = []
# Don't edit pages which contain certain texts.
exceptions = {
'title': [],
'text-contains': [],
'inside': [],
'inside-tags': [],
'require-title': [], # using a seperate requirements dict needs some
} # major refactoring of code.
# Should the elements of 'replacements' and 'exceptions' be interpreted
# as regular expressions?
regex = False
# Predefined fixes from dictionary 'fixes' (see above).
fixes_set = []
# the dump's path, either absolute or relative, which will be used
# if -xml flag is present
xmlFilename = None
useSql = False
# will become True when the user presses a ('yes to all') or uses the
# -always flag.
acceptall = False
# Will become True if the user inputs the commandline parameter -nocase
caseInsensitive = False
# Will become True if the user inputs the commandline parameter -dotall
dotall = False
# Will become True if the user inputs the commandline parameter -multiline
multiline = False
# Do all hits when they overlap
allowoverlap = False
# Do not recurse replacement
recursive = False
# Between a regex and another (using -fix) sleep some time (not to waste
# too much CPU
sleep = None
# Request manual replacements even if replacements are already defined
manual_input = False
# Replacements loaded from a file
replacement_file = None
replacement_file_arg_misplaced = False
# Read commandline parameters.
local_args = pywikibot.handle_args(args)
genFactory = pagegenerators.GeneratorFactory()
for arg in local_args:
if genFactory.handleArg(arg):
continue
if arg == '-regex':
regex = True
elif arg.startswith('-xmlstart'):
if len(arg) == 9:
xmlStart = pywikibot.input(
u'Please enter the dumped article to start with:')
else:
xmlStart = arg[10:]
elif arg.startswith('-xml'):
if len(arg) == 4:
xmlFilename = i18n.input('pywikibot-enter-xml-filename')
else:
xmlFilename = arg[5:]
elif arg == '-sql':
useSql = True
elif arg.startswith('-excepttitle:'):
exceptions['title'].append(arg[13:])
elif arg.startswith('-requiretitle:'):
exceptions['require-title'].append(arg[14:])
elif arg.startswith('-excepttext:'):
exceptions['text-contains'].append(arg[12:])
elif arg.startswith('-exceptinside:'):
exceptions['inside'].append(arg[14:])
elif arg.startswith('-exceptinsidetag:'):
exceptions['inside-tags'].append(arg[17:])
elif arg.startswith('-fix:'):
fixes_set += [arg[5:]]
elif arg.startswith('-sleep:'):
sleep = float(arg[7:])
elif arg == '-always':
acceptall = True
elif arg == '-recursive':
recursive = True
elif arg == '-nocase':
caseInsensitive = True
elif arg == '-dotall':
dotall = True
elif arg == '-multiline':
multiline = True
elif arg.startswith('-addcat:'):
add_cat = arg[8:]
elif arg.startswith('-summary:'):
edit_summary = arg[9:]
elif arg.startswith('-automaticsummary'):
edit_summary = True
elif arg.startswith('-allowoverlap'):
allowoverlap = True
elif arg.startswith('-manualinput'):
manual_input = True
elif arg.startswith('-replacementfile'):
issue_deprecation_warning('-replacementfile', '-pairsfile', 2,
ArgumentDeprecationWarning)
elif arg.startswith('-pairsfile'):
if len(commandline_replacements) % 2:
replacement_file_arg_misplaced = True
if arg == '-pairsfile':
replacement_file = pywikibot.input(
u'Please enter the filename to read replacements from:')
else:
replacement_file = arg[len('-pairsfile:'):]
else:
commandline_replacements.append(arg)
site = pywikibot.Site()
if len(commandline_replacements) % 2:
pywikibot.error('Incomplete command line pattern replacement pair.')
return False
if replacement_file_arg_misplaced:
pywikibot.error('-pairsfile used between a pattern replacement pair.')
return False
if replacement_file:
try:
with codecs.open(replacement_file, 'r', 'utf-8') as f:
# strip newlines, but not other characters
file_replacements = f.read().splitlines()
except (IOError, OSError) as e:
pywikibot.error(u'Error loading {0}: {1}'.format(
replacement_file, e))
return False
if len(file_replacements) % 2:
pywikibot.error(
'{0} contains an incomplete pattern replacement pair.'.format(
replacement_file))
return False
# Strip BOM from first line
file_replacements[0].lstrip(u'\uFEFF')
commandline_replacements.extend(file_replacements)
if not (commandline_replacements or fixes_set) or manual_input:
old = pywikibot.input('Please enter the text that should be replaced:')
while old:
new = pywikibot.input(u'Please enter the new text:')
commandline_replacements += [old, new]
old = pywikibot.input(
'Please enter another text that should be replaced,'
'\nor press Enter to start:')
# The summary stored here won't be actually used but is only an example
single_summary = None
for i in range(0, len(commandline_replacements), 2):
replacement = Replacement(commandline_replacements[i],
commandline_replacements[i + 1])
if not single_summary:
single_summary = i18n.twtranslate(site, 'replace-replacing', {
'description':
' (-%s +%s)' % (replacement.old, replacement.new)
})
replacements.append(replacement)
# Perform one of the predefined actions.
missing_fixes_summaries = [] # which a fixes/replacements miss a summary
generators_given = bool(genFactory.gens)
for fix_name in fixes_set:
try:
fix = fixes.fixes[fix_name]
except KeyError:
pywikibot.output(u'Available predefined fixes are: %s' %
', '.join(fixes.fixes.keys()))
if not fixes.user_fixes_loaded:
pywikibot.output('The user fixes file could not be found: '
'{0}'.format(fixes.filename))
return
if not fix['replacements']:
pywikibot.warning('No replacements defined for fix '
'"{0}"'.format(fix_name))
continue
if "msg" in fix:
if isinstance(fix['msg'], basestring):
set_summary = i18n.twtranslate(site, str(fix['msg']))
else:
set_summary = i18n.translate(site, fix['msg'], fallback=True)
else:
set_summary = None
if not generators_given and 'generator' in fix:
gen_args = fix['generator']
if isinstance(gen_args, basestring):
gen_args = [gen_args]
for gen_arg in gen_args:
genFactory.handleArg(gen_arg)
replacement_set = ReplacementList(fix.get('regex'),
fix.get('exceptions'),
fix.get('nocase'),
set_summary,
name=fix_name)
# Whether some replacements have a summary, if so only show which
# have none, otherwise just mention the complete fix
missing_fix_summaries = []
for index, replacement in enumerate(fix['replacements'], start=1):
summary = None if len(replacement) < 3 else replacement[2]
if not set_summary and not summary:
missing_fix_summaries.append('"{0}" (replacement #{1})'.format(
fix_name, index))
if chars.contains_invisible(replacement[0]):
pywikibot.warning('The old string "{0}" contains formatting '
'characters like U+200E'.format(
chars.replace_invisible(replacement[0])))
if (not callable(replacement[1])
and chars.contains_invisible(replacement[1])):
pywikibot.warning('The new string "{0}" contains formatting '
'characters like U+200E'.format(
chars.replace_invisible(replacement[1])))
replacement_set.append(
ReplacementListEntry(
old=replacement[0],
new=replacement[1],
fix_set=replacement_set,
edit_summary=summary,
))
# Exceptions specified via 'fix' shall be merged to those via CLI.
if replacement_set:
replacements.extend(replacement_set)
if replacement_set._exceptions is not None:
for k, v in replacement_set._exceptions.items():
if k in exceptions:
exceptions[k] = list(set(exceptions[k]) | set(v))
else:
exceptions[k] = v
if len(fix['replacements']) == len(missing_fix_summaries):
missing_fixes_summaries.append(
'"{0}" (all replacements)'.format(fix_name))
else:
missing_fixes_summaries += missing_fix_summaries
if ((not edit_summary or edit_summary is True)
and (missing_fixes_summaries or single_summary)):
if single_summary:
pywikibot.output(u'The summary message for the command line '
'replacements will be something like: %s' %
single_summary)
if missing_fixes_summaries:
pywikibot.output('The summary will not be used when the fix has '
'one defined but the following fix(es) do(es) '
'not have a summary defined: '
'{0}'.format(', '.join(missing_fixes_summaries)))
if edit_summary is not True:
edit_summary = pywikibot.input(
'Press Enter to use this automatic message, or enter a '
'description of the\nchanges your bot will make:')
else:
edit_summary = ''
# Set the regular expression flags
flags = re.UNICODE
if caseInsensitive:
flags = flags | re.IGNORECASE
if dotall:
flags = flags | re.DOTALL
if multiline:
flags = flags | re.MULTILINE
# Pre-compile all regular expressions here to save time later
for replacement in replacements:
replacement.compile(regex, flags)
precompile_exceptions(exceptions, regex, flags)
if xmlFilename:
try:
xmlStart
except NameError:
xmlStart = None
gen = XmlDumpReplacePageGenerator(xmlFilename, xmlStart, replacements,
exceptions, site)
elif useSql:
whereClause = 'WHERE (%s)' % ' OR '.join([
"old_text RLIKE '%s'" % prepareRegexForMySQL(old_regexp.pattern)
for (old_regexp, new_text) in replacements
])
if exceptions:
exceptClause = 'AND NOT (%s)' % ' OR '.join([
"old_text RLIKE '%s'" % prepareRegexForMySQL(exc.pattern)
for exc in exceptions
])
else:
exceptClause = ''
query = u"""
SELECT page_namespace, page_title
FROM page
JOIN text ON (page_id = old_id)
%s
%s
LIMIT 200""" % (whereClause, exceptClause)
gen = pagegenerators.MySQLPageGenerator(query)
gen = genFactory.getCombinedGenerator(gen, preload=True)
if not gen:
pywikibot.bot.suggest_help(missing_generator=True)
return False
bot = ReplaceRobot(gen,
replacements,
exceptions,
allowoverlap,
recursive,
add_cat,
sleep,
edit_summary,
always=acceptall,
site=site)
site.login()
bot.run()
# Explicitly call pywikibot.stopme(). It will make sure the callback is
# triggered before replace.py is unloaded.
pywikibot.stopme()
pywikibot.output(u'\n%s pages changed.' % bot.changed_pages)
def main(*args):
"""
Process command line arguments and perform task.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
# Loading the comments
global categoryToCheck, project_inserted
# always, define a generator to understand if the user sets one,
# defining what's genFactory
always = False
generator = False
show = False
moveBlockCheck = False
protectedpages = False
protectType = 'edit'
namespace = 0
# To prevent Infinite loops
errorCount = 0
# Process global args and prepare generator args parser
local_args = pywikibot.handle_args(args)
genFactory = pagegenerators.GeneratorFactory()
# Process local args
for arg in local_args:
option, sep, value = arg.partition(':')
if option == '-always':
always = True
elif option == '-move':
moveBlockCheck = True
elif option == '-show':
show = True
elif option in ('-protectedpages', '-moveprotected'):
protectedpages = True
if option == '-moveprotected':
protectType = 'move'
if value:
namespace = int(value)
else:
genFactory.handleArg(arg)
if config.mylang not in project_inserted:
pywikibot.output(u"Your project is not supported by this script.\n"
u"You have to edit the script and add it!")
return
site = pywikibot.Site()
if protectedpages:
generator = site.protectedpages(namespace=namespace, type=protectType)
# Take the right templates to use, the category and the comment
TSP = i18n.translate(site, templateSemiProtection)
TTP = i18n.translate(site, templateTotalProtection)
TSMP = i18n.translate(site, templateSemiMoveProtection)
TTMP = i18n.translate(site, templateTotalMoveProtection)
TNR = i18n.translate(site, templateNoRegex)
TU = i18n.translate(site, templateUnique)
categories = i18n.translate(site, categoryToCheck)
commentUsed = i18n.twtranslate(site, 'blockpageschecker-summary')
if not generator:
generator = genFactory.getCombinedGenerator()
if not generator:
generator = list()
pywikibot.output(u'Loading categories...')
# Define the category if no other generator has been setted
for CAT in categories:
cat = pywikibot.Category(site, CAT)
# Define the generator
gen = pagegenerators.CategorizedPageGenerator(cat)
for pageCat in gen:
generator.append(pageCat)
pywikibot.output(u'Categories loaded, start!')
# Main Loop
preloadingGen = pagegenerators.PreloadingGenerator(generator, groupsize=60)
for page in preloadingGen:
pagename = page.title(asLink=True)
pywikibot.output('Loading %s...' % pagename)
try:
text = page.text
except pywikibot.NoPage:
pywikibot.output("%s doesn't exist! Skipping..." % pagename)
continue
except pywikibot.IsRedirectPage:
pywikibot.output("%s is a redirect! Skipping..." % pagename)
if show:
showQuest(page)
continue
# FIXME: This check does not work :
# PreloadingGenerator cannot set correctly page.editRestriction
# (see bug T57322)
# if not page.canBeEdited():
# pywikibot.output("%s is sysop-protected : this account can't edit "
# "it! Skipping..." % pagename)
# continue
restrictions = page.protection()
try:
editRestr = restrictions['edit']
except KeyError:
editRestr = None
if not page.canBeEdited():
pywikibot.output(u"%s is protected: "
u"this account can't edit it! Skipping..." %
pagename)
continue
# Understand, according to the template in the page, what should be the
# protection and compare it with what there really is.
TemplateInThePage = understandBlock(text, TTP, TSP, TSMP, TTMP, TU)
# Only to see if the text is the same or not...
oldtext = text
# keep track of the changes for each step (edit then move)
changes = -1
if not editRestr:
# page is not edit-protected
# Deleting the template because the page doesn't need it.
if not (TTP or TSP):
raise pywikibot.Error(
'This script is not localized to use it on \n{0}. '
'Missing "templateSemiProtection" or'
'"templateTotalProtection"'.format(site.sitename))
if TU:
replaceToPerform = u'|'.join(TTP + TSP + TU)
else:
replaceToPerform = u'|'.join(TTP + TSP)
text, changes = re.subn(
'<noinclude>(%s)</noinclude>' % replaceToPerform, '', text)
if changes == 0:
text, changes = re.subn('(%s)' % replaceToPerform, '', text)
msg = u'The page is editable for all'
if not moveBlockCheck:
msg += u', deleting the template..'
pywikibot.output(u'%s.' % msg)
elif editRestr[0] == 'sysop':
# total edit protection
if (TemplateInThePage[0] == 'sysop-total' and TTP) or \
(TemplateInThePage[0] == 'unique' and TU):
msg = 'The page is protected to the sysop'
if not moveBlockCheck:
msg += ', skipping...'
pywikibot.output(msg)
else:
if not TNR or TU and not TNR[4] or not (TU or TNR[1]):
raise pywikibot.Error(
'This script is not localized to use it on \n{0}. '
'Missing "templateNoRegex"'.format(site.sitename))
pywikibot.output(
u'The page is protected to the sysop, but the '
u'template seems not correct. Fixing...')
if TU:
text, changes = re.subn(TemplateInThePage[1], TNR[4], text)
else:
text, changes = re.subn(TemplateInThePage[1], TNR[1], text)
elif TSP or TU:
# implicitely editRestr[0] = 'autoconfirmed', edit-Semi-protection
if TemplateInThePage[0] == 'autoconfirmed-total' or \
TemplateInThePage[0] == 'unique':
msg = 'The page is editable only for the autoconfirmed users'
if not moveBlockCheck:
msg += ', skipping...'
pywikibot.output(msg)
else:
if not TNR or TU and not TNR[4] or not (TU or TNR[1]):
raise pywikibot.Error(
'This script is not localized to use it on \n{0}. '
'Missing "templateNoRegex"'.format(site.sitename))
pywikibot.output(u'The page is editable only for the '
u'autoconfirmed users, but the template '
u'seems not correct. Fixing...')
if TU:
text, changes = re.subn(TemplateInThePage[1], TNR[4], text)
else:
text, changes = re.subn(TemplateInThePage[1], TNR[0], text)
if changes == 0:
# We tried to fix edit-protection templates, but it did not work.
pywikibot.warning('No edit-protection template could be found')
if moveBlockCheck and changes > -1:
# checking move protection now
try:
moveRestr = restrictions['move']
except KeyError:
moveRestr = False
changes = -1
if not moveRestr:
pywikibot.output(u'The page is movable for all, deleting the '
u'template...')
# Deleting the template because the page doesn't need it.
if TU:
replaceToPerform = u'|'.join(TSMP + TTMP + TU)
else:
replaceToPerform = u'|'.join(TSMP + TTMP)
text, changes = re.subn(
'<noinclude>(%s)</noinclude>' % replaceToPerform, '', text)
if changes == 0:
text, changes = re.subn('(%s)' % replaceToPerform, '',
text)
elif moveRestr[0] == 'sysop':
# move-total-protection
if (TemplateInThePage[0] == 'sysop-move' and TTMP) or \
(TemplateInThePage[0] == 'unique' and TU):
pywikibot.output(u'The page is protected from moving to '
u'the sysop, skipping...')
if TU:
# no changes needed, better to revert the old text.
text = oldtext
else:
pywikibot.output(u'The page is protected from moving to '
u'the sysop, but the template seems not '
u'correct. Fixing...')
if TU:
text, changes = re.subn(TemplateInThePage[1], TNR[4],
text)
else:
text, changes = re.subn(TemplateInThePage[1], TNR[3],
text)
elif TSMP or TU:
# implicitely moveRestr[0] = 'autoconfirmed',
# move-semi-protection
if TemplateInThePage[0] == 'autoconfirmed-move' or \
TemplateInThePage[0] == 'unique':
pywikibot.output(u'The page is movable only for the '
u'autoconfirmed users, skipping...')
if TU:
# no changes needed, better to revert the old text.
text = oldtext
else:
pywikibot.output(u'The page is movable only for the '
u'autoconfirmed users, but the template '
u'seems not correct. Fixing...')
if TU:
text, changes = re.subn(TemplateInThePage[1], TNR[4],
text)
else:
text, changes = re.subn(TemplateInThePage[1], TNR[2],
text)
if changes == 0:
# We tried to fix move-protection templates, but it did not work
pywikibot.warning('No move-protection template could be found')
if oldtext != text:
# Ok, asking if the change has to be performed and do it if yes.
pywikibot.output(
color_format('\n\n>>> {lightpurple}{0}{default} <<<',
page.title()))
pywikibot.showDiff(oldtext, text)
if not always:
choice = pywikibot.input_choice(
u'Do you want to accept these '
u'changes?', [('Yes', 'y'), ('No', 'n'), ('All', 'a')],
'n')
if choice == 'a':
always = True
if always or choice == 'y':
while True:
try:
page.put(text, commentUsed, force=True)
except pywikibot.EditConflict:
pywikibot.output(u'Edit conflict! skip!')
break
except pywikibot.ServerError:
# Sometimes there is this error that's quite annoying
# because can block the whole process for nothing.
errorCount += 1
if errorCount < 5:
pywikibot.output(u'Server Error! Wait..')
time.sleep(3)
continue
else:
# Prevent Infinite Loops
raise pywikibot.ServerError(u'Fifth Server Error!')
except pywikibot.SpamfilterError as e:
pywikibot.output(u'Cannot change %s because of '
u'blacklist entry %s' %
(page.title(), e.url))
break
except pywikibot.LockedPage:
pywikibot.output(u'The page is still protected. '
u'Skipping...')
break
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Error putting page: %s' %
(error.args, ))
break
else:
# Break only if the errors are one after the other
errorCount = 0
break
checkbroken = False
elif arg.startswith('-exist'):
checkbroken = False
elif arg.startswith('-keepparent'):
removeparent = False
elif arg.startswith('-all'):
main = False
elif not workingcatname:
workingcatname = arg
if not workingcatname:
pywikibot.showHelp()
sys.exit(0)
mysite = pywikibot.Site()
pywikibot.setAction(i18n.twtranslate(mysite, 'makecat-create', {'cat': workingcatname}))
workingcat = pywikibot.Category(mysite,
u'%s:%s'
% (mysite.category_namespace(),
workingcatname))
filename = pywikibot.config.datafilepath('category',
workingcatname.encode('ascii', 'xmlcharrefreplace') + '_exclude.txt')
try:
f = codecs.open(filename, 'r', encoding=mysite.encoding())
for line in f.readlines():
# remove trailing newlines and carriage returns
try:
while line[-1] in ['\n', '\r']:
line = line[:-1]
except IndexError:
pass
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
options = {}
message_properties = {}
generator = None
protections = {}
default_level = 'sysop'
default_summaries = {
'cat': 'category',
'links': 'links',
'ref': 'ref',
'imageused': 'images',
'file': 'simple',
}
# read command line parameters
local_args = pywikibot.handle_args(args)
genFactory = pagegenerators.GeneratorFactory()
site = pywikibot.Site()
generator_type = None
protection_levels = set(site.protection_levels())
protection_types = site.protection_types()
if '' in protection_levels:
protection_levels.add('all')
for arg in local_args:
if arg == '-always':
options['always'] = True
elif arg.startswith('-summary'):
if len(arg) == len('-summary'):
# fill dummy value to prevent automatic generation
options['summary'] = None
else:
options['summary'] = arg[len('-summary:'):]
elif arg.startswith('-images'):
pywikibot.output(color_format(
'\n{lightred}-image option is deprecated. '
'Please use -imagelinks instead.{default}\n'))
local_args.append('-imagelinks' + arg[7:])
elif arg.startswith('-unprotect'):
default_level = 'all'
elif arg.startswith('-default'):
if len(arg) == len('-default'):
default_level = 'sysop'
else:
default_level = arg[len('-default:'):]
else:
is_p_type = False
if arg.startswith('-'):
delimiter = arg.find(':')
if delimiter > 0:
p_type_arg = arg[1:delimiter]
level = arg[delimiter + 1:]
if p_type_arg in protection_types:
protections[p_type_arg] = level
is_p_type = True
if not is_p_type:
if not genFactory.handleArg(arg):
raise ValueError('Unknown parameter "{0}"'.format(arg))
found = arg.find(':')
if found:
message_properties.update({'cat': arg[found + 1:],
'page': arg[found + 1:]})
if 'summary' not in options:
generator_type = arg[1:found] if found > 0 else arg[1:]
if generator_type in default_summaries:
message_type = default_summaries[generator_type]
if message_type == 'simple' or message_properties:
if default_level == 'all':
options['summary'] = i18n.twtranslate(
site, 'unprotect-{0}'.format(message_type),
message_properties)
else:
options['summary'] = i18n.twtranslate(
site, 'protect-{0}'.format(message_type),
message_properties)
generator = genFactory.getCombinedGenerator()
# We are just protecting pages, so we have no need of using a preloading
# page generator to actually get the text of those pages.
if generator:
if default_level:
default_level = check_protection_level('Default level',
default_level,
protection_levels)
# set the default value for all
# None (not the string 'none') will be ignored by Site.protect()
combined_protections = dict(
(p_type, default_level) for p_type in protection_types)
for p_type, level in protections.items():
level = check_protection_level(p_type, level, protection_levels,
default_level)
# '' is equivalent to 'all'
if level == 'none' or level == '':
level = 'all'
combined_protections[p_type] = level
if not options.get('summary'):
options['summary'] = pywikibot.input(
u'Enter a reason for the protection change:')
bot = ProtectionRobot(generator, combined_protections, site, **options)
bot.run()
return True
else:
pywikibot.bot.suggest_help(missing_generator=True)
return False
def reportBadAccount(self, name=None, final=False):
"""Report bad account."""
# Queue process
if name:
if globalvar.confirm:
answer = pywikibot.input_choice(
u'%s may have an unwanted username, do you want to report '
u'this user?' % name,
[('Yes', 'y'), ('No', 'n'), ('All', 'a')], 'n',
automatic_quit=False)
if answer in ['a', 'all']:
answer = 'y'
globalvar.confirm = False
else:
answer = 'y'
if answer.lower() in ['yes', 'y'] or not globalvar.confirm:
showStatus()
pywikibot.output(
u'%s is possibly an unwanted username. It will be reported.'
% name)
if hasattr(self, '_BAQueue'):
self._BAQueue.append(name)
else:
self._BAQueue = [name]
if len(self._BAQueue) >= globalvar.dumpToLog or final:
rep_text = ''
# name in queue is max, put detail to report page
pywikibot.output("Updating badname accounts to report page...")
rep_page = pywikibot.Page(self.site,
i18n.translate(self.site,
report_page))
if rep_page.exists():
text_get = rep_page.get()
else:
text_get = ('This is a report page for the Bad-username, '
'please translate me. --~~~')
pos = 0
# The talk page includes "_" between the two names, in this way
# replace them to " ".
for usrna in self._BAQueue:
username = pywikibot.url2link(usrna, self.site, self.site)
n = re.compile(re.escape(username), re.UNICODE)
y = n.search(text_get, pos)
if y:
pywikibot.output(u'%s is already in the report page.'
% username)
else:
# Adding the log.
rep_text += i18n.translate(self.site,
report_text) % username
if self.site.code == 'it':
rep_text = "%s%s}}" % (rep_text, self.bname[username])
com = i18n.twtranslate(self.site, 'welcome-bad_username')
if rep_text != '':
rep_page.put(text_get + rep_text, summary=com, force=True,
minorEdit=True)
showStatus(5)
pywikibot.output(u'Reported')
self.BAQueue = list()
else:
return True
def __init__(self, generator, templates, **kwargs):
"""
Initializer.
@param generator: the pages to work on
@type generator: iterable
@param templates: a dictionary which maps old template names to
their replacements. If remove or subst is True, it maps the
names of the templates that should be removed/resolved to None.
@type templates: dict
"""
self.availableOptions.update({
'addcat': None,
'remove': False,
'subst': False,
'summary': None,
})
SingleSiteBot.__init__(self, generator=generator, **kwargs)
self.templates = templates
# get edit summary message if it's empty
if not self.getOption('summary'):
comma = self.site.mediawiki_message('comma-separator')
params = {'list': comma.join(self.templates.keys()),
'num': len(self.templates)}
if self.getOption('remove'):
self.options['summary'] = i18n.twtranslate(
self.site, 'template-removing', params)
elif self.getOption('subst'):
self.options['summary'] = i18n.twtranslate(
self.site, 'template-substituting', params)
else:
self.options['summary'] = i18n.twtranslate(
self.site, 'template-changing', params)
replacements = []
exceptions = {}
builder = textlib._MultiTemplateMatchBuilder(self.site)
for old, new in self.templates.items():
template_regex = builder.pattern(old)
if self.getOption('subst') and self.getOption('remove'):
replacements.append((template_regex,
r'{{subst:%s\g<parameters>}}' % new))
exceptions['inside-tags'] = ['ref', 'gallery', 'poem',
'pagelist', ]
elif self.getOption('subst'):
replacements.append(
(template_regex, r'{{%s:%s\g<parameters>}}' %
(self.getOption('subst'), old)))
exceptions['inside-tags'] = ['ref', 'gallery', 'poem',
'pagelist', ]
elif self.getOption('remove'):
separate_line_regex = re.compile(
r'^[*#:]* *{0} *\n'.format(template_regex.pattern),
re.DOTALL | re.MULTILINE)
replacements.append((separate_line_regex, ''))
spaced_regex = re.compile(
r' +{0} +'.format(template_regex.pattern),
re.DOTALL)
replacements.append((spaced_regex, ' '))
replacements.append((template_regex, ''))
else:
template = pywikibot.Page(self.site, new, ns=10)
if not template.exists():
pywikibot.warning('Template "{0}" does not exist.'
.format(new))
if not pywikibot.input_yn('Do you want to proceed anyway?',
default=False,
automatic_quit=False):
continue
replacements.append((template_regex,
r'{{%s\g<parameters>}}' % new))
super(TemplateRobot, self).__init__(
generator, replacements, exceptions,
always=self.getOption('always'),
addcat=self.getOption('addcat'),
summary=self.getOption('summary'))
def delete_1_broken_redirect(self, redir_name):
"""Treat one broken redirect."""
if isinstance(redir_name, basestring):
redir_page = pywikibot.Page(self.site, redir_name)
else:
redir_page = redir_name
# Show the title of the page we're working on.
# Highlight the title in purple.
done = not self.getOption('delete')
pywikibot.output(
color_format('\n\n>>> {lightpurple}{0}{default} <<<',
redir_page.title()))
try:
targetPage = redir_page.getRedirectTarget()
except pywikibot.IsNotRedirectPage:
pywikibot.output(u'%s is not a redirect.' % redir_page.title())
except pywikibot.CircularRedirect:
pywikibot.output('%s is a circular redirect.' % redir_page.title())
except pywikibot.NoPage:
pywikibot.output(u'%s doesn\'t exist.' % redir_page.title())
except pywikibot.InvalidTitle:
pywikibot.exception()
except pywikibot.InterwikiRedirectPage:
pywikibot.output('%s is on another site.' % redir_page.title())
else:
try:
targetPage.get()
except pywikibot.BadTitle as e:
pywikibot.warning(
u'Redirect target %s is not a valid page title.' %
str(e)[10:])
pass
except pywikibot.InvalidTitle:
pywikibot.exception()
pass
except pywikibot.NoPage:
movedTarget = None
try:
movedTarget = targetPage.moved_target()
except pywikibot.NoMoveTarget:
pass
if movedTarget:
if not movedTarget.exists():
# FIXME: Test to another move
pywikibot.output(u'Target page %s does not exist' %
(movedTarget))
elif redir_page == movedTarget:
pywikibot.output(
'Redirect to target page forms a redirect loop')
else:
pywikibot.output(u'%s has been moved to %s' %
(redir_page, movedTarget))
reason = i18n.twtranslate(
self.site, 'redirect-fix-broken-moved',
{'to': movedTarget.title(asLink=True)})
content = redir_page.get(get_redirect=True)
redir_page.set_redirect_target(movedTarget,
keep_section=True,
save=False)
pywikibot.showDiff(content, redir_page.text)
pywikibot.output(u'Summary - %s' % reason)
if self.user_confirm(
u'Redirect target %s has been moved to %s.\n'
u'Do you want to fix %s?' %
(targetPage, movedTarget, redir_page)):
try:
redir_page.save(reason)
except pywikibot.NoUsername:
pywikibot.output(
u"Page [[%s]] not saved; "
u"sysop privileges required." %
redir_page.title())
except pywikibot.LockedPage:
pywikibot.output(u'%s is locked.' %
redir_page.title())
except pywikibot.PageSaveRelatedError:
pywikibot.exception()
else:
done = True
if not done and self.user_confirm(
u'Redirect target %s does not exist.\n'
u'Do you want to delete %s?' %
(targetPage.title(asLink=True),
redir_page.title(asLink=True))):
self.delete_redirect(redir_page, 'redirect-remove-broken')
elif not (self.getOption('delete') or movedTarget):
pywikibot.output(
'Cannot fix or delete the broken redirect')
except pywikibot.IsRedirectPage:
pywikibot.output(
"Redirect target {0} is also a redirect! {1}".format(
targetPage.title(asLink=True), "Won't delete anything."
if self.getOption('delete') else "Skipping."))
else:
# we successfully get the target page, meaning that
# it exists and is not a redirect: no reason to touch it.
pywikibot.output("Redirect target {0} does exist! {1}".format(
targetPage.title(asLink=True), "Won't delete anything."
if self.getOption('delete') else "Skipping."))
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
pageName = ''
summary = None
generator = None
options = {}
# read command line parameters
local_args = pywikibot.handle_args(args)
genFactory = pagegenerators.GeneratorFactory()
mysite = pywikibot.Site()
for arg in local_args:
if arg == '-always':
options['always'] = True
elif arg.startswith('-summary'):
if len(arg) == len('-summary'):
summary = pywikibot.input(u'Enter a reason for the deletion:')
else:
summary = arg[len('-summary:'):]
elif arg.startswith('-images'):
warn('-image option is deprecated. Please use -imageused instead.',
exceptions.ArgumentDeprecationWarning)
local_args.append('-imageused' + arg[7:])
elif arg.startswith('-undelete'):
options['undelete'] = True
elif arg.startswith('-isorphan'):
options['isorphan'] = int(arg[10:]) if arg[10:] != '' else 0
if options['isorphan'] < 0:
options['isorphan'] = False
elif arg.startswith('-orphansonly'):
if arg[13:]:
namespaces = mysite.namespaces.resolve(arg[13:].split(","))
else:
namespaces = mysite.namespaces
options['orphansonly'] = namespaces
else:
genFactory.handleArg(arg)
found = arg.find(':') + 1
if found:
pageName = arg[found:]
if not summary:
un = 'un' if 'undelete' in options else ''
if pageName:
if arg.startswith('-cat') or arg.startswith('-subcats'):
summary = i18n.twtranslate(mysite, 'delete-from-category',
{'page': pageName})
elif arg.startswith('-links'):
summary = i18n.twtranslate(mysite,
un + 'delete-linked-pages',
{'page': pageName})
elif arg.startswith('-ref'):
summary = i18n.twtranslate(mysite, 'delete-referring-pages',
{'page': pageName})
elif arg.startswith('-imageused'):
summary = i18n.twtranslate(mysite, un + 'delete-images',
{'page': pageName})
elif arg.startswith('-file'):
summary = i18n.twtranslate(mysite, un + 'delete-from-file')
generator = genFactory.getCombinedGenerator()
# We are just deleting pages, so we have no need of using a preloading
# page generator to actually get the text of those pages.
if generator:
if summary is None:
summary = pywikibot.input(u'Enter a reason for the %sdeletion:'
% ['', 'un'][options.get('undelete',
False)])
bot = DeletionRobot(generator, summary, **options)
bot.run()
return True
else:
pywikibot.bot.suggest_help(missing_generator=True)
return False
def treat_page(self) -> None:
"""Upload page content."""
page = self.current_page
title = page.title()
# save the content retrieved from generator
contents = page.text
# delete page's text to get it from live wiki
del page.text
if self.opt.summary:
comment = self.opt.summary
else:
comment = i18n.twtranslate(self.site, 'pagefromfile-msg')
comment_top = comment + ' - ' + i18n.twtranslate(
self.site, 'pagefromfile-msg_top')
comment_bottom = comment + ' - ' + i18n.twtranslate(
self.site, 'pagefromfile-msg_bottom')
comment_force = '{} *** {} ***'.format(
comment, i18n.twtranslate(self.site, 'pagefromfile-msg_force'))
if page.exists():
if not self.opt.redirect and page.isRedirectPage():
pywikibot.output(
'Page {} is redirect, skipping!'.format(title))
return
pagecontents = page.text
nocontent = self.opt.nocontent
if (nocontent and (nocontent in pagecontents
or nocontent.lower() in pagecontents)):
pywikibot.output(
'Page has {} so it is skipped'.format(nocontent))
return
if self.opt.append:
separator = self.opt.append[1]
if separator == r'\n':
separator = '\n'
if self.opt.append[0] == 'top':
above, below = contents, pagecontents
comment = comment_top
else:
above, below = pagecontents, contents
comment = comment_bottom
pywikibot.output(
'Page {} already exists, appending on {}!'.format(
title, self.opt.append[0]))
contents = above + separator + below
elif self.opt.force:
pywikibot.output(
'Page {} already exists, ***overwriting!'.format(title))
comment = comment_force
else:
pywikibot.output(
'Page {} already exists, not adding!'.format(title))
return
else:
if self.opt.autosummary:
comment = config.default_edit_summary = ''
self.put_current(contents,
summary=comment,
minor=self.opt.minor,
show_diff=self.opt.showdiff)
def run(self):
"""Run thread."""
while not self.killed:
if len(self.queue) == 0:
if self.finishing:
break
else:
time.sleep(0.1)
else:
with self.semaphore:
url, errorReport, containingPage, archiveURL = \
self.queue[0]
self.queue = self.queue[1:]
talkPage = containingPage.toggleTalkPage()
pywikibot.output(color_format(
'{lightaqua}** Reporting dead link on '
'{0}...{default}',
talkPage.title(asLink=True)))
try:
content = talkPage.get() + '\n\n\n'
if url in content:
pywikibot.output(color_format(
'{lightaqua}** Dead link seems to have '
'already been reported on {0}{default}',
talkPage.title(asLink=True)))
continue
except (pywikibot.NoPage, pywikibot.IsRedirectPage):
content = ''
if archiveURL:
archiveMsg = '\n' + \
i18n.twtranslate(
containingPage.site,
'weblinkchecker-archive_msg',
{'URL': archiveURL})
else:
archiveMsg = ''
# The caption will default to "Dead link". But if there
# is already such a caption, we'll use "Dead link 2",
# "Dead link 3", etc.
caption = i18n.twtranslate(containingPage.site,
'weblinkchecker-caption')
i = 1
count = ''
# Check if there is already such a caption on
# the talk page.
while re.search('= *%s%s *=' % (caption, count),
content) is not None:
i += 1
count = ' ' + str(i)
caption += count
content += '== %s ==\n\n%s\n\n%s%s\n--~~~~' % \
(caption,
i18n.twtranslate(containingPage.site,
'weblinkchecker-report'),
errorReport,
archiveMsg)
comment = '[[%s#%s|→]] %s' % \
(talkPage.title(), caption,
i18n.twtranslate(containingPage.site,
'weblinkchecker-summary'))
try:
talkPage.put(content, comment)
except pywikibot.SpamfilterError as error:
pywikibot.output(color_format(
'{lightaqua}** SpamfilterError while trying to '
'change {0}: {1}{default}',
talkPage.title(asLink=True), error.url))
def fix_1_double_redirect(self, redir_name):
"""Treat one double redirect."""
if isinstance(redir_name, basestring):
redir = pywikibot.Page(self.site, redir_name)
else:
redir = redir_name
# Show the title of the page we're working on.
# Highlight the title in purple.
pywikibot.output(
color_format('\n\n>>> {lightpurple}{0}{default} <<<',
redir.title()))
newRedir = redir
redirList = [] # bookkeeping to detect loops
while True:
redirList.append(
u'%s:%s' %
(newRedir.site.lang, newRedir.title(withSection=False)))
try:
targetPage = newRedir.getRedirectTarget()
except pywikibot.IsNotRedirectPage:
if len(redirList) == 1:
pywikibot.output(u'Skipping: Page %s is not a redirect.' %
redir.title(asLink=True))
break # do nothing
elif len(redirList) == 2:
pywikibot.output(
u'Skipping: Redirect target %s is not a redirect.' %
newRedir.title(asLink=True))
break # do nothing
else:
pass # target found
except pywikibot.SectionError:
pywikibot.warning(
u"Redirect target section %s doesn't exist." %
newRedir.title(asLink=True))
except (pywikibot.CircularRedirect,
pywikibot.InterwikiRedirectPage) as e:
pywikibot.exception(e)
pywikibot.output(u"Skipping %s." % newRedir)
break
except pywikibot.BadTitle as e:
# str(e) is in the format 'BadTitle: [[Foo]]'
pywikibot.warning(
u'Redirect target %s is not a valid page title.' %
str(e)[10:])
break
except pywikibot.NoPage:
if len(redirList) == 1:
pywikibot.output(u'Skipping: Page %s does not exist.' %
redir.title(asLink=True))
break
else:
if self.getOption('always'):
pywikibot.output(
u"Skipping: Redirect target %s doesn't exist." %
newRedir.title(asLink=True))
break # skip if automatic
else:
pywikibot.warning(
u"Redirect target %s doesn't exist." %
newRedir.title(asLink=True))
except pywikibot.ServerError:
pywikibot.output(u'Skipping due to server error: '
u'No textarea found')
break
else:
pywikibot.output(u' Links to: %s.' %
targetPage.title(asLink=True))
try:
mw_msg = targetPage.site.mediawiki_message(
'wikieditor-toolbar-tool-redirect-example')
except KeyError:
pass
else:
if targetPage.title() == mw_msg:
pywikibot.output(
u"Skipping toolbar example: Redirect source is "
u"potentially vandalized.")
break
# watch out for redirect loops
if redirList.count(u'%s:%s' %
(targetPage.site.lang,
targetPage.title(withSection=False))):
pywikibot.warning(
u'Redirect target %s forms a redirect loop.' %
targetPage.title(asLink=True))
break # FIXME: doesn't work. edits twice!
if self.getOption('delete'):
# Delete the two redirects
# TODO: Check whether pages aren't vandalized
# and (maybe) do not have a version history
self.delete_redirect(targetPage,
'redirect-remove-loop')
self.delete_redirect(redir, 'redirect-remove-loop')
break
else: # redirect target found
if targetPage.isStaticRedirect():
pywikibot.output(
u" Redirect target is STATICREDIRECT.")
pass
else:
newRedir = targetPage
continue
try:
oldText = redir.get(get_redirect=True)
except pywikibot.BadTitle:
pywikibot.output(u"Bad Title Error")
break
if self.is_repo and redir.namespace() == self.repo.item_namespace:
redir = pywikibot.ItemPage(self.repo, redir.title())
targetPage = pywikibot.ItemPage(self.repo, targetPage.title())
pywikibot.output('Fixing double item redirect')
redir.set_redirect_target(targetPage)
break
redir.set_redirect_target(targetPage,
keep_section=True,
save=False)
summary = i18n.twtranslate(self.site, 'redirect-fix-double',
{'to': targetPage.title(asLink=True)})
pywikibot.showDiff(oldText, redir.text)
if self.user_confirm(u'Do you want to accept the changes?'):
try:
redir.save(summary)
except pywikibot.LockedPage:
pywikibot.output(u'%s is locked.' % redir.title())
except pywikibot.SpamfilterError as error:
pywikibot.output(
u"Saving page [[%s]] prevented by spam filter: %s" %
(redir.title(), error.url))
except pywikibot.PageNotSaved as error:
pywikibot.output(u"Saving page [[%s]] failed: %s" %
(redir.title(), error))
except pywikibot.NoUsername:
pywikibot.output(
u"Page [[%s]] not saved; sysop privileges required." %
redir.title())
except pywikibot.Error as error:
pywikibot.output(
u"Unexpected error occurred trying to save [[%s]]: %s"
% (redir.title(), error))
break
def run(self):
"""Run the bot."""
while True:
welcomed_count = 0
for users in self.parseNewUserLog():
if users.isBlocked():
showStatus(3)
pywikibot.output(u'%s has been blocked!' % users.name())
continue
if 'bot' in users.groups():
showStatus(3)
pywikibot.output(u'%s is a bot!' % users.name())
continue
if 'bot' in users.name().lower():
showStatus(3)
pywikibot.output(u'%s might be a global bot!'
% users.name())
continue
if users.editCount() >= globalvar.attachEditCount:
showStatus(2)
pywikibot.output(u'%s has enough edits to be welcomed.'
% users.name())
ustp = users.getUserTalkPage()
if ustp.exists():
showStatus(3)
pywikibot.output(u'%s has been already welcomed.'
% users.name())
continue
else:
if self.badNameFilter(users.name()):
self.reportBadAccount(users.name())
continue
welcome_text = i18n.translate(self.site, netext)
if globalvar.randomSign:
if self.site.family.name != 'wikinews':
welcome_text = (welcome_text
% choice(self.defineSign()))
if self.site.family.name == 'wiktionary' and \
self.site.code == 'it':
pass
else:
welcome_text += timeselected
elif (self.site.family.name != 'wikinews' and
self.site.code != 'it'):
welcome_text = (welcome_text
% globalvar.defaultSign)
final_text = i18n.translate(
self.site, final_new_text_additions)
if final_text:
welcome_text += final_text
welcome_comment = i18n.twtranslate(self.site,
'welcome-welcome')
try:
# append welcomed, welcome_count++
ustp.put(welcome_text, welcome_comment,
minorEdit=False)
welcomed_count += 1
self._totallyCount += 1
self.welcomed_users.append(users)
except pywikibot.EditConflict:
showStatus(4)
pywikibot.output(u'An edit conflict has occurred, '
u'skipping this user.')
if globalvar.makeWelcomeLog and \
i18n.translate(self.site, logbook):
showStatus(5)
if welcomed_count == 1:
pywikibot.output(u'One user has been welcomed.')
elif welcomed_count == 0:
pywikibot.output(u'No users have been welcomed.')
else:
pywikibot.output(u'%s users have been welcomed.'
% welcomed_count)
if welcomed_count >= globalvar.dumpToLog:
if self.makelogpage(self.welcomed_users):
self.welcomed_users = list()
welcomed_count = 0
else:
continue
# If we haven't to report, do nothing.
else:
if users.editCount() == 0:
if not globalvar.quiet:
showStatus(1)
pywikibot.output(u'%s has no contributions.'
% users.name())
else:
showStatus(1)
pywikibot.output(u'%s has only %d contributions.'
% (users.name(), users.editCount()))
# That user mustn't be welcomed.
continue
if globalvar.makeWelcomeLog and i18n.translate(
self.site, logbook) and welcomed_count > 0:
showStatus()
if welcomed_count == 1:
pywikibot.output(u'Putting the log of the latest user...')
else:
pywikibot.output(
u'Putting the log of the latest %d users...'
% welcomed_count)
if self.makelogpage(self.welcomed_users):
self.welcomed_users = list()
else:
continue
self.welcomed_users = list()
if hasattr(self, '_BAQueue'):
showStatus()
pywikibot.output("Putting bad name to report page....")
self.reportBadAccount(None, final=True)
try:
if globalvar.recursive:
showStatus()
if locale.getlocale()[1]:
strfstr = unicode(
time.strftime(u"%d %b %Y %H:%M:%S (UTC)",
time.gmtime()),
locale.getlocale()[1])
else:
strfstr = time.strftime(
u"%d %b %Y %H:%M:%S (UTC)", time.gmtime())
pywikibot.output(u'Sleeping %d seconds before rerun. %s'
% (globalvar.timeRecur, strfstr))
pywikibot.stopme()
time.sleep(globalvar.timeRecur)
else:
raise KeyboardInterrupt
except KeyboardInterrupt:
break
def run(self):
while not self.killed:
if len(self.queue) == 0:
if self.finishing:
break
else:
time.sleep(0.1)
else:
self.semaphore.acquire()
(url, errorReport, containingPage, archiveURL) = self.queue[0]
self.queue = self.queue[1:]
talkPage = containingPage.toggleTalkPage()
pywikibot.output(
u'\03{lightaqua}** Reporting dead link on %s...\03{default}'
% talkPage.title(asLink=True))
try:
content = talkPage.get() + "\n\n"
if url in content:
pywikibot.output(
u'\03{lightaqua}** Dead link seems to have already '
u'been reported on %s\03{default}'
% talkPage.title(asLink=True))
self.semaphore.release()
continue
except (pywikibot.NoPage, pywikibot.IsRedirectPage):
content = u''
if archiveURL:
archiveMsg = u'\n' + \
i18n.twtranslate(pywikibot.getSite(),
'weblinkchecker-archive_msg',
{'URL': archiveURL})
else:
archiveMsg = u''
# The caption will default to "Dead link". But if there is
# already such a caption, we'll use "Dead link 2",
# "Dead link 3", etc.
caption = i18n.twtranslate(pywikibot.getSite(),
'weblinkchecker-caption')
i = 1
count = u''
# Check if there is already such a caption on the talk page.
while re.search('= *%s%s *=' % (caption, count),
content) is not None:
i += 1
count = u' ' + str(i)
caption += count
content += '\n\n== %s ==\n\n%s\n\n%s%s--~~~~' % \
(caption,
i18n.twtranslate(pywikibot.getSite(),
'weblinkchecker-report'),
errorReport,
archiveMsg)
comment = u'[[%s#%s|→]] %s' % \
(talkPage.title(), caption,
i18n.twtranslate(pywikibot.getSite(),
'weblinkchecker-summary'))
try:
talkPage.put(content, comment)
except pywikibot.SpamfilterError, error:
pywikibot.output(
u'\03{lightaqua}** SpamfilterError while trying to '
u'change %s: %s\03{default}'
% (talkPage.title(asLink=True), error.url))
self.semaphore.release()
def main(*args):
add_cat = None
gen = None
# summary message
summary_commandline = False
edit_summary = u""
# Array which will collect commandline parameters.
# First element is original text, second element is replacement text.
commandline_replacements = []
# A list of 2-tuples of original text and replacement text.
replacements = []
# Don't edit pages which contain certain texts.
exceptions = {
'title': [],
'text-contains': [],
'inside': [],
'inside-tags': [],
'require-title': [], # using a seperate requirements dict needs some
} # major refactoring of code.
# Should the elements of 'replacements' and 'exceptions' be interpreted
# as regular expressions?
regex = False
# Predefined fixes from dictionary 'fixes' (see above).
fix = None
# the dump's path, either absolute or relative, which will be used
# if -xml flag is present
xmlFilename = None
useSql = False
PageTitles = []
# will become True when the user presses a ('yes to all') or uses the
# -always flag.
acceptall = False
# Will become True if the user inputs the commandline parameter -nocase
caseInsensitive = False
# Will become True if the user inputs the commandline parameter -dotall
dotall = False
# Will become True if the user inputs the commandline parameter -multiline
multiline = False
# Do all hits when they overlap
allowoverlap = False
# Do not recurse replacement
recursive = False
# Between a regex and another (using -fix) sleep some time (not to waste
# too much CPU
sleep = None
# Read commandline parameters.
local_args = pywikibot.handleArgs(*args)
genFactory = pagegenerators.GeneratorFactory()
for arg in local_args:
if genFactory.handleArg(arg):
continue
if arg == '-regex':
regex = True
elif arg.startswith('-xmlstart'):
if len(arg) == 9:
xmlStart = pywikibot.input(
u'Please enter the dumped article to start with:')
else:
xmlStart = arg[10:]
elif arg.startswith('-xml'):
if len(arg) == 4:
xmlFilename = i18n.input('pywikibot-enter-xml-filename')
else:
xmlFilename = arg[5:]
elif arg == '-sql':
useSql = True
elif arg.startswith('-page'):
if len(arg) == 5:
PageTitles.append(
pywikibot.input(u'Which page do you want to change?'))
else:
PageTitles.append(arg[6:])
elif arg.startswith('-excepttitle:'):
exceptions['title'].append(arg[13:])
elif arg.startswith('-requiretitle:'):
exceptions['require-title'].append(arg[14:])
elif arg.startswith('-excepttext:'):
exceptions['text-contains'].append(arg[12:])
elif arg.startswith('-exceptinside:'):
exceptions['inside'].append(arg[14:])
elif arg.startswith('-exceptinsidetag:'):
exceptions['inside-tags'].append(arg[17:])
elif arg.startswith('-fix:'):
fix = arg[5:]
elif arg.startswith('-sleep:'):
sleep = float(arg[7:])
elif arg == '-always':
acceptall = True
elif arg == '-recursive':
recursive = True
elif arg == '-nocase':
caseInsensitive = True
elif arg == '-dotall':
dotall = True
elif arg == '-multiline':
multiline = True
elif arg.startswith('-addcat:'):
add_cat = arg[8:]
elif arg.startswith('-summary:'):
edit_summary = arg[9:]
summary_commandline = True
elif arg.startswith('-allowoverlap'):
allowoverlap = True
else:
commandline_replacements.append(arg)
pywikibot.Site().login()
gen = genFactory.getCombinedGenerator()
if (len(commandline_replacements) % 2):
raise pywikibot.Error('require even number of replacements.')
elif (len(commandline_replacements) == 2 and fix is None):
replacements.append(
(commandline_replacements[0], commandline_replacements[1]))
if not summary_commandline:
edit_summary = i18n.twtranslate(
pywikibot.Site(), 'replace-replacing', {
'description':
' (-%s +%s)' %
(commandline_replacements[0], commandline_replacements[1])
})
elif (len(commandline_replacements) > 1):
if (fix is None):
for i in range(0, len(commandline_replacements), 2):
replacements.append((commandline_replacements[i],
commandline_replacements[i + 1]))
if not summary_commandline:
pairs = [(commandline_replacements[i],
commandline_replacements[i + 1])
for i in range(0, len(commandline_replacements), 2)]
replacementsDescription = '(%s)' % ', '.join(
[('-' + pair[0] + ' +' + pair[1]) for pair in pairs])
edit_summary = i18n.twtranslate(
pywikibot.Site(), 'replace-replacing',
{'description': replacementsDescription})
else:
raise pywikibot.Error(
'Specifying -fix with replacements is undefined')
elif fix is None:
old = pywikibot.input(
u'Please enter the text that should be replaced:')
new = pywikibot.input(u'Please enter the new text:')
change = '(-' + old + ' +' + new
replacements.append((old, new))
while True:
old = pywikibot.input(
u'Please enter another text that should be replaced,' +
u'\nor press Enter to start:')
if old == '':
change += ')'
break
new = i18n.input('pywikibot-enter-new-text')
change += ' & -' + old + ' +' + new
replacements.append((old, new))
if not summary_commandline:
default_summary_message = i18n.twtranslate(pywikibot.Site(),
'replace-replacing',
{'description': change})
pywikibot.output(u'The summary message will default to: %s' %
default_summary_message)
summary_message = pywikibot.input(
u'Press Enter to use this default message, or enter a ' +
u'description of the\nchanges your bot will make:')
if summary_message == '':
summary_message = default_summary_message
edit_summary = summary_message
else:
# Perform one of the predefined actions.
try:
fix = fixes.fixes[fix]
except KeyError:
pywikibot.output(u'Available predefined fixes are: %s' %
fixes.fixes.keys())
return
if "regex" in fix:
regex = fix['regex']
if "msg" in fix:
if isinstance(fix['msg'], basestring):
edit_summary = i18n.twtranslate(pywikibot.Site(),
str(fix['msg']))
else:
edit_summary = pywikibot.translate(pywikibot.Site(),
fix['msg'],
fallback=True)
if "exceptions" in fix:
exceptions = fix['exceptions']
if "nocase" in fix:
caseInsensitive = fix['nocase']
replacements = fix['replacements']
# Set the regular expression flags
flags = re.UNICODE
if caseInsensitive:
flags = flags | re.IGNORECASE
if dotall:
flags = flags | re.DOTALL
if multiline:
flags = flags | re.MULTILINE
# Pre-compile all regular expressions here to save time later
for i in range(len(replacements)):
old, new = replacements[i]
if not regex:
old = re.escape(old)
oldR = re.compile(old, flags)
replacements[i] = oldR, new
for exceptionCategory in [
'title', 'require-title', 'text-contains', 'inside'
]:
if exceptionCategory in exceptions:
patterns = exceptions[exceptionCategory]
if not regex:
patterns = [re.escape(pattern) for pattern in patterns]
patterns = [re.compile(pattern, flags) for pattern in patterns]
exceptions[exceptionCategory] = patterns
if xmlFilename:
try:
xmlStart
except NameError:
xmlStart = None
gen = XmlDumpReplacePageGenerator(xmlFilename, xmlStart, replacements,
exceptions)
elif useSql:
whereClause = 'WHERE (%s)' % ' OR '.join([
"old_text RLIKE '%s'" % prepareRegexForMySQL(old.pattern)
for (old, new) in replacements
])
if exceptions:
exceptClause = 'AND NOT (%s)' % ' OR '.join([
"old_text RLIKE '%s'" % prepareRegexForMySQL(exc.pattern)
for exc in exceptions
])
else:
exceptClause = ''
query = u"""
SELECT page_namespace, page_title
FROM page
JOIN text ON (page_id = old_id)
%s
%s
LIMIT 200""" % (whereClause, exceptClause)
gen = pagegenerators.MySQLPageGenerator(query)
elif PageTitles:
pages = [
pywikibot.Page(pywikibot.Site(), PageTitle)
for PageTitle in PageTitles
]
gen = iter(pages)
if not gen:
# syntax error, show help text from the top of this file
pywikibot.showHelp('replace')
return
preloadingGen = pagegenerators.PreloadingGenerator(gen)
bot = ReplaceRobot(preloadingGen, replacements, exceptions, acceptall,
allowoverlap, recursive, add_cat, sleep, edit_summary)
bot.run()
def run(self):
"""Run the bot."""
# validate L10N
self.template_list = self.site.category_redirects()
if not self.template_list:
pywikibot.warning('No redirect templates defined for {}'
.format(self.site))
return
if not self.get_cat():
pywikibot.warning('No redirect category found for {}'
.format(self.site))
return
user = self.site.user() # invokes login()
newredirs = []
localtime = time.localtime()
today = '%04d-%02d-%02d' % localtime[:3]
edit_request_page = pywikibot.Page(
self.site, 'User:{}/category edit requests'.format(user))
datafile = pywikibot.config.datafilepath('{}-catmovebot-data'
.format(self.site.dbName()))
try:
with open(datafile, 'rb') as inp:
record = pickle.load(inp)
except IOError:
record = {}
if record:
with open(datafile + '.bak', 'wb') as f:
pickle.dump(record, f, protocol=config.pickle_protocol)
# regex to match soft category redirects
# TODO: enhance and use textlib._MultiTemplateMatchBuilder
# note that any templates containing optional "category:" are
# incorrect and will be fixed by the bot
template_regex = re.compile(
r"""{{\s*(?:%(prefix)s\s*:\s*)? # optional "template:"
(?:%(template)s)\s*\| # catredir template name
(\s*%(catns)s\s*:\s*)? # optional "category:"
([^|}]+) # redirect target cat
(?:\|[^|}]*)*}} # optional arguments 2+, ignored
""" % {'prefix': self.site.namespace(10).lower(),
'template': '|'.join(item.replace(' ', '[ _]+')
for item in self.template_list),
'catns': self.site.namespace(14)},
re.I | re.X)
self.check_hard_redirect()
comment = i18n.twtranslate(self.site, self.move_comment)
counts = {}
nonemptypages = []
redircat = self.cat
pywikibot.output('\nChecking {} category redirect pages'
.format(redircat.categoryinfo['subcats']))
catpages = set()
for cat in redircat.subcategories():
catpages.add(cat)
cat_title = cat.title(with_ns=False)
if 'category redirect' in cat_title:
message = i18n.twtranslate(
self.site, 'category_redirect-log-ignoring',
{'oldcat': cat.title(as_link=True, textlink=True)})
self.log_text.append(message)
continue
if hasattr(cat, '_catinfo'):
# skip empty categories that don't return a "categoryinfo" key
catdata = cat.categoryinfo
if 'size' in catdata and int(catdata['size']):
# save those categories that have contents
nonemptypages.append(cat)
if cat_title not in record:
# make sure every redirect has a record entry
record[cat_title] = {today: None}
try:
newredirs.append('*# {} → {}'.format(
cat.title(as_link=True, textlink=True),
cat.getCategoryRedirectTarget().title(
as_link=True, textlink=True)))
except pywikibot.Error:
pass
# do a null edit on cat
try:
cat.save()
except Exception:
pass
# delete record entries for non-existent categories
for cat_name in list(record.keys()):
if pywikibot.Category(self.site,
self.catprefix + cat_name) not in catpages:
del record[cat_name]
pywikibot.output('\nMoving pages out of {} redirected categories.'
.format(len(nonemptypages)))
for cat in pagegenerators.PreloadingGenerator(nonemptypages):
try:
if not cat.isCategoryRedirect():
message = i18n.twtranslate(
self.site, 'category_redirect-log-false-positive',
{'oldcat': cat.title(as_link=True, textlink=True)})
self.log_text.append(message)
continue
except pywikibot.Error:
message = i18n.twtranslate(
self.site, 'category_redirect-log-not-loaded',
{'oldcat': cat.title(as_link=True, textlink=True)})
self.log_text.append(message)
continue
cat_title = cat.title(with_ns=False)
if not self.readyToEdit(cat):
counts[cat_title] = None
message = i18n.twtranslate(
self.site, 'category_redirect-log-skipping',
{'oldcat': cat.title(as_link=True, textlink=True)})
self.log_text.append(message)
continue
dest = cat.getCategoryRedirectTarget()
if not dest.exists():
message = i18n.twtranslate(
self.site, 'category_redirect-problem-redirects', {
'oldcat': cat.title(as_link=True, textlink=True),
'redpage': dest.title(as_link=True, textlink=True)
})
self.problems.append(message)
# do a null edit on cat to update any special redirect
# categories this wiki might maintain
try:
cat.save()
except Exception:
pass
continue
if dest.isCategoryRedirect():
double = dest.getCategoryRedirectTarget()
if double == dest or double == cat:
message = i18n.twtranslate(
self.site, 'category_redirect-log-loop',
{'oldcat': dest.title(as_link=True, textlink=True)})
self.log_text.append(message)
# do a null edit on cat
try:
cat.save()
except Exception:
pass
else:
message = i18n.twtranslate(
self.site, 'category_redirect-log-double', {
'oldcat': cat.title(as_link=True, textlink=True),
'newcat': dest.title(as_link=True, textlink=True),
'targetcat': double.title(
as_link=True, textlink=True)
})
self.log_text.append(message)
oldtext = cat.text
# remove the old redirect from the old text,
# leaving behind any non-redirect text
oldtext = template_regex.sub('', oldtext)
newtext = ('{{%(redirtemp)s|%(ncat)s}}'
% {'redirtemp': self.template_list[0],
'ncat': double.title(with_ns=False)})
newtext = newtext + oldtext.strip()
try:
cat.text = newtext
cat.save(i18n.twtranslate(self.site,
self.dbl_redir_comment))
except pywikibot.Error as e:
message = i18n.twtranslate(
self.site, 'category_redirect-log-failed',
{'error': e})
self.log_text.append(message)
continue
found, moved = self.move_contents(cat_title,
dest.title(with_ns=False),
editSummary=comment)
if found is None:
message = i18n.twtranslate(
self.site, 'category_redirect-log-move-error', {
'oldcat': cat.title(as_link=True, textlink=True)
})
self.log_text.append(message)
elif found:
record[cat_title][today] = found
message = i18n.twtranslate(
self.site, 'category_redirect-log-moved', {
'oldcat': cat.title(as_link=True, textlink=True),
'found': found,
'moved': moved
})
self.log_text.append(message)
counts[cat_title] = found
# do a null edit on cat
try:
cat.save()
except Exception:
pass
with open(datafile, 'wb') as f:
pickle.dump(record, f, protocol=config.pickle_protocol)
self.log_text.sort()
self.problems.sort()
newredirs.sort()
comment = i18n.twtranslate(self.site, self.maint_comment)
message = i18n.twtranslate(self.site, 'category_redirect-log-new')
self.log_page.text = ('\n== %i-%02i-%02iT%02i:%02i:%02iZ ==\n'
% time.gmtime()[:6]
+ '\n'.join(self.log_text)
+ '\n* ' + message + '\n'
+ '\n'.join(newredirs)
+ '\n' + '\n'.join(self.problems)
+ '\n' + self.get_log_text())
self.log_page.save(comment)
if self.edit_requests:
edit_request_page.text = (self.edit_request_text
% {'itemlist': '\n' + '\n'.join(
(self.edit_request_item % item)
for item in self.edit_requests)})
edit_request_page.save(comment)
def add_text(page,
addText,
summary=None,
regexSkip=None,
regexSkipUrl=None,
always=False,
up=False,
putText=True,
oldTextGiven=None,
reorderEnabled=True,
create=False):
"""
Add text to a page.
@rtype: tuple of (text, newtext, always)
"""
site = page.site
if not summary:
summary = i18n.twtranslate(site, 'add_text-adding',
{'adding': addText[:200]})
if putText:
pywikibot.output('Loading {}...'.format(page.title()))
text = get_text(page, oldTextGiven, create)
if text is None:
return (False, False, always)
# Understand if the bot has to skip the page or not
# In this way you can use both -except and -excepturl
if regexSkipUrl is not None:
url = page.full_url()
result = re.findall(regexSkipUrl, site.getUrl(url))
if result != []:
pywikibot.output('Exception! regex (or word) used with -exceptUrl '
'is in the page. Skip!\n'
'Match was: {}'.format(result))
return (False, False, always)
if regexSkip is not None:
result = re.findall(regexSkip, text)
if result != []:
pywikibot.output('Exception! regex (or word) used with -except '
'is in the page. Skip!\n'
'Match was: {}'.format(result))
return (False, False, always)
# If not up, text put below
if not up:
newtext = text
# Translating the \\n into binary \n
addText = addText.replace('\\n', config.line_separator)
if (reorderEnabled):
# Getting the categories
categoriesInside = textlib.getCategoryLinks(newtext, site)
# Deleting the categories
newtext = textlib.removeCategoryLinks(newtext, site)
# Getting the interwiki
interwikiInside = textlib.getLanguageLinks(newtext, site)
# Removing the interwiki
newtext = textlib.removeLanguageLinks(newtext, site)
# Adding the text
newtext += '{}{}'.format(config.line_separator, addText)
# Reputting the categories
newtext = textlib.replaceCategoryLinks(newtext, categoriesInside,
site, True)
# Adding the interwiki
newtext = textlib.replaceLanguageLinks(newtext, interwikiInside,
site)
else:
newtext += '{}{}'.format(config.line_separator, addText)
else:
newtext = addText + config.line_separator + text
if putText and text != newtext:
pywikibot.output(
color_format('\n\n>>> {lightpurple}{0}{default} <<<',
page.title()))
pywikibot.showDiff(text, newtext)
# Let's put the changes.
error_count = 0
while True:
# If someone load it as module, maybe it's not so useful to put the
# text in the page
if not putText:
return (text, newtext, always)
if not always:
try:
choice = pywikibot.input_choice(
'Do you want to accept these changes?',
[('Yes', 'y'), ('No', 'n'), ('All', 'a'),
('open in Browser', 'b')], 'n')
except QuitKeyboardInterrupt:
sys.exit('User quit bot run.')
if choice == 'a':
always = True
elif choice == 'n':
return (False, False, always)
elif choice == 'b':
pywikibot.bot.open_webbrowser(page)
if always or choice == 'y':
result = put_text(page,
newtext,
summary,
error_count,
asynchronous=not always)
if result is not None:
return (result, result, always)
error_count += 1
def add_template(self, source, dest, task, fromsite):
"""Place or remove the Link_GA/FA template on/from a page."""
def compile_link(site, templates):
"""Compile one link template list."""
findtemplate = '(%s)' % '|'.join(templates)
return re.compile(
r'\{\{%s\|%s\}\}' %
(findtemplate.replace(' ', '[ _]'), site.code), re.IGNORECASE)
tosite = dest.site
add_tl, remove_tl = self.getTemplateList(tosite.code, task)
re_link_add = compile_link(fromsite, add_tl)
re_link_remove = compile_link(fromsite, remove_tl)
text = dest.text
m1 = add_tl and re_link_add.search(text)
m2 = remove_tl and re_link_remove.search(text)
changed = False
interactive = self.getOption('interactive')
if add_tl:
if m1:
pywikibot.output('(already added)')
else:
# insert just before interwiki
if (not interactive
or pywikibot.input_yn('Connecting %s -> %s. Proceed?' %
(source.title(), dest.title()),
default=False,
automatic_quit=False)):
if self.getOption('side'):
# Placing {{Link FA|xx}} right next to
# corresponding interwiki
text = (text[:m1.end()] + ' {{%s|%s}}' %
(add_tl[0], fromsite.code) + text[m1.end():])
else:
# Moving {{Link FA|xx}} to top of interwikis
iw = textlib.getLanguageLinks(text, tosite)
text = textlib.removeLanguageLinks(text, tosite)
text += '%s{{%s|%s}}%s' % (config.LS, add_tl[0],
fromsite.code, config.LS)
text = textlib.replaceLanguageLinks(text, iw, tosite)
changed = True
if remove_tl:
if m2:
if (changed # Don't force the user to say "Y" twice
or not interactive or
pywikibot.input_yn('Connecting %s -> %s. Proceed?' %
(source.title(), dest.title()),
default=False,
automatic_quit=False)):
text = re.sub(re_link_remove, '', text)
changed = True
elif task == 'former':
pywikibot.output('(already removed)')
if changed:
comment = i18n.twtranslate(tosite, 'featured-' + task,
{'page': source})
try:
dest.put(text, comment)
self._save_counter += 1
except pywikibot.LockedPage:
pywikibot.output('Page %s is locked!' % dest.title())
except pywikibot.PageSaveRelatedError:
pywikibot.output('Page not saved')
def run(self):
"""Run bot."""
self.site.login()
while True:
wait = False
now = time.strftime("%d %b %Y %H:%M:%S (UTC)", time.gmtime())
for sandbox_page in self.generator:
pywikibot.output(u'Preparing to process sandbox page %s' %
sandbox_page.title(as_link=True))
if sandbox_page.isRedirectPage():
pywikibot.warning(
u'%s is a redirect page, cleaning it anyway' %
sandbox_page.title(as_link=True))
try:
text = sandbox_page.text
if not self.getOption('text'):
translated_content = i18n.translate(self.site, content)
else:
translated_content = self.getOption('text')
if self.getOption('summary'):
translated_msg = self.getOption('summary')
else:
translated_msg = i18n.twtranslate(
self.site, 'clean_sandbox-cleaned')
subst = 'subst:' in translated_content
pos = text.find(translated_content.strip())
if text.strip() == translated_content.strip():
pywikibot.output(
u'The sandbox is still clean, no change necessary.'
)
elif subst and sandbox_page.userName() == self.site.user():
pywikibot.output(
u'The sandbox might be clean, no change necessary.'
)
elif pos != 0 and not subst:
sandbox_page.put(translated_content, translated_msg)
pywikibot.showDiff(text, translated_content)
pywikibot.output(u'Standard content was changed, '
u'sandbox cleaned.')
else:
edit_delta = (datetime.datetime.utcnow() -
sandbox_page.editTime())
delta = self.getOption('delay_td') - edit_delta
# Is the last edit more than 'delay' minutes ago?
if delta <= datetime.timedelta(0):
sandbox_page.put(translated_content,
translated_msg)
pywikibot.showDiff(text, translated_content)
pywikibot.output(u'Standard content was changed, '
u'sandbox cleaned.')
else: # wait for the rest
pywikibot.output(
u'Sandbox edited %.1f minutes ago...' %
(edit_delta.seconds / 60.0))
pywikibot.output(u'Sleeping for %d minutes.' %
(delta.seconds // 60))
time.sleep(delta.seconds)
wait = True
except pywikibot.EditConflict:
pywikibot.output(
u'*** Loading again because of edit conflict.\n')
except pywikibot.NoPage:
pywikibot.output(
u'*** The sandbox is not existent, skipping.')
continue
if self.getOption('no_repeat'):
pywikibot.output(u'\nDone.')
return
elif not wait:
if self.getOption('hours') < 1.0:
pywikibot.output('\nSleeping %s minutes, now %s' %
((self.getOption('hours') * 60), now))
else:
pywikibot.output('\nSleeping %s hours, now %s' %
(self.getOption('hours'), now))
time.sleep(self.getOption('hours') * 60 * 60)
def __init__(self, generator, templates, **kwargs):
"""
Constructor.
@param generator: the pages to work on
@type generator: iterable
@param templates: a dictionary which maps old template names to
their replacements. If remove or subst is True, it maps the
names of the templates that should be removed/resolved to None.
@type templates: dict
"""
self.availableOptions.update({
'subst': False,
'remove': False,
'summary': None,
'addedCat': None,
})
Bot.__init__(self, generator=generator, **kwargs)
self.templates = templates
# get edit summary message if it's empty
if not self.getOption('summary'):
comma = self.site.mediawiki_message('comma-separator')
params = {
'list': comma.join(self.templates.keys()),
'num': len(self.templates)
}
site = self.site
if self.getOption('remove'):
self.options['summary'] = i18n.twtranslate(
site, 'template-removing', params)
elif self.getOption('subst'):
self.options['summary'] = i18n.twtranslate(
site, 'template-substituting', params)
else:
self.options['summary'] = i18n.twtranslate(
site, 'template-changing', params)
# regular expression to find the original template.
# {{vfd}} does the same thing as {{Vfd}}, so both will be found.
# The old syntax, {{msg:vfd}}, will also be found.
# The group 'parameters' will either match the parameters, or an
# empty string if there are none.
replacements = []
exceptions = {}
namespace = self.site.namespaces[10]
for old, new in self.templates.items():
if namespace.case == 'first-letter':
pattern = '[' + \
re.escape(old[0].upper()) + \
re.escape(old[0].lower()) + \
']' + re.escape(old[1:])
else:
pattern = re.escape(old)
pattern = re.sub(r'_|\\ ', r'[_ ]', pattern)
templateRegex = re.compile(
r'\{\{ *(' + ':|'.join(namespace) + r':|[mM][sS][gG]:)?' +
pattern + r'(?P<parameters>\s*\|.+?|) *}}', re.DOTALL)
if self.getOption('subst') and self.getOption('remove'):
replacements.append(
(templateRegex, r'{{subst:%s\g<parameters>}}' % new))
exceptions['inside-tags'] = ['ref', 'gallery']
elif self.getOption('subst'):
replacements.append(
(templateRegex, r'{{subst:%s\g<parameters>}}' % old))
exceptions['inside-tags'] = ['ref', 'gallery']
elif self.getOption('remove'):
replacements.append((templateRegex, ''))
else:
template = pywikibot.Page(self.site, new, ns=10)
if not template.exists():
pywikibot.warning(u'Template "%s" does not exist.' % new)
if not pywikibot.input_yn('Do you want to proceed anyway?',
default=False,
automatic_quit=False):
continue
replacements.append(
(templateRegex, r'{{%s\g<parameters>}}' % new))
super(TemplateRobot,
self).__init__(generator,
replacements,
exceptions,
always=self.getOption('always'),
addedCat=self.getOption('addedCat'),
summary=self.getOption('summary'))
def transferImage(self, sourceImagePage):
"""
Download image and its description, and upload it to another site.
@return: the filename which was used to upload the image
"""
sourceSite = sourceImagePage.site
url = sourceImagePage.fileUrl().encode('utf-8')
pywikibot.output('URL should be: ' + url)
# localize the text that should be printed on image description page
try:
description = sourceImagePage.get()
# try to translate license templates
if (sourceSite.sitename,
self.targetSite.sitename) in licenseTemplates:
for old, new in licenseTemplates[
(sourceSite.sitename,
self.targetSite.sitename)].items():
new = '{{%s}}' % new
old = re.compile('{{%s}}' % old)
description = textlib.replaceExcept(description, old, new,
['comment', 'math',
'nowiki', 'pre'])
description = i18n.twtranslate(self.targetSite,
'imagetransfer-file_page_message',
{'site': sourceSite,
'description': description})
description += '\n\n'
description += sourceImagePage.getFileVersionHistoryTable()
# add interwiki link
if sourceSite.family == self.targetSite.family:
description += '\n\n{0}'.format(sourceImagePage)
except pywikibot.NoPage:
description = ''
pywikibot.output(
'Image does not exist or description page is empty.')
except pywikibot.IsRedirectPage:
description = ''
pywikibot.output('Image description page is redirect.')
else:
bot = UploadRobot(url=url, description=description,
targetSite=self.targetSite,
urlEncoding=sourceSite.encoding(),
keepFilename=self.keep_name,
verifyDescription=not self.keep_name,
ignoreWarning=self.ignore_warning)
# try to upload
targetFilename = bot.run()
if targetFilename and self.targetSite.family.name == 'commons' \
and self.targetSite.code == 'commons':
# upload to Commons was successful
reason = i18n.twtranslate(sourceSite,
'imagetransfer-nowcommons_notice')
# try to delete the original image if we have a sysop account
if sourceSite.family.name in config.sysopnames \
and sourceSite.lang in \
config.sysopnames[sourceSite.family.name]:
if sourceImagePage.delete(reason):
return
if sourceSite.lang in nowCommonsTemplate \
and sourceSite.family.name in config.usernames \
and sourceSite.lang in \
config.usernames[sourceSite.family.name]:
# add the nowCommons template.
pywikibot.output('Adding nowCommons template to '
+ sourceImagePage.title())
sourceImagePage.put(sourceImagePage.get() + '\n\n'
+ nowCommonsTemplate[sourceSite.lang]
% targetFilename,
summary=reason)
def run(self):
"""Run the bot."""
commons = pywikibot.Site('commons', 'commons')
comment = i18n.twtranslate(self.site,
'imagetransfer-nowcommons_notice')
for page in self.getPageGenerator():
if self.getOption('use_hash'):
# Page -> Has the namespace | commons image -> Not
images_list = page # 0 -> local image, 1 -> commons image
page = pywikibot.Page(self.site, images_list[0])
else:
# If use_hash is true, we have already print this before, no need
self.current_page = page
try:
localImagePage = pywikibot.FilePage(self.site, page.title())
if localImagePage.fileIsShared():
pywikibot.output(u'File is already on Commons.')
continue
sha1 = localImagePage.latest_file_info.sha1
if self.getOption('use_hash'):
filenameOnCommons = images_list[1]
else:
filenameOnCommons = self.findFilenameOnCommons(
localImagePage)
if not filenameOnCommons and not self.getOption('use_hash'):
pywikibot.output(u'NowCommons template not found.')
continue
commonsImagePage = pywikibot.FilePage(
commons, 'Image:%s' % filenameOnCommons)
if (localImagePage.title(withNamespace=False)
== commonsImagePage.title(withNamespace=False)
and self.getOption('use_hash')):
pywikibot.output(
u'The local and the commons images have the same name')
if (localImagePage.title(withNamespace=False) !=
commonsImagePage.title(withNamespace=False)):
usingPages = list(localImagePage.usingPages())
if usingPages and usingPages != [localImagePage]:
pywikibot.output(
color_format(
'"{lightred}{0}{default}" is still used in {1} pages.',
localImagePage.title(withNamespace=False),
len(usingPages)))
if self.getOption('replace') is True:
pywikibot.output(
color_format(
'Replacing "{lightred}{0}{default}" by '
'"{lightgreen}{1}{default}\".',
localImagePage.title(withNamespace=False),
commonsImagePage.title(
withNamespace=False)))
bot = ImageBot(
pg.FileLinksGenerator(localImagePage),
localImagePage.title(withNamespace=False),
commonsImagePage.title(withNamespace=False),
'', self.getOption('replacealways'),
self.getOption('replaceloose'))
bot.run()
# If the image is used with the urlname the
# previous function won't work
is_used = bool(
list(
pywikibot.FilePage(
self.site,
page.title()).usingPages(total=1)))
if is_used and self.getOption('replaceloose'):
bot = ImageBot(
pg.FileLinksGenerator(localImagePage),
localImagePage.title(withNamespace=False,
asUrl=True),
commonsImagePage.title(
withNamespace=False), '',
self.getOption('replacealways'),
self.getOption('replaceloose'))
bot.run()
# refresh because we want the updated list
usingPages = len(
list(
pywikibot.FilePage(
self.site, page.title()).usingPages()))
if usingPages > 0 and self.getOption('use_hash'):
# just an enter
pywikibot.input(
u'There are still %s pages with this \
image, confirm the manual removal from them please.'
% usingPages)
else:
pywikibot.output(u'Please change them manually.')
continue
else:
pywikibot.output(
color_format(
'No page is using "{lightgreen}{0}{default}" '
'anymore.',
localImagePage.title(withNamespace=False)))
commonsText = commonsImagePage.get()
if self.getOption('replaceonly') is False:
if sha1 == commonsImagePage.latest_file_info.sha1:
pywikibot.output(
u'The image is identical to the one on Commons.')
if (len(localImagePage.getFileVersionHistory()) > 1
and not self.getOption('use_hash')):
pywikibot.output(
u"This image has a version history. Please \
delete it manually after making sure that the \
old versions are not worth keeping."
"")
continue
if self.getOption('always') is False:
format_str = color_format(
'\n\n>>>> Description on {lightpurple}%s'
'{default} <<<<\n')
pywikibot.output(format_str % page.title())
pywikibot.output(localImagePage.get())
pywikibot.output(format_str %
commonsImagePage.title())
pywikibot.output(commonsText)
if pywikibot.input_yn(
u'Does the description on Commons contain '
'all required source and license\n'
'information?',
default=False,
automatic_quit=False):
localImagePage.delete(
'%s [[:commons:Image:%s]]' %
(comment, filenameOnCommons),
prompt=False)
else:
localImagePage.delete(
comment +
' [[:commons:Image:%s]]' % filenameOnCommons,
prompt=False)
else:
pywikibot.output(
u'The image is not identical to the one on Commons.'
)
except (pywikibot.NoPage, pywikibot.IsRedirectPage) as e:
pywikibot.output(u'%s' % e[0])
continue
def main(*args):
protectionLevels = ['sysop', 'autoconfirmed', 'none']
# This factory is responsible for processing command line arguments
# that are also used by other scripts and that determine on which pages
# to work on.
pageName = ''
summary = None
always = False
generator = None
edit = ''
move = ''
defaultProtection = 'sysop'
# read command line parameters
local_args = pywikibot.handleArgs(*args)
genFactory = pagegenerators.GeneratorFactory()
mysite = pywikibot.Site()
for arg in local_args:
if arg == '-always':
always = True
elif arg.startswith('-summary'):
if len(arg) == len('-summary'):
summary = pywikibot.input(u'Enter a reason for the protection:')
else:
summary = arg[len('-summary:'):]
elif arg.startswith('-images'):
pywikibot.output('\n\03{lightred}-image option is deprecated. '
'Please use -imagelinks instead.\03{default}\n')
local_args.append('-imagelinks' + arg[7:])
elif arg.startswith('-unprotect'):
defaultProtection = 'none'
elif arg.startswith('-edit'):
edit = arg[len('-edit:'):]
if edit not in protectionLevels:
edit = choiceProtectionLevel(
'edit', defaultProtection, protectionLevels)
elif arg.startswith('-move'):
move = arg[len('-move:'):]
if move not in protectionLevels:
move = choiceProtectionLevel(
'move', defaultProtection, protectionLevels)
elif arg.startswith('-create'):
create = arg[len('-create:'):]
if create not in protectionLevels:
create = choiceProtectionLevel(
'create', defaultProtection, protectionLevels)
else:
genFactory.handleArg(arg)
found = arg.find(':') + 1
if found:
pageName = arg[found:]
if not summary:
if pageName:
if arg.startswith('cat') or arg.startswith('subcats'):
summary = i18n.twtranslate(mysite, 'protect-category',
{'cat': pageName})
elif arg.startswith('links'):
summary = i18n.twtranslate(mysite, 'protect-links',
{'page': pageName})
elif arg.startswith('ref'):
summary = i18n.twtranslate(mysite, 'protect-ref',
{'page': pageName})
elif arg.startswith('imageused'):
summary = i18n.twtranslate(mysite, 'protect-images',
{'page': pageName})
elif arg.startswith('file'):
summary = i18n.twtranslate(mysite, 'protect-simple')
generator = genFactory.getCombinedGenerator()
# We are just protecting pages, so we have no need of using a preloading
# page generator to actually get the text of those pages.
if generator:
if summary is None:
summary = pywikibot.input(u'Enter a reason for the %sprotection:'
% ['', 'un'][protectionLevels == 'none'])
if not edit:
edit = defaultProtection
if not move:
move = defaultProtection
bot = ProtectionRobot(generator, summary, always, edit=edit, move=move)
bot.run()
else:
# Show help text from the top of this file
pywikibot.showHelp()
def handle_bad_page(self, *values):
"""Process one bad page."""
try:
self.content = self.page.get()
except pywikibot.IsRedirectPage:
pywikibot.output('Already redirected, skipping.')
return
except pywikibot.NoPage:
pywikibot.output('Already deleted')
return
for d in pywikibot.translate(self.site.code, done):
if d in self.content:
pywikibot.output(
'Found: "{}" in content, nothing necessary'.format(d))
return
pywikibot.output('---- Start content ----------------')
pywikibot.output(self.content)
pywikibot.output('---- End of content ---------------')
# Loop other user answer
answered = False
while not answered:
answer = pywikibot.input(self.question)
if answer == 'q':
raise QuitKeyboardInterrupt
if answer == 'd':
pywikibot.output('Trying to delete page [[{}]].'.format(
self.page.title()))
self.page.delete()
return
if answer == 'e':
old = self.content
new = editor.TextEditor().edit(old)
msg = pywikibot.input('Summary message:')
self.userPut(self.page, old, new, summary=msg)
return
if answer == 'b':
pywikibot.output('Blanking page [[{}]].'.format(
self.page.title()))
try:
self.page.put('',
summary=i18n.twtranslate(
self.site.lang, 'followlive-blanking',
{'content': self.content}))
except pywikibot.EditConflict:
pywikibot.output(
'An edit conflict occurred! Automatically retrying')
self.handle_bad_page(self)
return
if answer == '':
pywikibot.output('Page correct! Proceeding with next pages.')
return
# Check user input:
if answer[0] == '':
# Answer entered as an utf8 string
answer = answer[1:]
try:
choices = answer.split(',')
except ValueError:
# User entered wrong value
pywikibot.error('"{}" is not valid'.format(answer))
continue
# test input
for choice in choices:
try:
x = int(choice)
except ValueError:
break
else:
answered = (x >= 1 and x <= len(self.questionlist))
if not answered:
pywikibot.error('"{}" is not valid'.format(answer))
continue
summary = ''
for choice in choices:
answer = int(choice)
# grab the template parameters
tpl = pywikibot.translate(self.site,
templates)[self.questionlist[answer]]
if tpl['pos'] == 'top':
pywikibot.output('prepending {}...'.format(
self.questionlist[answer]))
self.content = self.questionlist[answer] + '\n' + self.content
elif tpl['pos'] == 'bottom':
pywikibot.output('appending {}...'.format(
self.questionlist[answer]))
self.content += '\n' + self.questionlist[answer]
else:
raise RuntimeError(
'"pos" should be "top" or "bottom" for template {}. '
'Contact a developer.'.format(self.questionlist[answer]))
summary += tpl['msg'] + ' '
pywikibot.output('Probably added ' + self.questionlist[answer])
self.page.put(self.content, summary=summary)
pywikibot.output('with comment {}\n'.format(summary))
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
:param args: command line arguments
:type args: str
"""
global workingcat, parentcats
global checked
global excludefile
checked = set()
workingcatname = ''
options = {}
local_args = pywikibot.handle_args(args)
for arg in local_args:
option = arg[1:]
if not arg.startswith('-'):
if not workingcatname:
options['catnames'] = workingcatname = arg
else:
pywikibot.warning(
'Working category "{}" is already given.'.format(
workingcatname))
else:
options[option] = True
if not workingcatname:
pywikibot.bot.suggest_help(missing_parameters=['working category'])
return
mysite = pywikibot.Site()
summary = i18n.twtranslate(mysite, 'makecat-create',
{'cat': workingcatname})
bot = MakeCatBot(site=mysite, summary=summary, **options)
workingcat = pywikibot.Category(
mysite, '{}{}'.format(mysite.namespaces.CATEGORY, workingcatname))
filename = pywikibot.config.datafilepath(
'category',
workingcatname.encode('ascii', 'xmlcharrefreplace').decode('ascii') +
'_exclude.txt')
try:
with codecs.open(filename, 'r', encoding=mysite.encoding()) as f:
for line in f.readlines():
# remove leading and trailing spaces, LF and CR
line = line.strip()
if not line:
continue
pl = pywikibot.Page(mysite, line)
checked.add(pl)
excludefile = codecs.open(filename, 'a', encoding=mysite.encoding())
except IOError:
# File does not exist
excludefile = codecs.open(filename, 'w', encoding=mysite.encoding())
# Get parent categories in order to `removeparent`
try:
parentcats = workingcat.categories()
except Error:
parentcats = []
# Do not include articles already in subcats; only checking direct subcats
subcatlist = list(workingcat.subcategories())
if subcatlist:
subcatlist = pagegenerators.PreloadingGenerator(subcatlist)
for cat in subcatlist:
artlist = list(cat.articles())
for page in artlist:
checked.add(page)
# Fetch articles in category, and mark as already checked (seen)
# If category is empty, ask user if they want to look for pages
# in a different category.
articles = list(workingcat.articles(content=True))
if not articles:
pywikibot.output('Category {} does not exist or is empty. '
'Which page to start with?'.format(workingcatname))
answer = pywikibot.input('(Default is [[{}]]):'.format(workingcatname))
if not answer:
answer = workingcatname
pywikibot.output('' + answer)
pl = pywikibot.Page(mysite, answer)
articles = [pl]
for pl in articles:
checked.add(pl)
bot.include(pl)
bot.run()
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: unicode
"""
gen = None
notitle = False
fmt = '1'
outputlang = None
page_get = False
base_dir = None
encoding = config.textfile_encoding
page_target = None
overwrite = False
summary = 'listpages-save-list'
# Process global args and prepare generator args parser
local_args = pywikibot.handle_args(args)
gen_factory = GeneratorFactory()
for arg in local_args:
option, sep, value = arg.partition(':')
if option == '-notitle':
notitle = True
elif option == '-format':
fmt = value.replace('\\03{{', '\03{{')
if not fmt.strip():
notitle = True
elif option == '-outputlang:':
outputlang = value
elif option == '-get':
page_get = True
elif option == '-save':
base_dir = value or '.'
elif option == '-encode':
encoding = value
elif option == '-put':
page_target = value
elif option == '-overwrite':
overwrite = True
elif option == '-summary':
summary = value
else:
gen_factory.handleArg(arg)
if base_dir:
base_dir = os.path.expanduser(base_dir)
if not os.path.isabs(base_dir):
base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir))
if not os.path.exists(base_dir):
pywikibot.output('Directory "{0}" does not exist.'
.format(base_dir))
choice = pywikibot.input_yn(
'Do you want to create it ("No" to continue without saving)?')
if choice:
os.makedirs(base_dir, mode=0o744)
else:
base_dir = None
elif not os.path.isdir(base_dir):
# base_dir is a file.
pywikibot.warning('Not a directory: "{0}"\n'
'Skipping saving ...'
.format(base_dir))
base_dir = None
if page_target:
site = pywikibot.Site()
page_target = pywikibot.Page(site, page_target)
if not overwrite and page_target.exists():
pywikibot.bot.suggest_help(
additional_text='Page {0} already exists.\n'
'You can use the -overwrite argument to '
'replace the content of this page.'
.format(page_target.title(as_link=True)))
return False
if re.match('^[a-z_-]+$', summary):
summary = i18n.twtranslate(site, summary)
gen = gen_factory.getCombinedGenerator()
if gen:
i = 0
output_list = []
for i, page in enumerate(gen, start=1):
if not notitle:
page_fmt = Formatter(page, outputlang)
output_list += [page_fmt.output(num=i, fmt=fmt)]
pywikibot.stdout(output_list[-1])
if page_get:
try:
pywikibot.stdout(page.text)
except pywikibot.Error as err:
pywikibot.output(err)
if base_dir:
filename = os.path.join(base_dir, page.title(as_filename=True))
pywikibot.output('Saving {0} to {1}'
.format(page.title(), filename))
with open(filename, mode='wb') as f:
f.write(page.text.encode(encoding))
pywikibot.output('{0} page(s) found'.format(i))
if page_target:
page_target.text = '\n'.join(output_list)
page_target.save(summary=summary)
return True
else:
pywikibot.bot.suggest_help(missing_generator=True)
return False
def MakeMoveSummary(self, fromTitle, toTitle):
return i18n.twtranslate(self.site, "casechecker-replacement-linklist",
{
'source': fromTitle,
'target': toTitle
})
newRedir = targetPage
continue
try:
oldText = redir.get(get_redirect=True)
except pywikibot.BadTitle:
pywikibot.output(u"Bad Title Error")
break
text = self.site.redirectRegex().sub(
'#%s %s' % (self.site.redirect(),
targetPage.title(asLink=True, textlink=True)),
oldText)
if redir.title() == targetPage.title() or text == oldText:
pywikibot.output(u"Note: Nothing left to do on %s" %
redir.title(asLink=True))
break
summary = i18n.twtranslate(self.site, 'redirect-fix-double',
{'to': targetPage.title(asLink=True)})
pywikibot.showDiff(oldText, text)
if self.prompt(u'Do you want to accept the changes?'):
try:
redir.put(text, summary)
except pywikibot.LockedPage:
pywikibot.output(u'%s is locked.' % redir.title())
except pywikibot.SpamfilterError, error:
pywikibot.output(
u"Saving page [[%s]] prevented by spam filter: %s" %
(redir.title(), error.url))
except pywikibot.PageNotSaved, error:
pywikibot.output(u"Saving page [[%s]] failed: %s" %
(redir.title(), error))
except pywikibot.NoUsername:
pywikibot.output(
def ProcessDataBlock(self, data):
if 'query' not in data or 'pages' not in data['query']:
return
firstItem = True
for pageID, page in data['query']['pages'].items():
printed = False
title = page['title']
self.currentTitle = title
if 'missing' in page:
continue
if firstItem:
if self.lastLetter != title[0]:
pywikibot.ui.output('Processing %s\n' % title)
self.lastLetter = title[0]
firstItem = False
if self.titles:
err = self.ProcessTitle(title)
if err:
changed = False
if self.replace:
if len(err[1]) == 1:
newTitle = err[1][0]
editSummary = i18n.twtranslate(
self.site, "casechecker-rename")
dst = self.Page(newTitle)
if 'redirect' in page:
src = self.Page(title)
redir = src.getRedirectTarget()
redirTitle = redir.title(asLink=True,
textlink=True)
if not dst.exists():
src.move(newTitle,
editSummary,
movesubpages=True)
changed = True
replErrors = False
for p in src.getReferences(
follow_redirects=False):
if p.namespace() == 2:
continue
oldText = p.text
newText = self.ReplaceLink(
oldText, title, newTitle)
if not self.PutNewPage(
p, newText, [
self.MakeMoveSummary(
title, newTitle)
]):
replErrors = True
if not replErrors:
editSummary = i18n.twtranslate(
self.site,
"casechecker-delete-summary")
newText = i18n.twtranslate(
self.site, "casechecker-delete-reason",
redirTitle)
if newText:
src.text = u'{{delete}}\n\n' + newText
src.save(editSummary, minor=False)
changed = True
elif not dst.exists():
src = self.Page(title)
if page['ns'] == 14:
import category
dst = self.Page(newTitle)
bot = category.CategoryMoveRobot(
src.title(withNamespace=False),
dst.title(withNamespace=False),
self.autonomous, editSummary + u' ' +
self.MakeMoveSummary(title, newTitle),
True)
bot.run()
else:
src.move(newTitle,
editSummary,
movesubpages=True)
changed = True
if not changed:
if len(err[1]) > 0:
self.AppendLineToLog(self.failedTitles, title)
else:
self.AddNoSuggestionTitle(title)
self.WikiLog(u"* " + err[0])
printed = True
if self.links:
allLinks = None
if 'links' in page:
allLinks = page['links']
if 'categories' in page:
if allLinks:
allLinks = allLinks + page['categories']
else:
allLinks = page['categories']
if allLinks:
pageObj = None
pageTxt = None
msg = []
foundSuggestions = False
for l in allLinks:
ltxt = l['title']
err = self.ProcessTitle(ltxt)
if err:
if len(err[1]) > 0:
foundSuggestions = True
elif self.AddNoSuggestionTitle(ltxt):
continue
newTitle = None
if self.replace:
newTitle = self.PickTarget(title, ltxt, err[1])
if newTitle:
if pageObj is None:
pageObj = self.Page(title)
pageTxt = pageObj.get()
msg.append(
self.MakeMoveSummary(ltxt, newTitle))
pageTxt = self.ReplaceLink(
pageTxt, ltxt, newTitle)
if not newTitle:
if not printed:
self.WikiLog(
u"* %s: link to %s" %
(self.MakeLink(title, False), err[0]))
printed = True
else:
self.WikiLog(u"** link to %s" % err[0])
if pageObj is not None:
if self.PutNewPage(pageObj, pageTxt, msg):
# done, no need to log anything
foundSuggestions = False
if foundSuggestions:
self.AppendLineToLog(self.failedTitles, title)
if self.stopAfter > 0:
self.stopAfter -= 1
if self.stopAfter == 0:
raise ValueError(u'Stopping because we are done')
def run(self):
"""Run the bot."""
tosend = {
'language': self.imagePage.site.lang.encode('utf-8'),
'image': self.imagePage.title(with_ns=False).encode('utf-8'),
'newname': self.newname.encode('utf-8'),
'project': self.imagePage.site.family.name.encode('utf-8'),
'username': '',
'commonsense': '1',
'remove_categories': '1',
'ignorewarnings': '1',
'doit': 'Uitvoeren'
}
pywikibot.output(tosend)
CH = pageTextPost('http://tools.wmflabs.org/commonshelper/index.php',
tosend)
pywikibot.output('Got CH desc.')
tablock = CH.split('<textarea ')[1].split('>')[0]
CH = CH.split('<textarea ' + tablock + '>')[1].split('</textarea>')[0]
CH = CH.replace('×', '×')
CH = self.fixAuthor(CH)
pywikibot.output(CH)
# I want every picture to be tagged with the bottemplate so i can check
# my contributions later.
CH = ('\n\n{{BotMoveToCommons|%s.%s|year={{subst:CURRENTYEAR}}'
'|month={{subst:CURRENTMONTHNAME}}|day={{subst:CURRENTDAY}}}}' %
(self.imagePage.site.lang, self.imagePage.site.family.name) + CH)
if self.category:
CH = CH.replace(
'{{subst:Unc}} <!-- Remove this line once you have '
'added categories -->', '')
CH += '[[Category:' + self.category + ']]'
bot = UploadRobot(url=self.imagePage.fileUrl(),
description=CH,
useFilename=self.newname,
keepFilename=True,
verifyDescription=False,
ignoreWarning=True,
targetSite=self.image_repo)
bot.run()
# Should check if the image actually was uploaded
if pywikibot.Page(self.image_repo, 'Image:' + self.newname).exists():
# Get a fresh copy, force to get the page so we don't run into edit
# conflicts
imtxt = self.imagePage.get(force=True)
# Remove the move to commons templates
if self.imagePage.site.lang in moveToCommonsTemplate:
for moveTemplate in moveToCommonsTemplate[
self.imagePage.site.lang]:
imtxt = re.sub(r'(?i)\{\{' + moveTemplate + r'[^\}]*\}\}',
'', imtxt)
# add {{NowCommons}}
if self.imagePage.site.lang in nowCommonsTemplate:
addTemplate = nowCommonsTemplate[
self.imagePage.site.lang] % self.newname
else:
addTemplate = nowCommonsTemplate['_default'] % self.newname
commentText = i18n.twtranslate(
self.imagePage.site, 'commons-file-now-available', {
'localfile': self.imagePage.title(with_ns=False),
'commonsfile': self.newname
})
pywikibot.showDiff(self.imagePage.get(), imtxt + addTemplate)
self.imagePage.put(imtxt + addTemplate, comment=commentText)
self.gen = pagegenerators.FileLinksGenerator(self.imagePage)
self.preloadingGen = pagegenerators.PreloadingGenerator(self.gen)
moveSummary = i18n.twtranslate(
self.imagePage.site, 'commons-file-moved', {
'localfile': self.imagePage.title(with_ns=False),
'commonsfile': self.newname
})
# If the image is uploaded under a different name, replace all
# instances
if self.imagePage.title(with_ns=False) != self.newname:
imagebot = ImageRobot(
generator=self.preloadingGen,
oldImage=self.imagePage.title(with_ns=False),
newImage=self.newname,
summary=moveSummary,
always=True,
loose=True)
imagebot.run()
# If the user want to delete the page and
# the user has sysops privilege, delete the page, otherwise
# it will be marked for deletion.
if self.delete_after_done:
self.imagePage.delete(moveSummary, False)
return
def __init__(self,
oldcat,
newcat=None,
batch=False,
comment='',
inplace=False,
move_oldcat=True,
delete_oldcat=True,
title_regex=None,
history=False,
pagesonly=False):
"""Store all given parameters in the objects attributes.
@param oldcat: The move source.
@param newcat: The move target.
@param batch: If True the user has not to confirm the deletion.
@param comment: The edit summary for all pages where the
category is changed.
@param inplace: If True the categories are not reordered.
@param move_oldcat: If True the category page (and talkpage) is
copied to the new category.
@param delete_oldcat: If True the oldcat page and talkpage are
deleted (or nominated for deletion) if it is empty.
@param title_regex: Only pages (and subcats) with a title that
matches the regex are moved.
@param history: If True the history of the oldcat is posted on
the talkpage of newcat.
@param pagesonly: If True only move pages, not subcategories.
"""
self.site = pywikibot.Site()
# Create attributes for the categories and their talk pages.
self.oldcat = self._makecat(oldcat)
self.oldtalk = self.oldcat.toggleTalkPage()
if newcat:
self.newcat = self._makecat(newcat)
self.newtalk = self.newcat.toggleTalkPage()
else:
self.newcat = None
self.newtalk = None
# Set boolean settings.
self.inplace = inplace
self.move_oldcat = move_oldcat
self.delete_oldcat = delete_oldcat
self.batch = batch
self.title_regex = title_regex
self.history = history
self.pagesonly = pagesonly
# Set edit summary for changed pages.
self.comment = comment
if not self.comment:
if self.newcat:
template_vars = {
'oldcat': self.oldcat.title(withNamespace=False),
'newcat': self.newcat.title(withNamespace=False)
}
self.comment = i18n.twtranslate(self.site,
'category-replacing',
template_vars)
else:
template_vars = {
'oldcat': self.oldcat.title(withNamespace=False)
}
self.comment = i18n.twtranslate(self.site, 'category-removing',
template_vars)
