def copy_pronunciations(language, language_name, ipa_or_pron='IPA'):
pron_regex = re.compile('\{\{([a-z]+)\-%s\|(.*)\}\}' % ipa_or_pron)
for mg_page in get_pages_from_category('mg', language_name):
print('>>>>', mg_page.title(), '<<<<')
en_page = pywikibot.Page(pywikibot.Site('en', 'wiktionary'), mg_page.title())
if en_page.isRedirectPage():
print('redirect')
continue
if en_page.exists():
en_content = en_page.get()
match = [x for x in pron_regex.findall(en_content) if x[0] == language]
if not match:
print('no match')
continue
old_content = mg_content = mg_page.get()
if '{{fanononana||%s}}' % language in mg_content:
print(match)
concat_pron = ''
for m in match:
concat_pron += m[1]
mg_content = mg_content.replace(
'{{fanononana||%s}}' % language,
'{{fanononana-%s|%s}}' % (language, concat_pron))
pywikibot.showDiff(old_content, mg_content)
mg_page.put(mg_content, "%s: manampy fanononana" % language_name)
else:
print('{{fanononana||%s}} not found' % language)
else:
print('english page does not exist')
continue
def save(self, page, newText):
"""
Saves the page to the wiki, if the user accepts the changes made.
"""
pywikibot.showDiff(page.get(), newText)
if not self.always:
choice = pywikibot.inputChoice(
u'Do you want to accept these changes?',
['Yes', 'No', 'Always yes'], 'yNa', 'Y')
if choice == 'n':
return
elif choice == 'a':
self.always = True
page.text = newText
if self.always:
try:
page.save(self.comment)
except pywikibot.EditConflict:
pywikibot.output(u'Skipping %s because of edit conflict'
% (page.title(),))
except pywikibot.SpamfilterError as e:
pywikibot.output(
u'Cannot change %s because of blacklist entry %s'
% (page.title(), e.url))
except pywikibot.LockedPage:
pywikibot.output(u'Skipping %s (locked page)' % (page.title(),))
else:
# Save the page in the background. No need to catch exceptions.
page.save(self.comment, async=True)
def update_or_create_page(self, old_page, new_text):
"""
Reads the current text of page old_page,
compare it with new_text, prompts the user,
and uploads the page
"""
# Read the original content
old_text = old_page.get()
# Give the user some context
if old_text != new_text:
pywikibot.output(new_text)
pywikibot.showDiff(old_text, new_text)
# Get a decision
prompt = u'Modify this page ?'
# Did anything change ?
if old_text == new_text:
pywikibot.output(u'No changes necessary to %s' % old_page.title());
else:
if not self.acceptall:
choice = pywikibot.input_choice(u'Do you want to accept these changes?', [('Yes', 'Y'), ('No', 'n'), ('All', 'a')], 'N')
if choice == 'a':
self.acceptall = True
if self.acceptall or choice == 'y':
# Write out the new version
old_page.put(new_text, summary)
def addMissingCommonsWikidataLink(self, filename, wikidataitem):
"""
Try to add a missing link to Commons. Returns True if it worked and False if it failed
"""
filepage = pywikibot.FilePage(self.commons, title=filename)
text = filepage.get()
replaceregex = u'\{\{(Artwork|Painting|Art Photo|Google Art Project|Google Cultural Institute|Walters Art Museum artwork|NARA-image-full)'
emptywikidataregex = u'(\s*\|\s*wikidata\s*=)\s*\n'
wikidataregex = u'[wW]ikidata\s*=\s*(Q\d+)\s*'
pywikibot.output(u'Working on %s' % (filepage.title(),))
wdmatch = re.search(wikidataregex, text)
if wdmatch:
# Template with duplicate template problems might hit this one or when database query is a bit stale
pywikibot.output(u'Seems to already link to Wikidata %s' % (wdmatch.group(1),))
return False
# First try to update an existing field
newtext = re.sub(emptywikidataregex, u'\\1%s\n' % (wikidataitem,), text, count=1)
if text==newtext:
#Ok, that didn't work, just slap it at the top
newtext = re.sub(replaceregex, u'{{\\1\n|wikidata=%s' % (wikidataitem,), text, count=1, flags=re.I)
if text==newtext:
pywikibot.output(u'Unable to add Wikidata link to %s' % (filename,))
return False
pywikibot.showDiff(text, newtext)
summary = u'Adding link to [[:d:%s]] based on usage on that item' % (wikidataitem,)
pywikibot.output(summary)
filepage.put(newtext, summary=summary)
return True
def process_page(self, page):
talk_page = page
page = talk_page.toggleTalkPage()
#find the edit where {{good article}] was added
found_oldid = False
oldid = None
while not found_oldid:
self.site.loadrevisions(page, getText=True, rvdir=False,
step=10, total=10, startid=oldid)
hist = page.fullVersionHistory(total=10) # This should fetch nothing...
for revision in hist:
if re.search('\{\{(good|ga) article\}\}', revision[3], re.IGNORECASE):
oldid = revision[0]
else:
#current oldid is the right one
found_oldid = True
break
#add the oldid in the template
if not oldid:
self.output('* ERROR: Could not find oldid for [[%s]]' % talk_page.title())
return
self.output('* Adding |oldid=%s to [[%s]]' % (oldid, talk_page.title()))
oldtext = talk_page.get()
search = re.search('\{\{GA\s?\|(.*?)\}\}', oldtext)
newtext = oldtext.replace(search.group(0), '{{GA|%s|oldid=%s}}' % (search.group(1), oldid))
pywikibot.showDiff(oldtext, newtext)
talk_page.put(newtext, 'BOT: Adding |oldid=%s to {{[[Template:GA|GA]]}}' % oldid)
def put_mg_pron(): # Fanononana ho an'ny teny malagasy
uselang = 'mg'
cat = wikipedia.Category(wikipedia.Site('mg','wiktionary'),
"Zavatra tsy ampy/fanononana tsy ampy amin'ny teny malagasy").articles()
for page in cat:
pagename = page.title()
o_txt = u""
n=0
for i in range(5):
try:
o_txt= page.get()
break
except Exception as e:
n=2**i
print '%s : Mamerina afaka %d segondra'%(e, n)
time.sleep(n)
txt = o_txt
prons = getpron(pagename)
makepron = u'{{pron|' + prons + '|' + uselang + u'}}'
txt = txt.replace('{{pron| |'+ uselang + '}}', makepron)
txt = txt.replace('{{pron||' + uselang + '}}', makepron)
txt = txt.replace('}}}}', '}}')
txt = txt.replace('}}mg}}', '}}')
wikipedia.output(' >>>> ' + pagename + ' <<<<')
page.put_async(txt, 'fametrahana ny fanononana')
wikipedia.showDiff(o_txt, txt)
def add_template(self):
if not self.adtTitle:
return # silently fail
adtPage = pywikibot.Page(self.site, self.adtTitle, ns=1)
code = mwparserfromhell.parse(adtPage.text)
war_adt_added = False
for template in code.filter_templates(recursive=False):
if template.name.matches("AdT-Vorschlag Hinweis"):
code.remove(template)
pywikibot.output(u'D:AdT: {{AdT-Vorschlag Hinweis}} gefunden,'
u'entfernt')
if template.name.matches("War AdT"):
if not any(self.snapDate in p for p in template.params):
template.add(str(len(template.params)+1), self.snapDate)
pywikibot.output(u'D:AdT: {{War AdT}} '
u'gefunden, füge heute hinzu')
war_adt_added = True
text = unicode(code)
if not war_adt_added:
template = u'{{War AdT|1=' + self.snapDate + u'}}\n'
text = self.__add_templ(text, template)
if adtPage.text != text:
pywikibot.showDiff(adtPage.text, text) # debug
adtPage.text = text
if not self.dry:
adtPage.save(comment=templateComment, botflag=True, minor=True)
def avvisaAdmin(utente, num, anno):
talk = utente.getUserTalkPage()
template = u'\n\n== Riconferma {anno} ==\n\n{{{{Avviso riconferma{num}}}}} --~~~~'.format(anno=anno, num=('|' + str(num) if num > 0 else ''))
text = talk.get(force=True)
talk.text += template
pywikibot.showDiff(text, talk.text)
talk.save(comment=u'[[Wikipedia:Bot|Bot]]: avviso riconferma {}'.format(anno), minor=False, botflag=True)
def userPut(self, page, oldtext, newtext):
"""
Print differences, ask user for confirmation,
and puts the page if needed.
Option used:
* 'always'
"""
if oldtext == newtext:
pywikibot.output(u'No changes were needed on %s'
% page.title(asLink=True))
return
pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
% page.title())
pywikibot.showDiff(oldtext, newtext)
choice = 'a'
if not self.getOption('always'):
choice = pywikibot.inputChoice(
u'Do you want to accept these changes?',
['Yes', 'No', 'All'],
['y', 'N', 'a'],
'N'
)
if choice == 'a':
# Remember the choice
self.options['always'] = True
if choice != 'n':
page.put(newtext, async=(choice == 'a'))
def revert(self, item):
history = pywikibot.Page(self.site, item['title']).fullVersionHistory(
total=2, rollback=self.rollback)
if len(history) > 1:
rev = history[1]
else:
return False
comment = i18n.twtranslate(pywikibot.Site(), 'revertbot-revert', {'revid': rev[0], 'author': rev[2], 'timestamp': rev[1]})
if self.comment:
comment += ': ' + self.comment
page = pywikibot.Page(self.site, item['title'])
pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
% page.title(asLink=True, forceInterwiki=True,
textlink=True))
if not self.rollback:
old = page.text
page.text = rev[3]
pywikibot.showDiff(old, page.text)
page.save(comment)
return comment
try:
pywikibot.data.api.Request(action="rollback", title=page.title(), user=self.user,
token=rev[4], markbot=1).submit()
except pywikibot.data.api.APIError as e:
if e == "badtoken: Invalid token":
pywikibot.out("There is an issue for rollbacking the edit, Giving up")
return False
return u"The edit(s) made in %s by %s was rollbacked" % (page.title(), self.user)
def tagNowcommons(self, imagepage, filename):
"""Tagged the imag which has been moved to Commons for deletion."""
if pywikibot.Page(pywikibot.Site('commons', 'commons'),
u'File:' + filename).exists():
# Get a fresh copy, force to get the page so we dont run into edit
# conflicts
imtxt = imagepage.get(force=True)
# Remove the move to commons templates
if imagepage.site.language() in moveToCommonsTemplate:
for moveTemplate in moveToCommonsTemplate[
imagepage.site.language()]:
imtxt = re.sub(u'(?i)\{\{' + moveTemplate +
u'[^\}]*\}\}', u'', imtxt)
# add {{NowCommons}}
if imagepage.site.language() in nowCommonsTemplate:
addTemplate = nowCommonsTemplate[
imagepage.site.language()] % filename
else:
addTemplate = nowCommonsTemplate['_default'] % filename
commentText = i18n.twtranslate(
imagepage.site(), 'commons-file-now-available',
{'localfile': imagepage.title(withNamespace=False),
'commonsfile': filename})
pywikibot.showDiff(imagepage.get(), imtxt + addTemplate)
imagepage.put(imtxt + addTemplate, comment=commentText)
def update(self, push=True):
self.fetch_info()
self.parse_info()
print self.LOCATION
print self.CATEGORY
print self.ABOUT
print self.MOVEMENT
print self.PRESSURE
print self.WINDS
#print self.UTC_TIMESTAMP
#actually update crap
#return
text = self.wikipage.get()
code = mwparserfromhell.parse(text)
main = pywikibot.Page(self.wikipage.site, '2012 Atlantic hurricane season')
main_text = main.get()
main_code = mwparserfromhell.parse(main_text)
for template in code.filter_templates():
name = template.name.lower().strip()
if name == 'Infobox hurricane current'.lower():
if template.get('name').value.strip() == 'Hurricane Sandy':
template.get('time').value = self.UTC_TIMESTAMP
template.get('category').value = self.CATEGORY
template.get('gusts').value = self.format_wind(self.WINDS)
template.get('lat').value = self.LOCATION['latc']
template.get(1).value = self.LOCATION['latd']
template.get('lon').value = self.LOCATION['lonc']
template.get(2).value = self.LOCATION['lond']
template.get('movement').value = self.format_movement(self.MOVEMENT)
template.get('pressure').value = self.format_pressure(self.PRESSURE)
pywikibot.showDiff(text, unicode(code))
if push:
self.wikipage.put(unicode(code), 'Bot: Updating hurricane infobox. Errors? [[User talk:Legoktm|report them!]]')
def save(self, text, page, comment, minorEdit=True, botflag=True):
"""Save the text."""
if text != page.text:
# Show the title of the page we're working on.
# Highlight the title in purple.
pywikibot.output(color_format(
'\n\n>>> {lightpurple}{0}{default} <<<', page.title()))
# show what was changed
pywikibot.showDiff(page.get(), text)
pywikibot.output(u'Comment: %s' % comment)
if not self.dry:
if pywikibot.input_yn(
u'Do you want to accept these changes?',
default=False, automatic_quit=False):
page.text = text
try:
# Save the page
page.save(summary=comment, minorEdit=minorEdit,
botflag=botflag)
except pywikibot.LockedPage:
pywikibot.output(u"Page %s is locked; skipping."
% page.title(asLink=True))
except pywikibot.EditConflict:
pywikibot.output(
u'Skipping %s because of edit conflict'
% (page.title()))
except pywikibot.SpamfilterError as error:
pywikibot.output(
u'Cannot change %s because of spam blacklist entry '
u'%s' % (page.title(), error.url))
else:
return True
def crawl():
a = Canagram()
for page in file(batchfile,'r').readlines():
try:
page = page.replace('\n','')
wikipedia.output('>>> %s <<<'%page)
if page:
if a.getAnagramsFor(page.title()):
page = wikipedia.Page('mg', page.decode('utf8'))
i = page.get()
else:
continue
else:
print "nahitana hadisoana : anaram-pejy tsy azo ampiasaina."
continue
if i:
c=insert(i, page.title())
else: continue
if c:
wikipedia.output('>>> %s <<<'%page.title())
wikipedia.showDiff(i, c)
if i!=c:
page.put(c, '+anagrama')
pass
else:
print "tsy hisy fiovana"
else:
print "tsy hisy fiovana"
except SyntaxError:
print "Nisy hadisoana..."
continue
def save(self, page, text):
if text != page.get():
# Show the title of the page we're working on.
# Highlight the title in purple.
pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
% page.title())
pywikibot.showDiff(page.get(), text)
if not self.always:
choice = pywikibot.inputChoice(
u'Do you want to accept these changes?',
['Yes', 'No', 'Always yes'], ['y', 'N', 'a'], 'N')
if choice == 'n':
return
elif choice == 'a':
self.always = True
if self.always:
try:
page.put(text, comment=self.comment)
except pywikibot.EditConflict:
pywikibot.output(u'Skipping %s because of edit conflict'
% (page.title(),))
except pywikibot.SpamfilterError, e:
pywikibot.output(
u'Cannot change %s because of blacklist entry %s'
% (page.title(), e.url))
except pywikibot.LockedPage:
pywikibot.output(u'Skipping %s (locked page)'
% (page.title(),))
def treat(self, page):
self.current_page = page
try:
text = page.get()
for author in self.authors:
if re.search(r'(\b%s\b)' % re.escape(author), text):
if self.authors[author] == author:
text = text.replace(
author, u'{{AutoreCitato|%s}}' % author)
else:
text = text.replace(
author, u'{{AutoreCitato|%s|%s}}'
% (self.authors[author], author))
if page.text != text:
if self.auto:
page.text = text
page.save(summary)
else:
pywikibot.showDiff(page.text, text)
choice = pywikibot.input('Agree? [Y]es, [N]o')
if choice.lower() == 'y':
page.text = text
page.save(summary)
except pywikibot.NoPage:
pywikibot.output(u"Page %s does not exist?!"
% page.title(asLink=True))
except pywikibot.IsRedirectPage:
pywikibot.output(u"Page %s is a redirect; skipping."
% page.title(asLink=True))
except pywikibot.LockedPage:
pywikibot.output(u"Page %s is locked?!" % page.title(asLink=True))
def doTagging( imageList, startTag, endTag, summary ) :
global site, debug
for image in imageList :
page = pywikibot.Page(site, image )
if page.exists() :
print "Tagging " + image.encode("utf-8")
# follow redirects to the actual image page
while True :
try :
text = page.get()
break
except pywikibot.IsRedirectPage :
page = page.getRedirectTarget()
# already tagged maybe?
oldtext = text.replace(unassessedCat, '')
if string.find(text, startTag ) < 0 :
text += "\n" + startTag + endTag + "\n"
if not debug:
#try :
# page.put(text, comment=summary, minorEdit=False)
#except pywikibot.LockedPage :
# print image.encode("utf-8") + " has an editprotected description page!"
tryPut(page,text,summary)
else:
pywikibot.output(u">>> \03{lightpurple}%s\03{default} <<<" % page.title())
pywikibot.showDiff(oldtext, text)
else :
print "Oops " + image.encode("utf-8") + " doesn't exist..."
def put_page(self, page, new):
""" Print diffs between orginal and new (text), put new text for page
"""
pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
% page.title())
pywikibot.showDiff(page.get(), new)
if not self.acceptall:
choice = pywikibot.inputChoice(u'Do you want to accept ' +
u'these changes?',
['Yes', 'No', 'All'],
['y', 'N', 'a'], 'N')
if choice == 'a':
self.acceptall = True
if choice == 'y':
page.text = new
page.save(self.msg, async=True)
if self.acceptall:
try:
page.text = new
page.save(self.msg)
except pywikibot.EditConflict:
pywikibot.output(u'Skipping %s because of edit conflict'
% (page.title(),))
except pywikibot.SpamfilterError as e:
pywikibot.output(
u'Cannot change %s because of blacklist entry %s'
% (page.title(), e.url))
except pywikibot.PageNotSaved as error:
pywikibot.error(u'putting page: %s' % (error.args,))
except pywikibot.LockedPage:
pywikibot.output(u'Skipping %s (locked page)'
% (page.title(),))
except pywikibot.ServerError as e:
pywikibot.output(u'Server Error : %s' % e)
def process_page(self, page):
talk_page = page
page = talk_page.toggleTalkPage()
# find the edit where {{good article}] was added
found_oldid = False
oldid = None
while not found_oldid:
self.site.loadrevisions(page, getText=True, rvdir=False,
step=10, total=10, startid=oldid)
for revid, rev in sorted(page._revisions.items(), reverse=True):
revision = rev.full_hist_entry()
print(revision[0])
if revision[3] is None:
oldid = revision[0]
continue
# print(revision[3].splitlines()[0])
if re.search('\{\{(good|ga) (icon|article)\}\}', revision[3], re.IGNORECASE):
oldid = revision[0]
else:
# current oldid is the right one
found_oldid = True
break
# add the oldid in the template
if not oldid:
print('* ERROR: Could not find oldid for [[%s]]' % talk_page.title())
return
print('* Adding |oldid=%s to [[%s]]' % (oldid, talk_page.title()))
oldtext = talk_page.get()
search = re.search('\{\{GA\s?\|(.*?)\}\}', oldtext)
if not search:
print('* ERROR: Could not find template to replace for [[%s]]' % talk_page.title())
return
newtext = oldtext.replace(search.group(0), '{{GA|%s|oldid=%s}}' % (search.group(1), oldid))
pywikibot.showDiff(oldtext, newtext)
talk_page.put(newtext, 'BOT: Adding |oldid=%s to {{[[Template:GA|GA]]}}' % oldid)
def save(self, text, page, comment=None, minorEdit=True,
botflag=True):
"""Update the given page with new text."""
# only save if something was changed
if text != page.get():
# Show the title of the page we're working on.
# Highlight the title in purple.
pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
% page.title())
# show what was changed
pywikibot.showDiff(page.get(), text)
pywikibot.output(u'Comment: %s' % comment)
if not self.dry:
try:
page.text = text
# Save the page
page.save(summary=comment or self.comment,
minor=minorEdit, botflag=botflag)
except pywikibot.LockedPage:
pywikibot.output(u"Page %s is locked; skipping."
% page.title(asLink=True))
except pywikibot.EditConflict:
pywikibot.output(
u'Skipping %s because of edit conflict'
% (page.title()))
except pywikibot.SpamfilterError as error:
pywikibot.output(
u'Cannot change %s because of spam blacklist entry %s'
% (page.title(), error.url))
else:
return True
return False
def createCategory(page, parent, basename):
title = page.title(withNamespace=False)
newpage = pywikibot.Page(pywikibot.Site(u'commons', u'commons'),
u'Category:' + basename + u' ' + title)
newtext = u''
newtext += u'[[Category:' + parent + u'|' + title + u']]\n'
newtext += u'[[Category:' + title + u']]\n'
if not newpage.exists():
#FIXME: Add user prompting and -always option
pywikibot.output(newpage.title())
pywikibot.showDiff(u'', newtext)
comment = u'Creating new category'
try:
newpage.put(newtext, comment)
except pywikibot.EditConflict:
pywikibot.output(u'Skipping %s due to edit conflict' % newpage.title())
except pywikibot.ServerError:
pywikibot.output(u'Skipping %s due to server error' % newpage.title())
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Error putting page: %s' % error.args)
else:
#FIXME: Add overwrite option
pywikibot.output(u'%s already exists, skipping' % newpage.title())
def treat(self, page):
"""Remove links pointing to the configured page from the given page."""
self.current_page = page
try:
oldText = page.get()
text = oldText
curpos = 0
while curpos < len(text):
match = self.linkR.search(text, pos=curpos)
if not match:
break
# Make sure that next time around we will not find this same
# hit.
curpos = match.start() + 1
text, jumpToBeginning = self.handleNextLink(text, match)
if jumpToBeginning:
curpos = 0
if oldText == text:
pywikibot.output(u'No changes necessary.')
else:
pywikibot.showDiff(oldText, text)
page.text = text
page.save(self.comment)
except pywikibot.NoPage:
pywikibot.output(u"Page %s does not exist?!"
% page.title(asLink=True))
except pywikibot.IsRedirectPage:
pywikibot.output(u"Page %s is a redirect; skipping."
% page.title(asLink=True))
except pywikibot.LockedPage:
pywikibot.output(u"Page %s is locked?!" % page.title(asLink=True))
def creditByFlickrUrl():
commons = pywikibot.Site('commons', 'commons')
flickrurls = [
'http://flickr.com/people/96396586@N07',
'http://www.flickr.com/people/96396586@N07',
'https://flickr.com/people/96396586@N07',
'https://www.flickr.com/people/96396586@N07',
]
for flickrurl in flickrurls:
images = ['hola']
while images:
linksearch = 'https://commons.wikimedia.org/w/index.php?target=%s&title=Special:LinkSearch' % (flickrurl)
req = urllib.request.Request(linksearch, headers={ 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0' })
raw = urllib.request.urlopen(req).read().strip().decode('utf-8')
images = re.findall(r'title="(File:[^<>]+?)">File:', raw)
print(images)
for image in images:
page = pywikibot.Page(commons, image)
text = page.text
newtext = page.text
newtext = re.sub(r'(\|\s*Author\s*=\s*)\[%s [^\]]*?\]\s*(de|from)?\s*(España|Spain)?' % (flickrurl), r'\1{{User:Emijrp/credit}}', newtext)
if text != newtext:
pywikibot.showDiff(text, newtext)
page.text = newtext
page.save('BOT - Updating credit template')
def cleanup_templates(self):
for adt in self.erl_props:
if adt in self.props:
# mehrmals für AdT vorgeschlagen
continue
page = pywikibot.Page(self.site, adt, ns=1)
if not page.exists():
pywikibot.error(u'ERROR: disc for AdT-Vorschlag ' + adt
+ u' does not exist!')
return
oldtext = page.text
code = mwparser.parse(page.text)
for template in code.filter_templates(recursive=False):
if template.name.matches("AdT-Vorschlag Hinweis"):
code.remove(template)
pywikibot.output(adt +
u': {{AdT-Vorschlag Hinweis}} '
u'gefunden, entfernt')
page.text = unicode(code)
if page.text == oldtext:
continue
page.text = page.text.lstrip(u'\n')
pywikibot.showDiff(oldtext, page.text)
comment = u'Bot: [[Vorlage:AdT-Vorschlag Hinweis]] entfernt'
if not self.dry:
page.save(comment=comment, botflag=True, minor=True)
def treat(self, page):
# Show the title of the page we're working on.
# Highlight the title in purple.
pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
% page.title())
try:
oldText = page.get()
text = oldText
curpos = 0
while curpos < len(text):
match = self.linkR.search(text, pos=curpos)
if not match:
break
# Make sure that next time around we will not find this same
# hit.
curpos = match.start() + 1
text, jumpToBeginning = self.handleNextLink(text, match)
if jumpToBeginning:
curpos = 0
if oldText == text:
pywikibot.output(u'No changes necessary.')
else:
pywikibot.showDiff(oldText, text)
page.text = text
page.save(self.comment)
except pywikibot.NoPage:
pywikibot.output(u"Page %s does not exist?!"
% page.title(asLink=True))
except pywikibot.IsRedirectPage:
pywikibot.output(u"Page %s is a redirect; skipping."
% page.title(asLink=True))
except pywikibot.LockedPage:
pywikibot.output(u"Page %s is locked?!" % page.title(asLink=True))
def save(self, page, newText):
"""
Saves the page to the wiki, if the user accepts the changes made.
"""
pywikibot.showDiff(page.get(), newText)
if not self.always:
choice = pywikibot.inputChoice(
u'Do you want to accept these changes?',
['Yes', 'No', 'Always yes'], ['y', 'N', 'a'], 'Y')
if choice == 'n':
return
elif choice == 'a':
self.always = True
if self.always:
try:
page.put(newText)
except pywikibot.EditConflict:
pywikibot.output(u'Skipping %s because of edit conflict'
% (page.title(),))
except pywikibot.SpamfilterError, e:
pywikibot.output(
u'Cannot change %s because of blacklist entry %s'
% (page.title(), e.url))
except pywikibot.LockedPage:
pywikibot.output(u'Skipping %s (locked page)' % (page.title(),))
def run(self):
for template in self._templates_generator():
title = template.title(withNamespace=False).lower()
for page in pagegenerators.ReferringPageGenerator(template, onlyTemplateInclusion=True):
try:
text = page.get()
except pywikibot.Error:
continue
else:
code = mwparserfromhell.parse(text)
for temp in code.ifilter_templates():
if temp.has_param("nosubst") or temp.has_param("demo"):
continue
replace = False
if temp.name.lower().strip() == title:
replace = True
if temp.name.lower().strip().startswith("template:") and temp.name.lower().strip()[9:] == title:
replace = True
if replace:
temp.name = "subst:%s" % template.title()
temp.add("subst", "subst:")
if text != code:
pywikibot.showDiff(text, code)
try:
page.put(
code,
"Bot: Substituting {{%s}}" % template.title(asLink=True, allowInterwiki=False)
)
except pywikibot.Error:
continue
def appendtext(page, apptext, always):
if page.isRedirectPage():
page = page.getRedirectTarget()
if not page.exists():
if page.isTalkPage():
text = u''
else:
raise pywikibot.NoPage(u"Page '%s' does not exist" % page.title())
else:
text = page.text
# Here you can go editing. If you find you do not
# want to edit this page, just return
oldtext = text
text += apptext
if text != oldtext:
pywikibot.showDiff(oldtext, text)
if not always:
choice = pywikibot.inputChoice(
u'Do you want to accept these changes?', ['Yes', 'No', 'All'],
'yNa', 'N')
if choice == 'a':
always = True
if always or choice == 'y':
page.text = text
page.save(i18n.translate(pywikibot.Site(), comment,
fallback=True))
def replaceSubpageWithDimensions(sourcePrefix, destPrefix):
generator = pagegenerators.PrefixingPageGenerator(prefix=sourcePrefix)
for page in generator:
title = page.title()
suffix = title[title.find('/') + 1:]
repl = u"{{" + title
repl2 = u"{{" + title.replace(u"Format:", u"")
repl3 = u"{{" + title.replace(u"Format:", u"").replace(u"'", u"'")
repl4 = u"{{" + title.replace(u"Format:", u"").lower().replace(suffix.lower(), suffix)
try:
tpl = sf.tl2Dict(sf.extractTemplate(page.get(), u"Demografia"))[0]
x = tpl["dimx"].strip()
y = tpl["dimy"].strip()
except:
continue
call = u"{{" + destPrefix + u'|' + suffix + u'|lățime=' + x + u'|înălțime=' + y
generator2 = pagegenerators.ReferringPageGenerator(page)
for page2 in generator2:
if page2.namespace() > 0:
continue
try:
text = page2.get()
newtext = text.replace(repl, call)
newtext = newtext.replace(repl2, call)
newtext = newtext.replace(repl3, call)
newtext = newtext.replace(repl4, call)
if newtext != text:
pywikibot.showDiff(text, newtext)
page2.put(newtext, comment="Folosesc noul format pentru grafice de demografie")
except:
continue
def userPut(self, page, oldtext, newtext, **kwargs):
"""
Save a new revision of a page, with user confirmation as required.
Print differences, ask user for confirmation,
and puts the page if needed.
Option used:
* 'always'
"""
if oldtext == newtext:
pywikibot.output(u'No changes were needed on %s'
% page.title(asLink=True))
return
self.current_page = page
pywikibot.showDiff(oldtext, newtext)
if 'comment' in kwargs:
pywikibot.output(u'Comment: %s' % kwargs['comment'])
if not self.user_confirm('Do you want to accept these changes?'):
return
if 'async' not in kwargs and self.getOption('always'):
kwargs['async'] = True
page.text = newtext
page.save(**kwargs)
def add_text(page,
addText,
summary=None,
regexSkip=None,
regexSkipUrl=None,
always=False,
up=False,
putText=True,
oldTextGiven=None,
reorderEnabled=True,
create=False):
"""
Add text to a page.
@param page: The page to add text to
@type page: pywikibot.page.BasePage
@param addText: Text to add
@type addText: str
@param summary: Summary of changes. If None, beginning of addText is used.
@type summary: str
@param regexSkip: Abort if text on page matches
@type regexSkip: str
@param regexSkipUrl: Abort if full url matches
@type regexSkipUrl: str
@param always: Always add text without user confirmation
@type always: bool
@param up: If True, add text to top of page, else add at bottom.
@type up: bool
@param putText: If True, save changes to the page, else return
(text, newtext, always)
@type putText: bool
@param oldTextGiven: If None fetch page text, else use this text
@type oldTextGiven: str
@param reorderEnabled: If True place text above categories and
interwiki, else place at page bottom. No effect if up = False.
@type reorderEnabled: bool
@param create: Create page if it does not exist
@type create: bool
@return: If putText=True: (success, success, always)
else: (text, newtext, always)
@rtype: Tuple of (bool, bool, bool) or (str, str, bool)
depending on value of putText parameter
"""
site = page.site
if not summary:
summary = i18n.twtranslate(site, 'add_text-adding',
{'adding': addText[:200]})
if putText:
pywikibot.output('Loading {}...'.format(page.title()))
text = get_text(page, oldTextGiven, create)
if text is None:
return (False, False, always)
# Understand if the bot has to skip the page or not
# In this way you can use both -except and -excepturl
if regexSkipUrl is not None:
url = page.full_url()
result = re.findall(regexSkipUrl, site.getUrl(url))
if result != []:
pywikibot.output('Exception! regex (or word) used with -exceptUrl '
'is in the page. Skip!\n'
'Match was: {}'.format(result))
return (False, False, always)
if regexSkip is not None:
result = re.findall(regexSkip, text)
if result != []:
pywikibot.output('Exception! regex (or word) used with -except '
'is in the page. Skip!\n'
'Match was: {}'.format(result))
return (False, False, always)
# If not up, text put below
if not up:
newtext = text
# Translating the \\n into binary \n
addText = addText.replace('\\n', config.line_separator)
if (reorderEnabled):
# Getting the categories
categoriesInside = textlib.getCategoryLinks(newtext, site)
# Deleting the categories
newtext = textlib.removeCategoryLinks(newtext, site)
# Getting the interwiki
interwikiInside = textlib.getLanguageLinks(newtext, site)
# Removing the interwiki
newtext = textlib.removeLanguageLinks(newtext, site)
# Adding the text
newtext += '{}{}'.format(config.line_separator, addText)
# Reputting the categories
newtext = textlib.replaceCategoryLinks(newtext, categoriesInside,
site, True)
# Adding the interwiki
newtext = textlib.replaceLanguageLinks(newtext, interwikiInside,
site)
else:
newtext += '{}{}'.format(config.line_separator, addText)
else:
newtext = addText + config.line_separator + text
if putText and text != newtext:
pywikibot.output(
color_format('\n\n>>> {lightpurple}{0}{default} <<<',
page.title()))
pywikibot.showDiff(text, newtext)
# Let's put the changes.
error_count = 0
while True:
# If someone load it as module, maybe it's not so useful to put the
# text in the page
if not putText:
return (text, newtext, always)
if not always:
try:
choice = pywikibot.input_choice(
'Do you want to accept these changes?',
[('Yes', 'y'), ('No', 'n'), ('All', 'a'),
('open in Browser', 'b')], 'n')
except QuitKeyboardInterrupt:
sys.exit('User quit bot run.')
if choice == 'a':
always = True
elif choice == 'n':
return (False, False, always)
elif choice == 'b':
pywikibot.bot.open_webbrowser(page)
if always or choice == 'y':
result = put_text(page,
newtext,
summary,
error_count,
asynchronous=not always)
if result is not None:
return (result, result, always)
error_count += 1
def run(self):
"""Run the bot."""
tosend = {
'language': self.imagePage.site.lang.encode('utf-8'),
'image': self.imagePage.title(withNamespace=False).encode('utf-8'),
'newname': self.newname.encode('utf-8'),
'project': self.imagePage.site.family.name.encode('utf-8'),
'username': '',
'commonsense': '1',
'remove_categories': '1',
'ignorewarnings': '1',
'doit': 'Uitvoeren'
}
pywikibot.output(tosend)
CH = pageTextPost('http://tools.wmflabs.org/commonshelper/index.php',
tosend)
pywikibot.output('Got CH desc.')
tablock = CH.split('<textarea ')[1].split('>')[0]
CH = CH.split('<textarea ' + tablock + '>')[1].split('</textarea>')[0]
CH = CH.replace(u'×', u'×')
CH = self.fixAuthor(CH)
pywikibot.output(CH)
# I want every picture to be tagged with the bottemplate so i can check
# my contributions later.
CH = ('\n\n{{BotMoveToCommons|' + self.imagePage.site.lang + '.' +
self.imagePage.site.family.name +
'|year={{subst:CURRENTYEAR}}|month={{subst:CURRENTMONTHNAME}}'
'|day={{subst:CURRENTDAY}}}}' + CH)
if self.category:
CH = CH.replace(
'{{subst:Unc}} <!-- Remove this line once you have '
'added categories -->', '')
CH += u'[[Category:' + self.category + u']]'
bot = UploadRobot(url=self.imagePage.fileUrl(),
description=CH,
useFilename=self.newname,
keepFilename=True,
verifyDescription=False,
ignoreWarning=True,
targetSite=self.image_repo)
bot.run()
# Should check if the image actually was uploaded
if pywikibot.Page(self.image_repo, u'Image:' + self.newname).exists():
# Get a fresh copy, force to get the page so we dont run into edit
# conflicts
imtxt = self.imagePage.get(force=True)
# Remove the move to commons templates
if self.imagePage.site.lang in moveToCommonsTemplate:
for moveTemplate in moveToCommonsTemplate[
self.imagePage.site.lang]:
imtxt = re.sub(r'(?i)\{\{' + moveTemplate + r'[^\}]*\}\}',
'', imtxt)
# add {{NowCommons}}
if self.imagePage.site.lang in nowCommonsTemplate:
addTemplate = nowCommonsTemplate[
self.imagePage.site.lang] % self.newname
else:
addTemplate = nowCommonsTemplate['_default'] % self.newname
commentText = i18n.twtranslate(
self.imagePage.site, 'commons-file-now-available', {
'localfile': self.imagePage.title(withNamespace=False),
'commonsfile': self.newname
})
pywikibot.showDiff(self.imagePage.get(), imtxt + addTemplate)
self.imagePage.put(imtxt + addTemplate, comment=commentText)
self.gen = pagegenerators.FileLinksGenerator(self.imagePage)
self.preloadingGen = pagegenerators.PreloadingGenerator(self.gen)
moveSummary = i18n.twtranslate(
self.imagePage.site, 'commons-file-moved', {
'localfile': self.imagePage.title(withNamespace=False),
'commonsfile': self.newname
})
# If the image is uploaded under a different name, replace all
# instances
if self.imagePage.title(withNamespace=False) != self.newname:
imagebot = image.ImageRobot(
generator=self.preloadingGen,
oldImage=self.imagePage.title(withNamespace=False),
newImage=self.newname,
summary=moveSummary,
always=True,
loose=True)
imagebot.run()
# If the user want to delete the page and
# the user has sysops privilege, delete the page, otherwise
# it will be marked for deletion.
if self.delete_after_done:
self.imagePage.delete(moveSummary, False)
return
text = re.sub(r"\*\n", "", text)
text = re.sub(r"====相关词组====\n+(\[\[Category)", r"\1", text)
text = re.sub(r"====相关词汇====\n\*同音词：\n+(\[\[Category)", r"\1", text)
text = re.sub(
r"{{(-(?:akin|anton|decl|deriv|etym|expr|link|pron|refer|synon|trans|usage)-)}}",
r"{{subst:\1}}", text)
text = re.sub(r"{{(-(?:v|n|adj|adv)-)}}", r"{{subst:\1}}", text)
text = re.sub(r"{{(-(?:de)-)}}", r"{{subst:\1}}", text)
text = re.sub(r"({{subst:-(?:v|n|adj|adv)-}})\n+", r"\1", text)
text = re.sub(r"({{subst:.*?}}\n)\n+", r"\1", text)
text = re.sub(r"<!--及物动词。第组动词。变位表格见下。-->\n", "", text)
text = re.sub(r"\n\n\n+", "\n\n", text)
if page.text == text:
print("nothing changed")
input()
continue
pywikibot.showDiff(page.text, text)
summary = "機器人：修復[[:Category:含有受损文件链接的页面|含有受损文件链接的页面]]及清理語法"
print("summary = {}".format(summary))
save = input("save?")
if save == "":
page.text = text
page.save(summary=summary, minor=True, botflag=True)
cnt += 1
else:
print("skip")
def run(self) -> None:
"""Run bot."""
self.site.login()
while True:
wait = False
now = time.strftime('%d %b %Y %H:%M:%S (UTC)', time.gmtime())
for sandbox_page in self.generator:
pywikibot.output('Preparing to process sandbox page ' +
sandbox_page.title(as_link=True))
if sandbox_page.isRedirectPage():
pywikibot.warning(
'{} is a redirect page, cleaning it anyway'.format(
sandbox_page.title(as_link=True)))
try:
text = sandbox_page.text
if self.opt.summary:
translated_msg = self.opt.summary
else:
translated_msg = i18n.twtranslate(
self.site, 'clean_sandbox-cleaned')
subst = 'subst:' in self.translated_content
pos = text.find(self.translated_content.strip())
if text.strip() == self.translated_content.strip():
pywikibot.output(
'The sandbox is still clean, no change necessary.')
elif subst and sandbox_page.userName() == self.site.user():
pywikibot.output(
'The sandbox might be clean, no change necessary.')
elif pos != 0 and not subst:
sandbox_page.put(self.translated_content,
translated_msg)
pywikibot.showDiff(text, self.translated_content)
pywikibot.output('Standard content was changed, '
'sandbox cleaned.')
else:
edit_delta = (datetime.datetime.utcnow() -
sandbox_page.editTime())
delta = self.delay_td - edit_delta
# Is the last edit more than 'delay' minutes ago?
if delta <= datetime.timedelta(0):
sandbox_page.put(self.translated_content,
translated_msg)
pywikibot.showDiff(text, self.translated_content)
pywikibot.output('Standard content was changed, '
'sandbox cleaned.')
else: # wait for the rest
pywikibot.output(
'Sandbox edited {0:.1f} minutes ago...'.format(
edit_delta.seconds / 60.0))
pywikibot.output('Sleeping for {} minutes.'.format(
delta.seconds // 60))
pywikibot.sleep(delta.seconds)
wait = True
except EditConflictError:
pywikibot.output(
'*** Loading again because of edit conflict.\n')
except NoPageError:
pywikibot.output(
'*** The sandbox is not existent, skipping.')
continue
if self.opt.hours < 0:
pywikibot.output('\nDone.')
return
if not wait:
if self.opt.hours < 1.0:
pywikibot.output('\nSleeping {} minutes, now {}'.format(
(self.opt.hours * 60), now))
else:
pywikibot.output('\nSleeping {} hours, now {}'.format(
self.opt.hours, now))
pywikibot.sleep(self.opt.hours * 60 * 60)
def run(self):
"""Start the bot."""
# Run the generator which will yield Pages which might need to be
# changed.
for page in self.generator:
if self.isTitleExcepted(page.title()):
pywikibot.output(
u'Skipping %s because the title is on the exceptions list.'
% page.title(asLink=True))
continue
try:
# Load the page's text from the wiki
original_text = page.get(get_redirect=True)
if not page.canBeEdited():
pywikibot.output(u"You can't edit page %s" %
page.title(asLink=True))
continue
except pywikibot.NoPage:
pywikibot.output(u'Page %s not found' %
page.title(asLink=True))
continue
applied = set()
new_text = original_text
while True:
if self.isTextExcepted(new_text):
pywikibot.output(u'Skipping %s because it contains text '
u'that is on the exceptions list.' %
page.title(asLink=True))
break
last_text = None
while new_text != last_text:
last_text = new_text
new_text = self.apply_replacements(last_text, applied,
page)
if not self.recursive:
break
if new_text == original_text:
pywikibot.output(u'No changes were necessary in %s' %
page.title(asLink=True))
break
if hasattr(self, 'addedCat'):
# Fetch only categories in wikitext, otherwise the others will
# be explicitly added.
cats = textlib.getCategoryLinks(new_text, site=page.site)
if self.addedCat not in cats:
cats.append(self.addedCat)
new_text = textlib.replaceCategoryLinks(new_text,
cats,
site=page.site)
# Show the title of the page we're working on.
# Highlight the title in purple.
pywikibot.output(
u"\n\n>>> \03{lightpurple}%s\03{default} <<<" %
page.title())
pywikibot.showDiff(original_text, new_text)
if self.getOption('always'):
break
choice = pywikibot.input_choice(
u'Do you want to accept these changes?',
[('Yes', 'y'), ('No', 'n'), ('Edit', 'e'),
('open in Browser', 'b'), ('all', 'a')],
default='N')
if choice == 'e':
editor = editarticle.TextEditor()
as_edited = editor.edit(original_text)
# if user didn't press Cancel
if as_edited and as_edited != new_text:
new_text = as_edited
continue
if choice == 'b':
pywikibot.bot.open_webbrowser(page)
try:
original_text = page.get(get_redirect=True, force=True)
except pywikibot.NoPage:
pywikibot.output(u'Page %s has been deleted.' %
page.title())
break
new_text = original_text
continue
if choice == 'a':
self.options['always'] = True
if choice == 'y':
page.put_async(new_text,
self.generate_summary(applied),
callback=self.count_changes)
# choice must be 'N'
break
if self.getOption('always') and new_text != original_text:
try:
page.put(new_text,
self.generate_summary(applied),
callback=self.count_changes)
except pywikibot.EditConflict:
pywikibot.output(u'Skipping %s because of edit conflict' %
(page.title(), ))
except pywikibot.SpamfilterError as e:
pywikibot.output(
u'Cannot change %s because of blacklist entry %s' %
(page.title(), e.url))
except pywikibot.LockedPage:
pywikibot.output(u'Skipping %s (locked page)' %
(page.title(), ))
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Error putting page: %s' %
(error.args, ))
def run(self):
"""Run bot."""
self.site.login()
while True:
wait = False
now = time.strftime("%d %b %Y %H:%M:%S (UTC)", time.gmtime())
for sandboxPage in self.generator:
pywikibot.output(u'Preparing to process sandbox page %s' %
sandboxPage.title(asLink=True))
if sandboxPage.isRedirectPage():
pywikibot.warning(
u'%s is a redirect page, cleaning it anyway' %
sandboxPage.title(asLink=True))
try:
text = sandboxPage.text
if not self.getOption('text'):
translatedContent = i18n.translate(self.site, content)
else:
translatedContent = self.getOption('text')
if self.getOption('summary'):
translatedMsg = self.getOption('summary')
else:
translatedMsg = i18n.twtranslate(
self.site, 'clean_sandbox-cleaned')
subst = 'subst:' in translatedContent
pos = text.find(translatedContent.strip())
if text.strip() == translatedContent.strip():
pywikibot.output(
u'The sandbox is still clean, no change necessary.'
)
elif subst and sandboxPage.userName() == self.site.user():
pywikibot.output(
u'The sandbox might be clean, no change necessary.'
)
elif pos != 0 and not subst:
sandboxPage.put(translatedContent, translatedMsg)
pywikibot.showDiff(text, translatedContent)
pywikibot.output(u'Standard content was changed, '
u'sandbox cleaned.')
else:
edit_delta = (datetime.datetime.utcnow() -
sandboxPage.editTime())
delta = self.getOption('delay_td') - edit_delta
# Is the last edit more than 'delay' minutes ago?
if delta <= datetime.timedelta(0):
sandboxPage.put(translatedContent, translatedMsg)
pywikibot.showDiff(text, translatedContent)
pywikibot.output(u'Standard content was changed, '
u'sandbox cleaned.')
else: # wait for the rest
pywikibot.output(
u'Sandbox edited %.1f minutes ago...' %
(edit_delta.seconds / 60.0))
pywikibot.output(u'Sleeping for %d minutes.' %
(delta.seconds // 60))
time.sleep(delta.seconds)
wait = True
except pywikibot.EditConflict:
pywikibot.output(
u'*** Loading again because of edit conflict.\n')
except pywikibot.NoPage:
pywikibot.output(
u'*** The sandbox is not existent, skipping.')
continue
if self.getOption('no_repeat'):
pywikibot.output(u'\nDone.')
return
elif not wait:
if self.getOption('hours') < 1.0:
pywikibot.output('\nSleeping %s minutes, now %s' %
((self.getOption('hours') * 60), now))
else:
pywikibot.output('\nSleeping %s hours, now %s' %
(self.getOption('hours'), now))
time.sleep(self.getOption('hours') * 60 * 60)
def start(self):
query_string = """
select
wrd.word,
wrd.language,
wrd.part_of_speech,
edc.definition_id,
edc.old_definition as old,
edc.new_definition as new_
from
events_definition_changed edc
join definitions def on (
edc.definition_id = def.id)
join dictionary dict on (
def.id = dict.definition)
join word wrd on (
dict.word = wrd.id)
where status = 'PENDING'
"""
with self.input_database.engine.connect() as connection:
query = connection.execute(query_string)
change_map = {}
fail_map = {}
for line in query.fetchall():
word, language, pos, definition_id, old_definition, new_definition = line
if definition_id not in change_map:
change_map[definition_id] = 0
try:
print('>>>>', old_definition, '/', word, '<<<<')
mg_word_page = pywikibot.Page(
pywikibot.Site('mg', 'wiktionary'), word)
if mg_word_page.exists(
) and not mg_word_page.isRedirectPage():
content = mg_word_page.get()
new_content = content.replace(f'[[{old_definition}]]',
f'[[{new_definition}]]')
pywikibot.showDiff(content, new_content)
if content != new_content:
mg_word_page.put(
new_content,
f'fanitsiana: -{old_definition} +{new_definition}'
)
change_map[definition_id] += 1
else:
continue
except Exception as exc:
if definition_id not in fail_map:
fail_map[definition_id] = "%s failed: %s" % (word,
str(exc))
else:
fail_map[definition_id] += "\n%s failed: %s" % (
word, str(exc))
for definition_id, changes in change_map.items():
comment = f'{changes} changes applied.'
status = 'DONE'
connection.execute(f"""
UPDATE events_definition_changed SET
status = '{status}',
status_datetime = CURRENT_TIMESTAMP,
commentary = '{comment}'
WHERE definition_id = {definition_id}""")
for definition_id, comment in fail_map.items():
status = 'FAILED'
connection.execute(f"""
UPDATE events_definition_changed SET
status = '{status}',
status_datetime = CURRENT_TIMESTAMP,
commentary = '{comment}'
WHERE definition_id = {definition_id}""")
def run(self):
"""
Starts the robot.
"""
# Run the generator which will yield Pages which might need to be
# changed.
for page in self.generator:
if self.isTitleExcepted(page.title()):
pywikibot.output(
u'Skipping %s because the title is on the exceptions list.'
% page.title(asLink=True))
continue
try:
# Load the page's text from the wiki
original_text = page.get(get_redirect=True)
if not page.canBeEdited():
pywikibot.output(u"You can't edit page %s" %
page.title(asLink=True))
continue
except pywikibot.NoPage:
pywikibot.output(u'Page %s not found' %
page.title(asLink=True))
continue
new_text = original_text
while True:
if self.isTextExcepted(new_text):
pywikibot.output(
u'Skipping %s because it contains text that is on the exceptions list.'
% page.title(asLink=True))
break
new_text = self.doReplacements(new_text)
if new_text == original_text:
pywikibot.output(u'No changes were necessary in %s' %
page.title(asLink=True))
break
if self.recursive:
newest_text = self.doReplacements(new_text)
while (newest_text != new_text):
new_text = newest_text
newest_text = self.doReplacements(new_text)
if hasattr(self, "addedCat"):
cats = page.categories(nofollow_redirects=True)
if self.addedCat not in cats:
cats.append(self.addedCat)
new_text = pywikibot.replaceCategoryLinks(
new_text, cats)
# Show the title of the page we're working on.
# Highlight the title in purple.
pywikibot.output(
u"\n\n>>> \03{lightpurple}%s\03{default} <<<" %
page.title())
pywikibot.showDiff(original_text, new_text)
if self.acceptall:
break
choice = pywikibot.inputChoice(
u'Do you want to accept these changes?',
['Yes', 'No', 'Edit', 'open in Browser', 'All', 'Quit'],
['y', 'N', 'e', 'b', 'a', 'q'], 'N')
if choice == 'e':
editor = editarticle.TextEditor()
as_edited = editor.edit(original_text)
# if user didn't press Cancel
if as_edited and as_edited != new_text:
new_text = as_edited
continue
if choice == 'b':
webbrowser.open("http://%s%s" %
(page.site.hostname(),
page.site.nice_get_address(page.title())))
i18n.input('pywikibot-enter-finished-browser')
try:
original_text = page.get(get_redirect=True, force=True)
except pywikibot.NoPage:
pywikibot.output(u'Page %s has been deleted.' %
page.title())
break
new_text = original_text
continue
if choice == 'q':
return
if choice == 'a':
self.acceptall = True
if choice == 'y':
page.put_async(new_text, self.summary)
# choice must be 'N'
break
if self.acceptall and new_text != original_text:
try:
page.put(new_text, self.summary)
except pywikibot.EditConflict:
pywikibot.output(u'Skipping %s because of edit conflict' %
(page.title(), ))
except pywikibot.SpamfilterError, e:
pywikibot.output(
u'Cannot change %s because of blacklist entry %s' %
(page.title(), e.url))
except pywikibot.PageNotSaved, error:
pywikibot.output(u'Error putting page: %s' %
(error.args, ))
except pywikibot.LockedPage:
pywikibot.output(u'Skipping %s (locked page)' %
(page.title(), ))
def operate(self, page, liste_modeles):
dict = {}
modeles_modifies_str = u""
count_modeles_modifies = 0
add_date = False
text = page.get()
for couple in page.templatesWithParams():
for modele in liste_modeles:
if couple[0] == modele:
dict[modele] = couple[1]
for modele in dict:
# NB : modele est de type pywikibot.Page
titre_modele = modele.title(withNamespace=False)
add_date = True
change_date = False
re_params = u""
#str_params = u""
pywikibot.output(u"modele : %s" %
modele.title(withNamespace=False))
if dict[modele]:
for param in dict[modele]:
param = param.replace('(', '\(').replace(
')',
'\)').replace('[', '\[').replace(']', '\]').replace(
'{', '\{').replace('{', '\}').replace('?', '\?')
re_params += (u" *\| *%s" % param
) # Paramètres allant avec le modèle
#str_params += u"|%s" % param
if re.search(u"^date *=.+", param):
param_date = re.search(
u"^date *= *([0-9]{0,2}) *(janvier|février|mars|avril|mai|juin|juillet|août|septembre|octobre|novembre|décembre) *(20[0-9]{2})",
param)
add_date = False
if param_date and param_date.group(1):
text = text.replace(
param_date.group(0), "date=%s %s" %
(param_date.group(2), param_date.group(3)))
# NB : ne fait rien si un paramètre date est mal renseigné
# mais que la correction à appliquer est inconnue, pour éviter
# d'ajouter une fausse date.
# ex : quelqu'un ajoute 'date=octobr 2015' après recherche
# mais le script tourne en janvier 2016 donc le bot
# effacerait l'ancienne date pour rajouter une date
# incorrecte.
pywikibot.output("add_date = %s" % unicode(add_date))
#pywikibot.output("str_params = %s" % unicode(str_params))
pywikibot.output("re_params = %s" % unicode(re_params))
if add_date:
# Permet de rechercher le modèle (avec ses paramètres) dans le texte
pywikibot.output(
u"{ *{ *([%s%s]%s%s) *} *}" %
(titre_modele[0].lower(),
titre_modele[0].upper(), titre_modele[1:].replace(
' ', '[ _]'), unicode(re_params)))
str_re_titre_modele = u"{ *{ *([%s%s]%s%s) *} *}" % (
titre_modele[0].lower(), titre_modele[0].upper(),
titre_modele[1:].replace(' ', '[ _]'), unicode(re_params))
now = datetime.datetime.now()
date = now.strftime(u"%B %Y")
match_modele = re.search(str_re_titre_modele, text)
pywikibot.output(match_modele.group(1))
new = u"{{%s|date=%s}}" % (match_modele.group(1),
date.decode('utf-8'))
pywikibot.output(new)
text = text.replace(match_modele.group(0), new)
modeles_modifies_str += u"[[Modèle:%(modele)s|{{%(modele)s}}]], " % {
'modele': titre_modele
}
count_modeles_modifies += 1
modeles_modifies_str = modeles_modifies_str[
0:-2] # Enlever le ', ' en trop
if count_modeles_modifies == 1:
comment = u"Bot: Ajout du paramètre 'date' dans le modèle %s" % modeles_modifies_str
else:
comment = u"Bot: Ajout du paramètre 'date' dans les modèles %s" % modeles_modifies_str
pywikibot.showDiff(page.get(), text)
if not debug:
page.put(text, comment=comment)
def run(self):
"""Start the bot."""
# Run the generator which will yield Pages which might need to be
# changed.
for page in self.generator:
if self.isTitleExcepted(page.title()):
pywikibot.output(
'Skipping {0} because the title is on the exceptions list.'
.format(page.title(as_link=True)))
continue
try:
# Load the page's text from the wiki
original_text = page.get(get_redirect=True)
if not page.canBeEdited():
pywikibot.output("You can't edit page "
+ page.title(as_link=True))
continue
except pywikibot.NoPage:
pywikibot.output('Page {0} not found'
.format(page.title(as_link=True)))
continue
applied = set()
new_text = original_text
last_text = None
context = 0
while True:
if self.isTextExcepted(new_text):
pywikibot.output('Skipping {0} because it contains text '
'that is on the exceptions list.'
.format(page.title(as_link=True)))
break
while new_text != last_text:
last_text = new_text
new_text = self.apply_replacements(last_text, applied,
page)
if not self.recursive:
break
if new_text == original_text:
pywikibot.output('No changes were necessary in '
+ page.title(as_link=True))
break
if hasattr(self, 'addedCat'):
# Fetch only categories in wikitext, otherwise the others
# will be explicitly added.
cats = textlib.getCategoryLinks(new_text, site=page.site)
if self.addedCat not in cats:
cats.append(self.addedCat)
new_text = textlib.replaceCategoryLinks(new_text,
cats,
site=page.site)
# Show the title of the page we're working on.
# Highlight the title in purple.
pywikibot.output(color_format(
'\n\n>>> {lightpurple}{0}{default} <<<', page.title()))
pywikibot.showDiff(original_text, new_text, context=context)
if self.getOption('always'):
break
choice = pywikibot.input_choice(
'Do you want to accept these changes?',
[('Yes', 'y'), ('No', 'n'), ('Edit original', 'e'),
('edit Latest', 'l'), ('open in Browser', 'b'),
('More context', 'm'), ('All', 'a')],
default='N')
if choice == 'm':
context = context * 3 if context else 3
continue
if choice == 'e':
editor = editarticle.TextEditor()
as_edited = editor.edit(original_text)
# if user didn't press Cancel
if as_edited and as_edited != new_text:
new_text = as_edited
continue
if choice == 'l':
editor = editarticle.TextEditor()
as_edited = editor.edit(new_text)
# if user didn't press Cancel
if as_edited and as_edited != new_text:
new_text = as_edited
# prevent changes from being applied again
last_text = new_text
continue
if choice == 'b':
pywikibot.bot.open_webbrowser(page)
try:
original_text = page.get(get_redirect=True, force=True)
except pywikibot.NoPage:
pywikibot.output('Page {0} has been deleted.'
.format(page.title()))
break
new_text = original_text
last_text = None
continue
if choice == 'a':
self.options['always'] = True
if choice == 'y':
page.text = new_text
page.save(summary=self.generate_summary(applied),
asynchronous=True,
callback=self._replace_async_callback,
quiet=True)
while not self._pending_processed_titles.empty():
proc_title, res = self._pending_processed_titles.get()
pywikibot.output('Page {0}{1} saved'
.format(proc_title,
'' if res else ' not'))
# choice must be 'N'
break
if self.getOption('always') and new_text != original_text:
try:
page.text = new_text
page.save(summary=self.generate_summary(applied),
callback=self._replace_sync_callback, quiet=True)
except pywikibot.EditConflict:
pywikibot.output('Skipping {0} because of edit conflict'
.format(page.title(),))
except pywikibot.SpamfilterError as e:
pywikibot.output(
'Cannot change {0} because of blacklist entry {1}'
.format(page.title(), e.url))
except pywikibot.LockedPage:
pywikibot.output('Skipping {0} (locked page)'
.format(page.title(),))
except pywikibot.PageNotSaved as error:
pywikibot.output('Error putting page: {0}'
.format(error.args,))
if self._pending_processed_titles.qsize() > 50:
while not self._pending_processed_titles.empty():
proc_title, res = self._pending_processed_titles.get()
pywikibot.output('Page {0}{1} saved'
.format(proc_title,
'' if res else ' not'))
def delete_1_broken_redirect(self, redir_name):
"""Treat one broken redirect."""
if isinstance(redir_name, basestring):
redir_page = pywikibot.Page(self.site, redir_name)
else:
redir_page = redir_name
# Show the title of the page we're working on.
# Highlight the title in purple.
done = not self.getOption('delete')
pywikibot.output(
color_format('\n\n>>> {lightpurple}{0}{default} <<<',
redir_page.title()))
try:
targetPage = redir_page.getRedirectTarget()
except pywikibot.IsNotRedirectPage:
pywikibot.output(u'%s is not a redirect.' % redir_page.title())
except pywikibot.CircularRedirect:
pywikibot.output(u'%s is a circular redirect.' %
redir_page.title())
except pywikibot.NoPage:
pywikibot.output(u'%s doesn\'t exist.' % redir_page.title())
except pywikibot.InvalidTitle:
pywikibot.exception()
except pywikibot.InterwikiRedirectPage:
pywikibot.output('%s is on another site.' % redir_page.title())
else:
try:
targetPage.get()
except pywikibot.BadTitle as e:
pywikibot.warning(
u'Redirect target %s is not a valid page title.' %
str(e)[10:])
pass
except pywikibot.InvalidTitle:
pywikibot.exception()
pass
except pywikibot.NoPage:
movedTarget = None
try:
movedTarget = targetPage.moved_target()
except pywikibot.NoMoveTarget:
pass
if movedTarget:
if not movedTarget.exists():
# FIXME: Test to another move
pywikibot.output(u'Target page %s does not exist' %
(movedTarget))
elif redir_page == movedTarget:
pywikibot.output(
'Redirect to target page forms a redirect loop')
else:
pywikibot.output(u'%s has been moved to %s' %
(redir_page, movedTarget))
reason = i18n.twtranslate(
self.site, 'redirect-fix-broken-moved',
{'to': movedTarget.title(asLink=True)})
content = redir_page.get(get_redirect=True)
redir_page.set_redirect_target(movedTarget,
keep_section=True,
save=False)
pywikibot.showDiff(content, redir_page.text)
pywikibot.output(u'Summary - %s' % reason)
if self.user_confirm(
u'Redirect target %s has been moved to %s.\n'
u'Do you want to fix %s?' %
(targetPage, movedTarget, redir_page)):
try:
redir_page.save(reason)
except pywikibot.NoUsername:
pywikibot.output(
u"Page [[%s]] not saved; "
u"sysop privileges required." %
redir_page.title())
except pywikibot.LockedPage:
pywikibot.output(u'%s is locked.' %
redir_page.title())
except pywikibot.OtherPageSaveError:
pywikibot.exception()
else:
done = True
if not done and self.user_confirm(
u'Redirect target %s does not exist.\n'
u'Do you want to delete %s?' %
(targetPage.title(asLink=True),
redir_page.title(asLink=True))):
reason = i18n.twtranslate(self.site,
'redirect-remove-broken')
if self.site.logged_in(sysop=True):
redir_page.delete(reason, prompt=False)
else:
assert targetPage.site == self.site, (
u'target page is on different site %s' %
targetPage.site)
if i18n.twhas_key(self.site,
'redirect-broken-redirect-template'):
pywikibot.output(u"No sysop in user-config.py, "
u"put page to speedy deletion.")
content = redir_page.get(get_redirect=True)
# TODO: Add bot's signature if needed
# Not supported via TW yet
content = i18n.twtranslate(
targetPage.site,
'redirect-broken-redirect-template'
) + "\n" + content
try:
redir_page.put(content, reason)
except pywikibot.PageSaveRelatedError as e:
pywikibot.error(e)
else:
pywikibot.output(
u'No speedy deletion template available')
elif not (self.getOption('delete') or movedTarget):
pywikibot.output(
u'Cannot fix or delete the broken redirect')
except pywikibot.IsRedirectPage:
pywikibot.output(
"Redirect target {0} is also a redirect! {1}".format(
targetPage.title(asLink=True), "Won't delete anything."
if self.getOption('delete') else "Skipping."))
else:
# we successfully get the target page, meaning that
# it exists and is not a redirect: no reason to touch it.
pywikibot.output("Redirect target {0} does exist! {1}".format(
targetPage.title(asLink=True), "Won't delete anything."
if self.getOption('delete') else "Skipping."))
def treat(self, refPage, disambPage):
"""
Parameters:
disambPage - The disambiguation page or redirect we don't want
anything to link to
refPage - A page linking to disambPage
Returns False if the user pressed q to completely quit the program.
Otherwise, returns True.
"""
# TODO: break this function up into subroutines!
dn_template_str = pywikibot.translate(self.mysite, dn_template)
include = False
unlink = False
new_targets = []
try:
text = refPage.get(throttle=False)
ignoreReason = self.checkContents(text)
if ignoreReason:
pywikibot.output(
'\n\nSkipping %s because it contains %s.\n\n' %
(refPage.title(), ignoreReason))
else:
include = True
except pywikibot.IsRedirectPage:
pywikibot.output(u'%s is a redirect to %s' %
(refPage.title(), disambPage.title()))
if disambPage.isRedirectPage():
target = self.alternatives[0]
choice = pywikibot.inputChoice(
u'Do you want to make redirect %s point to %s?' %
(refPage.title(), target), ['yes', 'no'], ['y', 'N'], 'N')
if choice == 'y':
redir_text = '#%s [[%s]]' \
% (self.mysite.redirect(default=True), target)
try:
refPage.put_async(redir_text, comment=self.comment)
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Page not saved: %s' % error.args)
else:
choice = pywikibot.inputChoice(
u'Do you want to work on pages linking to %s?' %
refPage.title(), ['yes', 'no', 'change redirect'],
['y', 'N', 'c'], 'N')
if choice == 'y':
gen = ReferringPageGeneratorWithIgnore(
refPage, self.primary)
preloadingGen = pagegenerators.PreloadingGenerator(gen)
for refPage2 in preloadingGen:
# run until the user selected 'quit'
if not self.treat(refPage2, refPage):
break
elif choice == 'c':
text = refPage.get(throttle=False, get_redirect=True)
include = "redirect"
except pywikibot.NoPage:
pywikibot.output(
u'Page [[%s]] does not seem to exist?! Skipping.' %
refPage.title())
include = False
if include in (True, "redirect"):
# make a backup of the original text so we can show the changes later
original_text = text
n = 0
curpos = 0
dn = False
edited = False
# This loop will run until we have finished the current page
while True:
m = self.linkR.search(text, pos=curpos)
if not m:
if n == 0:
pywikibot.output(u"No changes necessary in %s" %
refPage.title())
return True
else:
# stop loop and save page
break
# Make sure that next time around we will not find this same hit.
curpos = m.start() + 1
try:
foundlink = pywikibot.Link(m.group('title'),
disambPage.site)
except pywikibot.Error:
continue
# ignore interwiki links
if foundlink.site != disambPage.site:
continue
# Check whether the link found is to disambPage.
try:
if foundlink.canonical_title() != disambPage.title():
continue
except pywikibot.Error:
# must be a broken link
pywikibot.log(u"Invalid link [[%s]] in page [[%s]]" %
(m.group('title'), refPage.title()))
continue
n += 1
# how many bytes should be displayed around the current link
context = 60
#there's a {{dn}} here already
already_dn = text[m.end():m.end() + 8].find(
dn_template_str[:4]) > -1
if already_dn and self.dnSkip:
continue
# This loop will run while the user doesn't choose an option
# that will actually change the page
while True:
# Show the title of the page where the link was found.
# Highlight the title in purple.
pywikibot.output(
u"\n\n>>> \03{lightpurple}%s\03{default} <<<" %
refPage.title())
if not self.always:
# at the beginning of the link, start red color.
# at the end of the link, reset the color to default
pywikibot.output(text[max(0,
m.start() -
context):m.start()] +
'\03{lightred}' +
text[m.start():m.end()] +
'\03{default}' +
text[m.end():m.end() + context])
if edited:
choice = pywikibot.input(
u"Option (#, r#, [s]kip link, [e]dit page, [n]ext page, [u]nlink, [q]uit,\n"
u" [t]ag template " + dn_template_str +
",\n"
u" [m]ore context, [l]ist, [a]dd new, x=save in this form):"
)
else:
choice = pywikibot.input(
u"Option (#, r#, [s]kip link, [e]dit page, [n]ext page, [u]nlink, [q]uit,\n"
u" [t]ag template " + dn_template_str +
",\n"
u" [m]ore context, show [d]isambiguation page, [l]ist, [a]dd new):"
)
else:
choice = self.always
if choice in ['a', 'A']:
newAlternative = pywikibot.input(u'New alternative:')
self.alternatives.append(newAlternative)
self.listAlternatives()
elif choice in ['e', 'E']:
editor = editarticle.TextEditor()
newText = editor.edit(text,
jumpIndex=m.start(),
highlight=disambPage.title())
# if user didn't press Cancel
if newText and newText != text:
text = newText
break
elif choice in ['d', 'D']:
editor = editarticle.TextEditor()
if disambPage.isRedirectPage():
disambredir = disambPage.getRedirectTarget()
editor.edit(disambredir.get(),
jumpIndex=m.start(),
highlight=disambredir.title())
else:
editor.edit(disambPage.get(),
jumpIndex=m.start(),
highlight=disambPage.title())
elif choice in ['l', 'L']:
self.listAlternatives()
elif choice in ['m', 'M']:
# show more text around the link we're working on
context *= 2
else:
break
if choice in ['e', 'E']:
# user has edited the page and then pressed 'OK'
edited = True
curpos = 0
continue
elif choice in ['n', 'N']:
# skip this page
if self.primary:
# If run with the -primary argument, skip this
# occurence next time.
self.primaryIgnoreManager.ignore(refPage)
return True
elif choice in ['q', 'Q']:
# quit the program
return False
elif choice in ['s', 'S']:
# Next link on this page
n -= 1
continue
elif choice in ['x', 'X'] and edited:
# Save the page as is
break
# The link looks like this:
# [[page_title|link_text]]trailing_chars
page_title = m.group('title')
link_text = m.group('label')
if not link_text:
# or like this: [[page_title]]trailing_chars
link_text = page_title
if m.group('section') is None:
section = ''
else:
section = m.group('section')
trailing_chars = m.group('linktrail')
if trailing_chars:
link_text += trailing_chars
# '?', '/' for old choice
if choice in ['t', 'T', '?', '/']:
# small chunk of text to search
search_text = text[m.end():m.end() + context]
# figure out where the link (and sentance) ends, put note
# there
end_of_word_match = re.search("\s", search_text)
if end_of_word_match:
position_split = end_of_word_match.start(0)
else:
position_split = 0
#insert dab needed template
text = (text[:m.end() + position_split] + dn_template_str +
text[m.end() + position_split:])
dn = True
continue
elif choice in ['u', 'U']:
# unlink - we remove the section if there's any
text = text[:m.start()] + link_text + text[m.end():]
unlink = True
continue
else:
if len(choice) > 0 and choice[0] == 'r':
# we want to throw away the original link text
replaceit = True
choice = choice[1:]
elif include == "redirect":
replaceit = True
else:
replaceit = False
try:
choice = int(choice)
except ValueError:
pywikibot.output(u"Unknown option")
# step back to ask the user again what to do with the
# current link
curpos -= 1
continue
if choice >= len(self.alternatives) or choice < 0:
pywikibot.output(
u"Choice out of range. Please select a number "
u"between 0 and %i." %
(len(self.alternatives) - 1))
# show list of possible choices
self.listAlternatives()
# step back to ask the user again what to do with the
# current link
curpos -= 1
continue
new_page_title = self.alternatives[choice]
repPl = pywikibot.Page(
pywikibot.Link(new_page_title, disambPage.site))
if (new_page_title[0].isupper() or link_text[0].isupper()):
new_page_title = repPl.title()
else:
new_page_title = repPl.title()
new_page_title = (new_page_title[0].lower() +
new_page_title[1:])
if new_page_title not in new_targets:
new_targets.append(new_page_title)
if replaceit and trailing_chars:
newlink = "[[%s%s]]%s" % (new_page_title, section,
trailing_chars)
elif replaceit or (new_page_title == link_text
and not section):
newlink = "[[%s]]" % new_page_title
# check if we can create a link with trailing characters
# instead of a pipelink
elif ((len(new_page_title) <= len(link_text))
and (firstcap(link_text[:len(new_page_title)])
== firstcap(new_page_title))
and (re.sub(self.trailR, '',
link_text[len(new_page_title):]) == '')
and (not section)):
newlink = "[[%s]]%s" \
% (link_text[:len(new_page_title)],
link_text[len(new_page_title):])
else:
newlink = "[[%s%s|%s]]" \
% (new_page_title, section, link_text)
text = text[:m.start()] + newlink + text[m.end():]
continue
pywikibot.output(text[max(0, m.start() - 30):m.end() + 30])
if text == original_text:
pywikibot.output(u'\nNo changes have been made:\n')
else:
pywikibot.output(u'\nThe following changes have been made:\n')
pywikibot.showDiff(original_text, text)
pywikibot.output(u'')
# save the page
self.setSummaryMessage(disambPage, new_targets, unlink, dn)
try:
refPage.put_async(text, comment=self.comment)
except pywikibot.LockedPage:
pywikibot.output(u'Page not saved: page is locked')
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Page not saved: %s' % error.args)
return True
def main(*args):
"""
Process command line arguments and perform task.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: str
"""
# Loading the comments
global categoryToCheck, project_inserted
# always, define a generator to understand if the user sets one,
# defining what's genFactory
always = False
generator = False
show = False
moveBlockCheck = False
protectedpages = False
protectType = 'edit'
namespace = 0
# Process global args and prepare generator args parser
local_args = pywikibot.handle_args(args)
genFactory = pagegenerators.GeneratorFactory()
# Process local args
for arg in local_args:
option, sep, value = arg.partition(':')
if option == '-always':
always = True
elif option == '-move':
moveBlockCheck = True
elif option == '-show':
show = True
elif option in ('-protectedpages', '-moveprotected'):
protectedpages = True
if option == '-moveprotected':
protectType = 'move'
if value:
namespace = int(value)
else:
genFactory.handleArg(arg)
if config.mylang not in project_inserted:
pywikibot.output('Your project is not supported by this script.\n'
'You have to edit the script and add it!')
return
site = pywikibot.Site()
if protectedpages:
generator = site.protectedpages(namespace=namespace, type=protectType)
# Take the right templates to use, the category and the comment
TSP = i18n.translate(site, templateSemiProtection)
TTP = i18n.translate(site, templateTotalProtection)
TSMP = i18n.translate(site, templateSemiMoveProtection)
TTMP = i18n.translate(site, templateTotalMoveProtection)
TNR = i18n.translate(site, templateNoRegex)
TU = i18n.translate(site, templateUnique)
categories = i18n.translate(site, categoryToCheck)
commentUsed = i18n.twtranslate(site, 'blockpageschecker-summary')
if not generator:
generator = genFactory.getCombinedGenerator()
if not generator:
generator = []
pywikibot.output('Loading categories...')
# Define the category if no other generator has been set
for CAT in categories:
cat = pywikibot.Category(site, CAT)
# Define the generator
gen = pagegenerators.CategorizedPageGenerator(cat)
for pageCat in gen:
generator.append(pageCat)
pywikibot.output('Categories loaded, start!')
# Main Loop
if not genFactory.nopreload:
generator = pagegenerators.PreloadingGenerator(generator, groupsize=60)
for page in generator:
pagename = page.title(as_link=True)
pywikibot.output('Loading %s...' % pagename)
try:
text = page.text
except pywikibot.NoPage:
pywikibot.output("%s doesn't exist! Skipping..." % pagename)
continue
except pywikibot.IsRedirectPage:
pywikibot.output('{} is a redirect! Skipping...'.format(pagename))
if show:
showQuest(page)
continue
# FIXME: This check does not work :
# PreloadingGenerator cannot set correctly page.editRestriction
# (see bug T57322)
# if not page.has_permission():
# pywikibot.output(
# "%s is sysop-protected : this account can't edit "
# "it! Skipping..." % pagename)
# continue
restrictions = page.protection()
try:
editRestr = restrictions['edit']
except KeyError:
editRestr = None
if not page.has_permission():
pywikibot.output('%s is protected: '
"this account can't edit it! Skipping..." %
pagename)
continue
# Understand, according to the template in the page, what should be the
# protection and compare it with what there really is.
TemplateInThePage = understandBlock(text, TTP, TSP, TSMP, TTMP, TU)
# Only to see if the text is the same or not...
oldtext = text
# keep track of the changes for each step (edit then move)
changes = -1
if not editRestr:
# page is not edit-protected
# Deleting the template because the page doesn't need it.
if not (TTP or TSP):
raise pywikibot.Error(
'This script is not localized to use it on \n{0}. '
'Missing "templateSemiProtection" or'
'"templateTotalProtection"'.format(site.sitename))
if TU:
replaceToPerform = '|'.join(TTP + TSP + TU)
else:
replaceToPerform = '|'.join(TTP + TSP)
text, changes = re.subn(
'<noinclude>(%s)</noinclude>' % replaceToPerform, '', text)
if changes == 0:
text, changes = re.subn('(%s)' % replaceToPerform, '', text)
msg = 'The page is editable for all'
if not moveBlockCheck:
msg += ', deleting the template..'
pywikibot.output(msg + '.')
elif editRestr[0] == 'sysop':
# total edit protection
if (TemplateInThePage[0] == 'sysop-total' and TTP) or \
(TemplateInThePage[0] == 'unique' and TU):
msg = 'The page is protected to the sysop'
if not moveBlockCheck:
msg += ', skipping...'
pywikibot.output(msg)
else:
if not TNR or TU and not TNR[4] or not (TU or TNR[1]):
raise pywikibot.Error(
'This script is not localized to use it on \n{0}. '
'Missing "templateNoRegex"'.format(site.sitename))
pywikibot.output('The page is protected to the sysop, but the '
'template seems not correct. Fixing...')
if TU:
text, changes = re.subn(TemplateInThePage[1], TNR[4], text)
else:
text, changes = re.subn(TemplateInThePage[1], TNR[1], text)
elif TSP or TU:
# implicitly editRestr[0] = 'autoconfirmed', edit-Semi-protection
if TemplateInThePage[0] == 'autoconfirmed-total' or \
TemplateInThePage[0] == 'unique':
msg = 'The page is editable only for the autoconfirmed users'
if not moveBlockCheck:
msg += ', skipping...'
pywikibot.output(msg)
else:
if not TNR or TU and not TNR[4] or not (TU or TNR[1]):
raise pywikibot.Error(
'This script is not localized to use it on \n{0}. '
'Missing "templateNoRegex"'.format(site.sitename))
pywikibot.output('The page is editable only for the '
'autoconfirmed users, but the template '
'seems not correct. Fixing...')
if TU:
text, changes = re.subn(TemplateInThePage[1], TNR[4], text)
else:
text, changes = re.subn(TemplateInThePage[1], TNR[0], text)
if changes == 0:
# We tried to fix edit-protection templates, but it did not work.
pywikibot.warning('No edit-protection template could be found')
if moveBlockCheck and changes > -1:
# checking move protection now
try:
moveRestr = restrictions['move']
except KeyError:
moveRestr = False
changes = -1
if not moveRestr:
pywikibot.output('The page is movable for all, deleting the '
'template...')
# Deleting the template because the page doesn't need it.
if TU:
replaceToPerform = '|'.join(TSMP + TTMP + TU)
else:
replaceToPerform = '|'.join(TSMP + TTMP)
text, changes = re.subn(
'<noinclude>(%s)</noinclude>' % replaceToPerform, '', text)
if changes == 0:
text, changes = re.subn('({})'.format(replaceToPerform),
'', text)
elif moveRestr[0] == 'sysop':
# move-total-protection
if (TemplateInThePage[0] == 'sysop-move' and TTMP) or \
(TemplateInThePage[0] == 'unique' and TU):
pywikibot.output('The page is protected from moving to '
'the sysop, skipping...')
if TU:
# no changes needed, better to revert the old text.
text = oldtext
else:
pywikibot.output('The page is protected from moving to '
'the sysop, but the template seems not '
'correct. Fixing...')
if TU:
text, changes = re.subn(TemplateInThePage[1], TNR[4],
text)
else:
text, changes = re.subn(TemplateInThePage[1], TNR[3],
text)
elif TSMP or TU:
# implicitly moveRestr[0] = 'autoconfirmed',
# move-semi-protection
if TemplateInThePage[0] == 'autoconfirmed-move' or \
TemplateInThePage[0] == 'unique':
pywikibot.output('The page is movable only for the '
'autoconfirmed users, skipping...')
if TU:
# no changes needed, better to revert the old text.
text = oldtext
else:
pywikibot.output('The page is movable only for the '
'autoconfirmed users, but the template '
'seems not correct. Fixing...')
if TU:
text, changes = re.subn(TemplateInThePage[1], TNR[4],
text)
else:
text, changes = re.subn(TemplateInThePage[1], TNR[2],
text)
if changes == 0:
# We tried to fix move-protection templates but it did not work
pywikibot.warning('No move-protection template could be found')
if oldtext != text:
# Ok, asking if the change has to be performed and do it if yes.
pywikibot.output(
color_format('\n\n>>> {lightpurple}{0}{default} <<<',
page.title()))
pywikibot.showDiff(oldtext, text)
if not always:
choice = pywikibot.input_choice(
'Do you want to accept these '
'changes?', [('Yes', 'y'), ('No', 'n'), ('All', 'a')], 'n')
if choice == 'a':
always = True
if always or choice == 'y':
save_page(page, text, commentUsed)
def fix_1_double_redirect(self, redir_name):
"""Treat one double redirect."""
if isinstance(redir_name, basestring):
redir = pywikibot.Page(self.site, redir_name)
else:
redir = redir_name
# Show the title of the page we're working on.
# Highlight the title in purple.
pywikibot.output(
color_format('\n\n>>> {lightpurple}{0}{default} <<<',
redir.title()))
newRedir = redir
redirList = [] # bookkeeping to detect loops
while True:
redirList.append(
u'%s:%s' %
(newRedir.site.lang, newRedir.title(withSection=False)))
try:
targetPage = newRedir.getRedirectTarget()
except pywikibot.IsNotRedirectPage:
if len(redirList) == 1:
pywikibot.output(u'Skipping: Page %s is not a redirect.' %
redir.title(asLink=True))
break # do nothing
elif len(redirList) == 2:
pywikibot.output(
u'Skipping: Redirect target %s is not a redirect.' %
newRedir.title(asLink=True))
break # do nothing
else:
pass # target found
except pywikibot.SectionError:
pywikibot.warning(
u"Redirect target section %s doesn't exist." %
newRedir.title(asLink=True))
except (pywikibot.CircularRedirect,
pywikibot.InterwikiRedirectPage) as e:
pywikibot.exception(e)
pywikibot.output(u"Skipping %s." % newRedir)
break
except pywikibot.BadTitle as e:
# str(e) is in the format 'BadTitle: [[Foo]]'
pywikibot.warning(
u'Redirect target %s is not a valid page title.' %
str(e)[10:])
break
except pywikibot.NoPage:
if len(redirList) == 1:
pywikibot.output(u'Skipping: Page %s does not exist.' %
redir.title(asLink=True))
break
else:
if self.getOption('always'):
pywikibot.output(
u"Skipping: Redirect target %s doesn't exist." %
newRedir.title(asLink=True))
break # skip if automatic
else:
pywikibot.warning(
u"Redirect target %s doesn't exist." %
newRedir.title(asLink=True))
except pywikibot.ServerError:
pywikibot.output(u'Skipping due to server error: '
u'No textarea found')
break
else:
pywikibot.output(u' Links to: %s.' %
targetPage.title(asLink=True))
try:
mw_msg = targetPage.site.mediawiki_message(
'wikieditor-toolbar-tool-redirect-example')
except KeyError:
pass
else:
if targetPage.title() == mw_msg:
pywikibot.output(
u"Skipping toolbar example: Redirect source is "
u"potentially vandalized.")
break
# watch out for redirect loops
if redirList.count(u'%s:%s' %
(targetPage.site.lang,
targetPage.title(withSection=False))):
pywikibot.warning(
u'Redirect target %s forms a redirect loop.' %
targetPage.title(asLink=True))
break # FIXME: doesn't work. edits twice!
try:
content = targetPage.get(get_redirect=True)
except pywikibot.SectionError:
content_page = pywikibot.Page(
targetPage.site,
targetPage.title(withSection=False))
content = content_page.get(get_redirect=True)
if i18n.twhas_key(
targetPage.site,
'redirect-broken-redirect-template') and \
i18n.twhas_key(targetPage.site,
'redirect-remove-loop'):
pywikibot.output(u"Tagging redirect for deletion")
# Delete the two redirects
content = i18n.twtranslate(
targetPage.site,
'redirect-broken-redirect-template'
) + "\n" + content
summ = i18n.twtranslate(targetPage.site,
'redirect-remove-loop')
targetPage.put(content, summ)
redir.put(content, summ)
break
else: # redirect target found
if targetPage.isStaticRedirect():
pywikibot.output(
u" Redirect target is STATICREDIRECT.")
pass
else:
newRedir = targetPage
continue
try:
oldText = redir.get(get_redirect=True)
except pywikibot.BadTitle:
pywikibot.output(u"Bad Title Error")
break
if self.is_repo and redir.namespace() == self.repo.item_namespace:
redir = pywikibot.ItemPage(self.repo, redir.title())
targetPage = pywikibot.ItemPage(self.repo, targetPage.title())
pywikibot.output('Fixing double item redirect')
redir.set_redirect_target(targetPage)
break
redir.set_redirect_target(targetPage,
keep_section=True,
save=False)
summary = i18n.twtranslate(self.site, 'redirect-fix-double',
{'to': targetPage.title(asLink=True)})
pywikibot.showDiff(oldText, redir.text)
if self.user_confirm(u'Do you want to accept the changes?'):
try:
redir.save(summary)
except pywikibot.LockedPage:
pywikibot.output(u'%s is locked.' % redir.title())
except pywikibot.SpamfilterError as error:
pywikibot.output(
u"Saving page [[%s]] prevented by spam filter: %s" %
(redir.title(), error.url))
except pywikibot.PageNotSaved as error:
pywikibot.output(u"Saving page [[%s]] failed: %s" %
(redir.title(), error))
except pywikibot.NoUsername:
pywikibot.output(
u"Page [[%s]] not saved; sysop privileges required." %
redir.title())
except pywikibot.Error as error:
pywikibot.output(
u"Unexpected error occurred trying to save [[%s]]: %s"
% (redir.title(), error))
break
def treat(self, page):
"""Work on each page retrieved from generator."""
try:
original_text = page.text
except InvalidPageError:
pywikibot.exception()
return
applied = set()
new_text = original_text
last_text = None
context = 0
while True:
if self.isTextExcepted(new_text):
pywikibot.output(
'Skipping {} because it contains text '
'that is on the exceptions list.'.format(page))
return
while new_text != last_text:
last_text = new_text
new_text = self.apply_replacements(last_text, applied, page)
if not self.opt.recursive:
break
if new_text == original_text:
pywikibot.output('No changes were necessary in ' +
page.title(as_link=True))
return
if self.opt.addcat:
# Fetch only categories in wikitext, otherwise the others
# will be explicitly added.
cats = textlib.getCategoryLinks(new_text, site=page.site)
if self.opt.addcat not in cats:
cats.append(self.opt.addcat)
new_text = textlib.replaceCategoryLinks(new_text,
cats,
site=page.site)
# Show the title of the page we're working on.
# Highlight the title in purple.
self.current_page = page
pywikibot.showDiff(original_text, new_text, context=context)
if self.opt.always:
break
choice = pywikibot.input_choice(
'Do you want to accept these changes?',
[('Yes', 'y'), ('No', 'n'), ('Edit original', 'e'),
('edit Latest', 'l'), ('open in Browser', 'b'),
('More context', 'm'), ('All', 'a')],
default='N')
if choice == 'm':
context = context * 3 if context else 3
continue
if choice in ('e', 'l'):
text_editor = editor.TextEditor()
edit_text = original_text if choice == 'e' else new_text
as_edited = text_editor.edit(edit_text)
# if user didn't press Cancel
if as_edited and as_edited != new_text:
new_text = as_edited
if choice == 'l':
# prevent changes from being applied again
last_text = new_text
continue
if choice == 'b':
pywikibot.bot.open_webbrowser(page)
try:
original_text = page.get(get_redirect=True, force=True)
except NoPageError:
pywikibot.output('Page {} has been deleted.'.format(
page.title()))
break
new_text = original_text
last_text = None
continue
if choice == 'a':
self.opt.always = True
if choice == 'y':
self.save(page,
original_text,
new_text,
applied,
show_diff=False,
asynchronous=True)
# choice must be 'N'
break
if self.opt.always and new_text != original_text:
self.save(page,
original_text,
new_text,
applied,
show_diff=False,
asynchronous=False)
print('Adding user to blacklist')
blackdata['targets'].append({'title': 'User:{}'.format(USERNAME)})
summary = cfg['summary_remove'].format('User:{}'.format(USERNAME), DIFFID)
edit_black = True
elif ACTION == ACTION_BLACK_ADD and on_white:
print('Removing user from whitelist')
for i in range(len(whitedata['targets']) - 1, -1, -1):
if re.search(USERREGEX, whitedata['targets'][i]['title']):
del whitedata['targets'][i]
summary = cfg['summary_black_add'].format('User:{}'.format(USERNAME),
DIFFID)
edit_white = True
else:
print('Nothing to do')
exit()
if edit_black:
text = json.dumps(blackdata, ensure_ascii=False, indent=4)
pywikibot.showDiff(blackpage.text, text)
blackpage.text = text
print(summary)
blackpage.save(summary=summary, minor=False)
elif edit_white:
text = json.dumps(whitedata, ensure_ascii=False, indent=4)
pywikibot.showDiff(whitepage.text, text)
whitepage.text = text
print(summary)
whitepage.save(summary=summary, minor=False)
def save(self):
# Init empty item
data = {
'labels': {},
'sitelinks': {},
'claims': [],
}
# Set lebels
data['labels']['zh-tw'] = {'language': 'zh-tw', 'value': self.title}
# Set sitelinks
data['sitelinks']['tnfshwiki'] = {
'site': 'tnfshwiki',
'title': self.title,
}
# 性質
new_claim = pywikibot.page.Claim(datasite, 'P1')
new_claim.setTarget(pywikibot.ItemPage(datasite, 'Q67')) # 老師
data['claims'].append(new_claim.toJSON())
# 性別
if self.gender_id:
new_claim = pywikibot.page.Claim(datasite, 'P31')
new_claim.setTarget(pywikibot.ItemPage(datasite, self.gender_id))
data['claims'].append(new_claim.toJSON())
# 科目
if self.subject_id:
new_claim = pywikibot.page.Claim(datasite, 'P21')
new_claim.setTarget(pywikibot.ItemPage(datasite, self.subject_id))
data['claims'].append(new_claim.toJSON())
# 行政職位
for job in self.jobs_id:
new_claim = pywikibot.page.Claim(datasite, 'P28')
new_claim.setTarget(pywikibot.ItemPage(datasite, job[0]))
if job[1]:
qualifier = pywikibot.page.Claim(datasite, 'P27') # 學年度
qualifier.setTarget(
pywikibot.ItemPage(datasite, self.YEAR_QID[job[1]]))
new_claim.addQualifier(qualifier)
data['claims'].append(new_claim.toJSON())
# 導師
for class1 in self.class_id:
new_claim = pywikibot.page.Claim(datasite, 'P25')
new_claim.setTarget(class1[0])
qualifier = pywikibot.page.Claim(datasite, 'P27') # 學年度
qualifier.setTarget(
pywikibot.ItemPage(datasite, self.YEAR_QID[class1[1]]))
new_claim.addQualifier(qualifier)
data['claims'].append(new_claim.toJSON())
# 任職狀況
if self.live_id:
new_claim = pywikibot.page.Claim(datasite, 'P22')
new_claim.setTarget(pywikibot.ItemPage(datasite, self.live_id))
data['claims'].append(new_claim.toJSON())
# 別稱
for nickname in self.nickname:
new_claim = pywikibot.page.Claim(datasite, 'P30')
new_claim.setTarget(nickname)
data['claims'].append(new_claim.toJSON())
# 學歷
for status in self.edustatus:
new_claim = pywikibot.page.Claim(datasite, 'P29')
new_claim.setTarget(status)
data['claims'].append(new_claim.toJSON())
# 圖片
if self.image:
new_claim = pywikibot.page.Claim(datasite, 'P33')
new_claim.setTarget(self.image)
if self.imageinfo:
qualifier = pywikibot.page.Claim(datasite, 'P34') # 圖片資訊
qualifier.setTarget(self.imageinfo)
new_claim.addQualifier(qualifier)
data['claims'].append(new_claim.toJSON())
print(json.dumps(data['labels'], indent=4, ensure_ascii=False))
for claim in data['claims']:
print(claim['mainsnak']['property'], claim['mainsnak']['datatype'],
claim['mainsnak']['datavalue'])
summary = '從[[{}]]匯入老師資料'.format(self.title)
print('Create item with summary: {}'.format(summary))
if self.image:
input('Check image')
item = datasite.editEntity({}, data, summary=summary)
print(item['entity']['id'])
newitemid = item['entity']['id']
text = self.page.text
text = re.sub(r'{{(簡介 老師|Infobox teacher)[\s\S]+?}}\n*',
r'{{老師資訊框}}\n', text)
text = re.sub(r'{{Expand\|.+}}\n*', '', text)
pywikibot.showDiff(self.page.text, text)
summary = '資料已匯入至[[Item:{}]]'.format(newitemid)
print('Save with summary: {}'.format(summary))
self.page.text = text
self.page.save(summary=summary, minor=False, asynchronous=True)
def add_text(page,
addText,
summary=None,
regexSkip=None,
regexSkipUrl=None,
always=False,
up=False,
putText=True,
oldTextGiven=None,
reorderEnabled=True,
create=False):
"""
Add text to a page.
@rtype: tuple of (text, newtext, always)
"""
site = page.site
if not summary:
summary = i18n.twtranslate(site, 'add_text-adding',
{'adding': addText[:200]})
if putText:
pywikibot.output('Loading {}...'.format(page.title()))
text = get_text(page, oldTextGiven, create)
if text is None:
return (False, False, always)
# Understand if the bot has to skip the page or not
# In this way you can use both -except and -excepturl
if regexSkipUrl is not None:
url = page.full_url()
result = re.findall(regexSkipUrl, site.getUrl(url))
if result != []:
pywikibot.output('Exception! regex (or word) used with -exceptUrl '
'is in the page. Skip!\n'
'Match was: {}'.format(result))
return (False, False, always)
if regexSkip is not None:
result = re.findall(regexSkip, text)
if result != []:
pywikibot.output('Exception! regex (or word) used with -except '
'is in the page. Skip!\n'
'Match was: {}'.format(result))
return (False, False, always)
# If not up, text put below
if not up:
newtext = text
# Translating the \\n into binary \n
addText = addText.replace('\\n', config.line_separator)
if (reorderEnabled):
# Getting the categories
categoriesInside = textlib.getCategoryLinks(newtext, site)
# Deleting the categories
newtext = textlib.removeCategoryLinks(newtext, site)
# Getting the interwiki
interwikiInside = textlib.getLanguageLinks(newtext, site)
# Removing the interwiki
newtext = textlib.removeLanguageLinks(newtext, site)
# Adding the text
newtext += '{}{}'.format(config.line_separator, addText)
# Reputting the categories
newtext = textlib.replaceCategoryLinks(newtext, categoriesInside,
site, True)
# Adding the interwiki
newtext = textlib.replaceLanguageLinks(newtext, interwikiInside,
site)
else:
newtext += '{}{}'.format(config.line_separator, addText)
else:
newtext = addText + config.line_separator + text
if putText and text != newtext:
pywikibot.output(
color_format('\n\n>>> {lightpurple}{0}{default} <<<',
page.title()))
pywikibot.showDiff(text, newtext)
# Let's put the changes.
error_count = 0
while True:
# If someone load it as module, maybe it's not so useful to put the
# text in the page
if not putText:
return (text, newtext, always)
if not always:
try:
choice = pywikibot.input_choice(
'Do you want to accept these changes?',
[('Yes', 'y'), ('No', 'n'), ('All', 'a'),
('open in Browser', 'b')], 'n')
except QuitKeyboardInterrupt:
sys.exit('User quit bot run.')
if choice == 'a':
always = True
elif choice == 'n':
return (False, False, always)
elif choice == 'b':
pywikibot.bot.open_webbrowser(page)
if always or choice == 'y':
result = put_text(page,
newtext,
summary,
error_count,
asynchronous=not always)
if result is not None:
return (result, result, always)
error_count += 1
def treat(self, refPage, disambPage):
"""
Treat a page.
Parameters:
disambPage - The disambiguation page or redirect we don't want
anything to link to
refPage - A page linking to disambPage
Returns False if the user pressed q to completely quit the program.
Otherwise, returns True.
"""
# TODO: break this function up into subroutines!
self.current_page = refPage
include = False
unlink_counter = 0
new_targets = []
try:
text = refPage.get()
ignoreReason = self.checkContents(text)
if ignoreReason:
pywikibot.output(
'\n\nSkipping %s because it contains %s.\n\n' %
(refPage.title(), ignoreReason))
else:
include = True
except pywikibot.IsRedirectPage:
pywikibot.output(u'%s is a redirect to %s' %
(refPage.title(), disambPage.title()))
if disambPage.isRedirectPage():
target = self.alternatives[0]
if pywikibot.input_yn(u'Do you want to make redirect %s point '
'to %s?' % (refPage.title(), target),
default=False,
automatic_quit=False):
redir_text = '#%s [[%s]]' \
% (self.mysite.redirect(), target)
try:
refPage.put_async(redir_text, summary=self.comment)
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Page not saved: %s' % error.args)
else:
choice = pywikibot.input_choice(
u'Do you want to work on pages linking to %s?' %
refPage.title(), [('yes', 'y'), ('no', 'n'),
('change redirect', 'c')],
'n',
automatic_quit=False)
if choice == 'y':
gen = ReferringPageGeneratorWithIgnore(
refPage, self.primary)
preloadingGen = pagegenerators.PreloadingGenerator(gen)
for refPage2 in preloadingGen:
# run until the user selected 'quit'
if not self.treat(refPage2, refPage):
break
elif choice == 'c':
text = refPage.get(get_redirect=True)
include = "redirect"
except pywikibot.NoPage:
pywikibot.output(
u'Page [[%s]] does not seem to exist?! Skipping.' %
refPage.title())
include = False
if include in (True, "redirect"):
# make a backup of the original text so we can show the changes later
original_text = text
n = 0
curpos = 0
dn = False
edited = False
# This loop will run until we have finished the current page
while True:
m = self.linkR.search(text, pos=curpos)
if not m:
if n == 0:
pywikibot.output(u"No changes necessary in %s" %
refPage.title())
return True
else:
# stop loop and save page
break
# Make sure that next time around we will not find this same hit.
curpos = m.start() + 1
try:
foundlink = pywikibot.Link(m.group('title'),
disambPage.site)
foundlink.parse()
except pywikibot.Error:
continue
# ignore interwiki links
if foundlink.site != disambPage.site:
continue
# Check whether the link found is to disambPage.
try:
if foundlink.canonical_title() != disambPage.title():
continue
except pywikibot.Error:
# must be a broken link
pywikibot.log(u"Invalid link [[%s]] in page [[%s]]" %
(m.group('title'), refPage.title()))
continue
n += 1
# how many bytes should be displayed around the current link
context = 60
# check if there's a dn-template here already
if (self.dnSkip and self.dn_template_str
and self.dn_template_str[:-2]
in text[m.end():m.end() + len(self.dn_template_str) +
8]):
continue
edit = EditOption('edit page', 'e', text, m.start(),
disambPage.title())
context_option = HighlightContextOption('more context',
'm',
text,
60,
start=m.start(),
end=m.end())
context_option.before_question = True
options = [
ListOption(self.alternatives, ''),
ListOption(self.alternatives, 'r'),
StandardOption('skip link', 's'), edit,
StandardOption('next page', 'n'),
StandardOption('unlink', 'u')
]
if self.dn_template_str:
# '?', '/' for old choice
options += [
AliasOption('tag template %s' % self.dn_template_str,
['t', '?', '/'])
]
options += [context_option]
if not edited:
options += [
ShowPageOption('show disambiguation page', 'd',
m.start(), disambPage)
]
options += [
OutputProxyOption('list', 'l',
SequenceOutputter(self.alternatives)),
AddAlternativeOption('add new', 'a',
SequenceOutputter(self.alternatives))
]
if edited:
options += [StandardOption('save in this form', 'x')]
# TODO: Output context on each question
answer = pywikibot.input_choice('Option',
options,
default=self.always)
if answer == 'x':
assert edited, 'invalid option before editing'
break
elif answer == 's':
n -= 1 # TODO what's this for?
continue
elif answer == 'e':
text = edit.new_text
edited = True
curpos = 0
continue
elif answer == 'n':
# skip this page
if self.primary:
# If run with the -primary argument, skip this
# occurrence next time.
self.primaryIgnoreManager.ignore(refPage)
return True
# The link looks like this:
# [[page_title|link_text]]trailing_chars
page_title = m.group('title')
link_text = m.group('label')
if not link_text:
# or like this: [[page_title]]trailing_chars
link_text = page_title
if m.group('section') is None:
section = ''
else:
section = m.group('section')
trailing_chars = m.group('linktrail')
if trailing_chars:
link_text += trailing_chars
if answer == 't':
assert self.dn_template_str
# small chunk of text to search
search_text = text[m.end():m.end() + context]
# figure out where the link (and sentance) ends, put note
# there
end_of_word_match = re.search(r'\s', search_text)
if end_of_word_match:
position_split = end_of_word_match.start(0)
else:
position_split = 0
# insert dab needed template
text = (text[:m.end() + position_split] +
self.dn_template_str +
text[m.end() + position_split:])
dn = True
continue
elif answer == 'u':
# unlink - we remove the section if there's any
text = text[:m.start()] + link_text + text[m.end():]
unlink_counter += 1
continue
else:
# Check that no option from above was missed
assert isinstance(answer, tuple), 'only tuple answer left.'
assert answer[0] in ['r', ''], 'only valid tuple answers.'
if answer[0] == 'r':
# we want to throw away the original link text
replaceit = link_text == page_title
elif include == "redirect":
replaceit = True
else:
replaceit = False
new_page_title = answer[1]
repPl = pywikibot.Page(
pywikibot.Link(new_page_title, disambPage.site))
if (new_page_title[0].isupper() or link_text[0].isupper()):
new_page_title = repPl.title()
else:
new_page_title = repPl.title()
new_page_title = first_lower(new_page_title)
if new_page_title not in new_targets:
new_targets.append(new_page_title)
if replaceit and trailing_chars:
newlink = "[[%s%s]]%s" % (new_page_title, section,
trailing_chars)
elif replaceit or (new_page_title == link_text
and not section):
newlink = "[[%s]]" % new_page_title
# check if we can create a link with trailing characters
# instead of a pipelink
elif ((len(new_page_title) <= len(link_text))
and (firstcap(link_text[:len(new_page_title)])
== firstcap(new_page_title))
and (re.sub(self.trailR, '',
link_text[len(new_page_title):]) == '')
and (not section)):
newlink = "[[%s]]%s" \
% (link_text[:len(new_page_title)],
link_text[len(new_page_title):])
else:
newlink = "[[%s%s|%s]]" \
% (new_page_title, section, link_text)
text = text[:m.start()] + newlink + text[m.end():]
continue
pywikibot.output(text[max(0, m.start() - 30):m.end() + 30])
if text == original_text:
pywikibot.output(u'\nNo changes have been made:\n')
else:
pywikibot.output(u'\nThe following changes have been made:\n')
pywikibot.showDiff(original_text, text)
pywikibot.output(u'')
# save the page
self.setSummaryMessage(disambPage, new_targets, unlink_counter,
dn)
try:
refPage.put_async(text, summary=self.comment)
except pywikibot.LockedPage:
pywikibot.output(u'Page not saved: page is locked')
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Page not saved: %s' % error.args)
return True
def run(self):
while True:
wait = False
now = time.strftime("%d %b %Y %H:%M:%S (UTC)", time.gmtime())
if self.getOption('page'):
localSandboxTitle = self.getOption('page')
else:
localSandboxTitle = i18n.translate(self.site, sandboxTitle)
if isinstance(localSandboxTitle, list):
titles = localSandboxTitle
else:
titles = [localSandboxTitle]
for title in titles:
sandboxPage = pywikibot.Page(self.site, title)
pywikibot.output(u'Preparing to process sandbox page %s' %
sandboxPage.title(asLink=True))
try:
text = sandboxPage.get()
if not self.getOption('text'):
translatedContent = i18n.translate(self.site, content)
else:
translatedContent = self.getOption('text')
if self.getOption('summary'):
translatedMsg = self.getOption('summary')
else:
translatedMsg = i18n.twtranslate(
self.site, 'clean_sandbox-cleaned')
subst = 'subst:' in translatedContent
pos = text.find(translatedContent.strip())
if text.strip() == translatedContent.strip():
pywikibot.output(
u'The sandbox is still clean, no change necessary.'
)
elif subst and \
sandboxPage.userName() == self.site.user():
pywikibot.output(
u'The sandbox might be clean, no change necessary.'
)
elif pos != 0 and not subst:
if self.getOption('user'):
endpos = pos + len(translatedContent.strip())
if (pos < 0) or (endpos == len(text)):
pywikibot.output(
u'The user sandbox is still '
u'clean, no change necessary.')
else:
sandboxPage.put(text[:endpos], translatedMsg)
pywikibot.showDiff(text, text[:endpos])
pywikibot.output(
u'Standard content was changed, user '
u'sandbox cleaned.')
else:
sandboxPage.put(translatedContent, translatedMsg)
pywikibot.showDiff(text, translatedContent)
pywikibot.output(u'Standard content was changed, '
u'sandbox cleaned.')
else:
edit_delta = (datetime.datetime.utcnow() -
sandboxPage.editTime())
delta = self.getOption('delay_td') - edit_delta
# Is the last edit more than 'delay' minutes ago?
if delta <= datetime.timedelta(0):
sandboxPage.put(translatedContent, translatedMsg)
pywikibot.showDiff(text, translatedContent)
pywikibot.output(u'Standard content was changed, '
u'sandbox cleaned.')
else: # wait for the rest
pywikibot.output(
u'Sandbox edited %.1f minutes ago...' %
(edit_delta.seconds / 60.0))
pywikibot.output(u'Sleeping for %d minutes.' %
(delta.seconds / 60))
time.sleep(delta.seconds)
wait = True
except pywikibot.EditConflict:
pywikibot.output(
u'*** Loading again because of edit conflict.\n')
except pywikibot.NoPage:
pywikibot.output(
u'*** The sandbox is not existent, skipping.')
continue
if self.getOption('no_repeat'):
pywikibot.output(u'\nDone.')
return
elif not wait:
if self.getOption('hours') < 1.0:
pywikibot.output('\nSleeping %s minutes, now %s' %
((self.getOption('hours') * 60), now))
else:
pywikibot.output('\nSleeping %s hours, now %s' %
(self.getOption('hours'), now))
time.sleep(self.getOption('hours') * 60 * 60)
def add_text(page,
addText,
summary=None,
regexSkip=None,
regexSkipUrl=None,
always=False,
up=False,
putText=True,
oldTextGiven=None,
reorderEnabled=True,
create=False):
"""
Add text to a page.
@rtype: tuple of (text, newtext, always)
"""
site = page.site
if not summary:
summary = i18n.twtranslate(site, 'add_text-adding',
{'adding': addText[:200]})
errorCount = 0
if putText:
pywikibot.output(u'Loading %s...' % page.title())
if oldTextGiven is None:
try:
text = page.get()
except pywikibot.NoPage:
if create:
pywikibot.output(u"%s doesn't exist, creating it!" %
page.title())
text = u''
else:
pywikibot.output(u"%s doesn't exist, skip!" % page.title())
return (False, False, always)
except pywikibot.IsRedirectPage:
pywikibot.output(u"%s is a redirect, skip!" % page.title())
return (False, False, always)
else:
text = oldTextGiven
# Understand if the bot has to skip the page or not
# In this way you can use both -except and -excepturl
if regexSkipUrl is not None:
url = page.full_url()
result = re.findall(regexSkipUrl, site.getUrl(url))
if result != []:
pywikibot.output('Exception! regex (or word) used with -exceptUrl '
'is in the page. Skip!\n'
'Match was: %s' % result)
return (False, False, always)
if regexSkip is not None:
result = re.findall(regexSkip, text)
if result != []:
pywikibot.output('Exception! regex (or word) used with -except '
'is in the page. Skip!\n'
'Match was: %s' % result)
return (False, False, always)
# If not up, text put below
if not up:
newtext = text
# Translating the \\n into binary \n
addText = addText.replace('\\n', config.line_separator)
if (reorderEnabled):
# Getting the categories
categoriesInside = textlib.getCategoryLinks(newtext, site)
# Deleting the categories
newtext = textlib.removeCategoryLinks(newtext, site)
# Getting the interwiki
interwikiInside = textlib.getLanguageLinks(newtext, site)
# Removing the interwiki
newtext = textlib.removeLanguageLinks(newtext, site)
# Adding the text
newtext += u"%s%s" % (config.line_separator, addText)
# Reputting the categories
newtext = textlib.replaceCategoryLinks(newtext, categoriesInside,
site, True)
# Adding the interwiki
newtext = textlib.replaceLanguageLinks(newtext, interwikiInside,
site)
else:
newtext += u"%s%s" % (config.line_separator, addText)
else:
newtext = addText + config.line_separator + text
if putText and text != newtext:
pywikibot.output(
color_format('\n\n>>> {lightpurple}{0}{default} <<<',
page.title()))
pywikibot.showDiff(text, newtext)
# Let's put the changes.
while True:
# If someone load it as module, maybe it's not so useful to put the
# text in the page
if putText:
if not always:
try:
choice = pywikibot.input_choice(
'Do you want to accept these changes?',
[('Yes', 'y'), ('No', 'n'), ('All', 'a'),
('open in Browser', 'b')], 'n')
except QuitKeyboardInterrupt:
sys.exit('User quit bot run.')
if choice == 'a':
always = True
elif choice == 'n':
return (False, False, always)
elif choice == 'b':
pywikibot.bot.open_webbrowser(page)
if always or choice == 'y':
try:
if always:
page.put(newtext,
summary,
minorEdit=page.namespace() != 3)
else:
page.put_async(newtext,
summary,
minorEdit=page.namespace() != 3)
except pywikibot.EditConflict:
pywikibot.output(u'Edit conflict! skip!')
return (False, False, always)
except pywikibot.ServerError:
errorCount += 1
if errorCount < config.max_retries:
pywikibot.output(u'Server Error! Wait..')
time.sleep(config.retry_wait)
continue
else:
raise pywikibot.ServerError(u'Fifth Server Error!')
except pywikibot.SpamfilterError as e:
pywikibot.output(
u'Cannot change %s because of blacklist entry %s' %
(page.title(), e.url))
return (False, False, always)
except pywikibot.LockedPage:
pywikibot.output(u'Skipping %s (locked page)' %
page.title())
return (False, False, always)
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Error putting page: %s' % error.args)
return (False, False, always)
else:
# Break only if the errors are one after the other...
errorCount = 0
return (True, True, always)
else:
return (text, newtext, always)
try:
if image.file_is_shared():
exit("mainpage exist (shared file).\n")
except Exception as e:
pass
if not talkpage.exists():
exit("talkpage not exist.\n")
if talkpage.namespace().id in [3, 9]:
exit("ignore namespace.\n")
if talkpage.depth > 0:
exit("ignore subpage.\n")
for template in talkpage.templates():
if template.title() in ["Template:Talk archive"]:
exit("ignore talk archive.\n")
if template.title() in ["Template:Delete"]:
exit("marked deletion.\n")
if len(list(talkpage.embeddedin(total=1))) > 0:
text = cfg["prepend_text_with_noinclude"] + talkpage.text
else:
text = cfg["prepend_text"] + talkpage.text
pywikibot.showDiff(talkpage.text, text)
talkpage.text = text
summary = cfg["summary"]
print(summary)
talkpage.save(summary=summary, minor=False)
archivelist[target] = []
archivestr = str(section).strip()
archivestr = re.sub(
r"{{bot-directive-archiver\|no-archive-begin}}[\s\S]+?{{bot-directive-archiver\|no-archive-end}}\n?",
"", archivestr)
archivelist[target].append(archivestr)
count += 1
section.remove(section)
print("archive to " + str(target), end="\t")
print()
text = str(wikicode)
if ewippage.text == text:
exit("nothing changed")
pywikibot.showDiff(ewippage.text, text)
ewippage.text = text
summary = cfg["main_page_summary"].format(count)
print(summary)
ewippage.save(summary=summary, minor=False)
for target in archivelist:
archivepage = pywikibot.Page(
site, cfg["archive_page_name"].format(target[0], target[1]))
text = archivepage.text
print(archivepage.title())
if not archivepage.exists():
text = cfg["archive_page_preload"]
text += "\n\n" + "\n\n".join(archivelist[target])
pywikibot.showDiff(archivepage.text, text)
def main():
site = pywikibot.Site('wikiapiary', 'wikiapiary')
catname = 'Category:Website'
cat = pywikibot.Category(site, catname)
gen = pagegenerators.CategorizedPageGenerator(cat, start='Spyropedia')
pre = pagegenerators.PreloadingGenerator(gen)
for page in pre:
if page.isRedirectPage():
continue
wtitle = page.title()
wtext = page.text
#if not wtitle.startswith('5'):
# continue
if re.search('Internet Archive', wtext):
#print('It has IA parameter')
pass
else:
print('\n', '#' * 50, '\n', wtitle, '\n', '#' * 50)
print('https://wikiapiary.com/wiki/%s' %
(re.sub(' ', '_', wtitle)))
print('Missing IA parameter')
if re.search(r'(?i)API URL=http', wtext):
apiurl = re.findall(r'(?i)API URL=(http[^\n]+?)\n', wtext)[0]
print('API:', apiurl)
else:
print('No API found in WikiApiary, skiping')
continue
indexurl = 'index.php'.join(apiurl.rsplit('api.php', 1))
urliasearch = 'https://archive.org/search.php?query=originalurl:"%s" OR originalurl:"%s"' % (
apiurl, indexurl)
f = urllib.request.urlopen(urliasearch)
raw = f.read().decode('utf-8')
if re.search(r'(?i)Your search did not match any items', raw):
print('No dumps found at Internet Archive')
else:
itemidentifier = re.findall(
r'<a href="/details/([^ ]+?)" title=', raw)[0]
itemurl = 'https://archive.org/details/%s' % (itemidentifier)
print('Item found:', itemurl)
metaurl = 'https://archive.org/download/%s/%s_files.xml' % (
itemidentifier, itemidentifier)
g = urllib.request.urlopen(metaurl)
raw2 = g.read().decode('utf-8')
raw2 = raw2.split('</file>')
itemfiles = []
for raw2_ in raw2:
try:
x = re.findall(
r'(?im)<file name="[^ ]+-(\d{8})-[^ ]+" source="original">',
raw2_)[0]
y = re.findall(r'(?im)<size>(\d+)</size>', raw2_)[0]
itemfiles.append([int(x), int(y)])
except:
pass
itemfiles.sort(reverse=True)
print(itemfiles)
itemdate = str(itemfiles[0][0])[0:4] + '/' + str(
itemfiles[0][0])[4:6] + '/' + str(itemfiles[0][0])[6:8]
itemsize = itemfiles[0][1]
iaparams = """|Internet Archive identifier=%s
|Internet Archive URL=%s
|Internet Archive added date=%s 00:00:00
|Internet Archive file size=%s""" % (itemidentifier, itemurl, itemdate,
itemsize)
newtext = page.text
newtext = re.sub(r'(?im)\n\}\}', '\n%s\n}}' % (iaparams),
newtext)
if page.text != newtext:
pywikibot.showDiff(page.text, newtext)
page.text = newtext
page.save('BOT - Adding dump details: %s, %s, %s bytes' %
(itemidentifier, itemdate, itemsize),
botflag=True)
def main(*args):
"""
Process command line arguments and perform task.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
# Loading the comments
global categoryToCheck, project_inserted
# always, define a generator to understand if the user sets one,
# defining what's genFactory
always = False
generator = False
show = False
moveBlockCheck = False
protectedpages = False
protectType = 'edit'
namespace = 0
# To prevent Infinite loops
errorCount = 0
# Process global args and prepare generator args parser
local_args = pywikibot.handle_args(args)
genFactory = pagegenerators.GeneratorFactory()
# Process local args
for arg in local_args:
if arg == '-always':
always = True
elif arg == '-move':
moveBlockCheck = True
elif arg == '-show':
show = True
elif arg.startswith('-protectedpages'):
protectedpages = True
if len(arg) > 15:
namespace = int(arg[16:])
elif arg.startswith('-moveprotected'):
protectedpages = True
protectType = 'move'
if len(arg) > 14:
namespace = int(arg[15:])
else:
genFactory.handleArg(arg)
if config.mylang not in project_inserted:
pywikibot.output(u"Your project is not supported by this script.\n"
u"You have to edit the script and add it!")
return
site = pywikibot.Site()
if protectedpages:
generator = site.protectedpages(namespace=namespace, type=protectType)
# Take the right templates to use, the category and the comment
TSP = i18n.translate(site, templateSemiProtection)
TTP = i18n.translate(site, templateTotalProtection)
TSMP = i18n.translate(site, templateSemiMoveProtection)
TTMP = i18n.translate(site, templateTotalMoveProtection)
TNR = i18n.translate(site, templateNoRegex)
TU = i18n.translate(site, templateUnique)
categories = i18n.translate(site, categoryToCheck)
commentUsed = i18n.twtranslate(site, 'blockpageschecker-summary')
if not generator:
generator = genFactory.getCombinedGenerator()
if not generator:
generator = list()
pywikibot.output(u'Loading categories...')
# Define the category if no other generator has been setted
for CAT in categories:
cat = pywikibot.Category(site, CAT)
# Define the generator
gen = pagegenerators.CategorizedPageGenerator(cat)
for pageCat in gen:
generator.append(pageCat)
pywikibot.output(u'Categories loaded, start!')
# Main Loop
preloadingGen = pagegenerators.PreloadingGenerator(generator, step=60)
for page in preloadingGen:
pagename = page.title(asLink=True)
pywikibot.output('Loading %s...' % pagename)
try:
text = page.text
except pywikibot.NoPage:
pywikibot.output("%s doesn't exist! Skipping..." % pagename)
continue
except pywikibot.IsRedirectPage:
pywikibot.output("%s is a redirect! Skipping..." % pagename)
if show:
showQuest(page)
continue
# FIXME: This check does not work :
# PreloadingGenerator cannot set correctly page.editRestriction
# (see bug 55322)
# if not page.canBeEdited():
# pywikibot.output("%s is sysop-protected : this account can't edit "
# "it! Skipping..." % pagename)
# continue
restrictions = page.protection()
try:
editRestr = restrictions['edit']
except KeyError:
editRestr = None
if not page.canBeEdited():
pywikibot.output(u"%s is protected: "
u"this account can't edit it! Skipping..." %
pagename)
continue
# Understand, according to the template in the page, what should be the
# protection and compare it with what there really is.
TemplateInThePage = understandBlock(text, TTP, TSP, TSMP, TTMP, TU)
# Only to see if the text is the same or not...
oldtext = text
# keep track of the changes for each step (edit then move)
changes = -1
if not editRestr:
# page is not edit-protected
# Deleting the template because the page doesn't need it.
if TU:
replaceToPerform = u'|'.join(TTP + TSP + TU)
else:
replaceToPerform = u'|'.join(TTP + TSP)
text, changes = re.subn(
'<noinclude>(%s)</noinclude>' % replaceToPerform, '', text)
if changes == 0:
text, changes = re.subn('(%s)' % replaceToPerform, '', text)
msg = u'The page is editable for all'
if not moveBlockCheck:
msg += u', deleting the template..'
pywikibot.output(u'%s.' % msg)
elif editRestr[0] == 'sysop':
# total edit protection
if (TemplateInThePage[0] == 'sysop-total' and TTP) or \
(TemplateInThePage[0] == 'unique' and TU):
msg = 'The page is protected to the sysop'
if not moveBlockCheck:
msg += ', skipping...'
pywikibot.output(msg)
else:
pywikibot.output(
u'The page is protected to the sysop, but the '
u'template seems not correct. Fixing...')
if TU:
text, changes = re.subn(TemplateInThePage[1], TNR[4], text)
else:
text, changes = re.subn(TemplateInThePage[1], TNR[1], text)
elif TSP or TU:
# implicitely editRestr[0] = 'autoconfirmed', edit-Semi-protection
if TemplateInThePage[0] == 'autoconfirmed-total' or \
TemplateInThePage[0] == 'unique':
msg = 'The page is editable only for the autoconfirmed users'
if not moveBlockCheck:
msg += ', skipping...'
pywikibot.output(msg)
else:
pywikibot.output(u'The page is editable only for the '
u'autoconfirmed users, but the template '
u'seems not correct. Fixing...')
if TU:
text, changes = re.subn(TemplateInThePage[1], TNR[4], text)
else:
text, changes = re.subn(TemplateInThePage[1], TNR[0], text)
if changes == 0:
# We tried to fix edit-protection templates, but it did not work.
pywikibot.warning('No edit-protection template could be found')
if moveBlockCheck and changes > -1:
# checking move protection now
try:
moveRestr = restrictions['move']
except KeyError:
moveRestr = False
changes = -1
if not moveRestr:
pywikibot.output(u'The page is movable for all, deleting the '
u'template...')
# Deleting the template because the page doesn't need it.
if TU:
replaceToPerform = u'|'.join(TSMP + TTMP + TU)
else:
replaceToPerform = u'|'.join(TSMP + TTMP)
text, changes = re.subn(
'<noinclude>(%s)</noinclude>' % replaceToPerform, '', text)
if changes == 0:
text, changes = re.subn('(%s)' % replaceToPerform, '',
text)
elif moveRestr[0] == 'sysop':
# move-total-protection
if (TemplateInThePage[0] == 'sysop-move' and TTMP) or \
(TemplateInThePage[0] == 'unique' and TU):
pywikibot.output(u'The page is protected from moving to '
u'the sysop, skipping...')
if TU:
# no changes needed, better to revert the old text.
text = oldtext
else:
pywikibot.output(u'The page is protected from moving to '
u'the sysop, but the template seems not '
u'correct. Fixing...')
if TU:
text, changes = re.subn(TemplateInThePage[1], TNR[4],
text)
else:
text, changes = re.subn(TemplateInThePage[1], TNR[3],
text)
elif TSMP or TU:
# implicitely moveRestr[0] = 'autoconfirmed',
# move-semi-protection
if TemplateInThePage[0] == 'autoconfirmed-move' or \
TemplateInThePage[0] == 'unique':
pywikibot.output(u'The page is movable only for the '
u'autoconfirmed users, skipping...')
if TU:
# no changes needed, better to revert the old text.
text = oldtext
else:
pywikibot.output(u'The page is movable only for the '
u'autoconfirmed users, but the template '
u'seems not correct. Fixing...')
if TU:
text, changes = re.subn(TemplateInThePage[1], TNR[4],
text)
else:
text, changes = re.subn(TemplateInThePage[1], TNR[2],
text)
if changes == 0:
# We tried to fix move-protection templates, but it did not work
pywikibot.warning('No move-protection template could be found')
if oldtext != text:
# Ok, asking if the change has to be performed and do it if yes.
pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<" %
page.title())
pywikibot.showDiff(oldtext, text)
if not always:
choice = pywikibot.input_choice(
u'Do you want to accept these '
u'changes?', [('Yes', 'y'), ('No', 'n'), ('All', 'a')],
'n')
if choice == 'a':
always = True
if always or choice == 'y':
while True:
try:
page.put(text, commentUsed, force=True)
except pywikibot.EditConflict:
pywikibot.output(u'Edit conflict! skip!')
break
except pywikibot.ServerError:
# Sometimes there is this error that's quite annoying
# because can block the whole process for nothing.
errorCount += 1
if errorCount < 5:
pywikibot.output(u'Server Error! Wait..')
time.sleep(3)
continue
else:
# Prevent Infinite Loops
raise pywikibot.ServerError(u'Fifth Server Error!')
except pywikibot.SpamfilterError as e:
pywikibot.output(u'Cannot change %s because of '
u'blacklist entry %s' %
(page.title(), e.url))
break
except pywikibot.LockedPage:
pywikibot.output(u'The page is still protected. '
u'Skipping...')
break
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Error putting page: %s' %
(error.args, ))
break
else:
# Break only if the errors are one after the other
errorCount = 0
break
def userPut(self, page, oldtext, newtext, **kwargs):
"""
Save a new revision of a page, with user confirmation as required.
Print differences, ask user for confirmation,
and puts the page if needed.
Option used:
* 'always'
Keyword args used:
* 'async' - passed to page.save
* 'comment' - passed to page.save
* 'show_diff' - show changes between oldtext and newtext (enabled)
* 'ignore_save_related_errors' - report and ignore (disabled)
* 'ignore_server_errors' - report and ignore (disabled)
"""
if oldtext == newtext:
pywikibot.output(u'No changes were needed on %s'
% page.title(asLink=True))
return
self.current_page = page
show_diff = kwargs.pop('show_diff', True)
if show_diff:
pywikibot.showDiff(oldtext, newtext)
if 'comment' in kwargs:
pywikibot.output(u'Comment: %s' % kwargs['comment'])
if not self.user_confirm('Do you want to accept these changes?'):
return
if 'async' not in kwargs and self.getOption('always'):
kwargs['async'] = True
page.text = newtext
ignore_save_related_errors = kwargs.pop('ignore_save_related_errors', False)
ignore_server_errors = kwargs.pop('ignore_server_errors', False)
try:
page.save(**kwargs)
except pywikibot.PageSaveRelatedError as e:
if not ignore_save_related_errors:
raise
if isinstance(e, pywikibot.EditConflict):
pywikibot.output(u'Skipping %s because of edit conflict'
% page.title())
elif isinstance(e, pywikibot.SpamfilterError):
pywikibot.output(
u'Cannot change %s because of blacklist entry %s'
% (page.title(), e.url))
elif isinstance(e, pywikibot.LockedPage):
pywikibot.output(u'Skipping %s (locked page)'
% page.title())
else:
pywikibot.error(
u'Skipping %s because of a save related error: %s'
% (page.title(), e))
except pywikibot.ServerError as e:
if not ignore_server_errors:
raise
pywikibot.error(u'Server Error while processing %s: %s'
% (page.title(), e))
def add_text(page,
addText,
summary=None,
regexSkip=None,
regexSkipUrl=None,
always=False,
up=False,
putText=True,
oldTextGiven=None,
reorderEnabled=True,
create=False):
"""
Add text to a page.
@rtype: tuple of (text, newtext, always)
"""
site = page.site
if not summary:
summary = i18n.twtranslate(site, 'add_text-adding',
{'adding': addText[:200]})
# When a page is tagged as "really well written" it has a star in the
# interwiki links. This is a list of all the templates used (in regex
# format) to make the stars appear.
errorCount = 0
if putText:
pywikibot.output(u'Loading %s...' % page.title())
if oldTextGiven is None:
try:
text = page.get()
except pywikibot.NoPage:
if create:
pywikibot.output(u"%s doesn't exist, creating it!" %
page.title())
text = u''
else:
pywikibot.output(u"%s doesn't exist, skip!" % page.title())
return (False, False, always)
except pywikibot.IsRedirectPage:
pywikibot.output(u"%s is a redirect, skip!" % page.title())
return (False, False, always)
else:
text = oldTextGiven
# Understand if the bot has to skip the page or not
# In this way you can use both -except and -excepturl
if regexSkipUrl is not None:
url = site.nice_get_address(page.title(asUrl=True))
result = re.findall(regexSkipUrl, site.getUrl(url))
if result != []:
pywikibot.output(
u'''Exception! regex (or word) used with -exceptUrl is in the page. Skip!
Match was: %s''' % result)
return (False, False, always)
if regexSkip is not None:
result = re.findall(regexSkip, text)
if result != []:
pywikibot.output(
u'''Exception! regex (or word) used with -except is in the page. Skip!
Match was: %s''' % result)
return (False, False, always)
# If not up, text put below
if not up:
newtext = text
# Translating the \\n into binary \n
addText = addText.replace('\\n', config.line_separator)
if (reorderEnabled):
# Getting the categories
categoriesInside = textlib.getCategoryLinks(newtext, site)
# Deleting the categories
newtext = textlib.removeCategoryLinks(newtext, site)
# Getting the interwiki
interwikiInside = textlib.getLanguageLinks(newtext, site)
# Removing the interwiki
newtext = textlib.removeLanguageLinks(newtext, site)
# Adding the text
newtext += u"%s%s" % (config.line_separator, addText)
# Reputting the categories
newtext = textlib.replaceCategoryLinks(newtext, categoriesInside,
site, True)
# Dealing the stars' issue
allstars = []
starstext = textlib.removeDisabledParts(text)
for star in starsList:
regex = re.compile(
'(\{\{(?:template:|)%s\|.*?\}\}[\s]*)' % star, re.I)
found = regex.findall(starstext)
if found != []:
newtext = regex.sub('', newtext)
allstars += found
if allstars != []:
newtext = newtext.strip() + config.line_separator * 2
allstars.sort()
for element in allstars:
newtext += '%s%s' % (element.strip(), config.LS)
# Adding the interwiki
newtext = textlib.replaceLanguageLinks(newtext, interwikiInside,
site)
else:
newtext += u"%s%s" % (config.line_separator, addText)
else:
newtext = addText + config.line_separator + text
if putText and text != newtext:
pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<" %
page.title())
pywikibot.showDiff(text, newtext)
# Let's put the changes.
while True:
# If someone load it as module, maybe it's not so useful to put the
# text in the page
if putText:
if not always:
choice = pywikibot.inputChoice(
u'Do you want to accept these changes?',
['Yes', 'No', 'All', 'open in Browser'],
['y', 'n', 'a', 'b'], 'n')
if choice == 'a':
always = True
elif choice == 'n':
return (False, False, always)
elif choice == 'b':
webbrowser.open(
"http://%s%s" %
(site.hostname(),
site.nice_get_address(page.title(asUrl=True))))
pywikibot.input("Press Enter when finished in browser.")
if always or choice == 'y':
try:
if always:
page.put(newtext,
summary,
minorEdit=page.namespace() != 3)
else:
page.put_async(newtext,
summary,
minorEdit=page.namespace() != 3)
except pywikibot.EditConflict:
pywikibot.output(u'Edit conflict! skip!')
return (False, False, always)
except pywikibot.ServerError:
errorCount += 1
if errorCount < config.max_retries:
pywikibot.output(u'Server Error! Wait..')
time.sleep(config.retry_wait)
continue
else:
raise pywikibot.ServerError(u'Fifth Server Error!')
except pywikibot.SpamfilterError as e:
pywikibot.output(
u'Cannot change %s because of blacklist entry %s' %
(page.title(), e.url))
return (False, False, always)
except pywikibot.LockedPage:
pywikibot.output(u'Skipping %s (locked page)' %
page.title())
return (False, False, always)
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Error putting page: %s' % error.args)
return (False, False, always)
else:
# Break only if the errors are one after the other...
errorCount = 0
return (True, True, always)
else:
return (text, newtext, always)
def operate(self, page, liste_modeles):
dict = {}
modeles_modifies_str = u""
count_modeles_modifies = 0
add_date = False
pywikibot.output("Doing page %s" % page)
if page.isRedirectPage():
pywikibot.output("Page is a redirection, skipping.")
return
text = page.get()
old_text = text
for couple in page.templatesWithParams():
#print couple
for modele in liste_modeles:
if couple[0] == modele:
dict[modele] = couple[1]
for modele in dict:
# NB : modele est de type pywikibot.Page
titre_modele = modele.title(withNamespace=False)
add_date = True
change_date = False
re_params = u""
#str_params = u""
#pywikibot.output(u"modele : %s" % modele.title(withNamespace=False))
if dict[modele]:
for param in dict[modele]:
param = param.replace('(', '\(').replace(
')',
'\)').replace('[', '\[').replace(']', '\]').replace(
'{', '\{').replace('{', '\}').replace(
'?', '\?').replace('|', '\|').replace(
'*', '\*').replace('+', '\+')
#param = re.escape(param)
re_params += (u" *\| *([0-9]+ *= *)?%s" % param
) # Paramètres allant avec le modèle
#str_params += u"|%s" % param
if re.search(u"^date *=.+", param):
param_date = re.search(
u"^date *= *([0-9]{0,2}) *(janvier|février|mars|avril|mai|juin|juillet|août|septembre|octobre|novembre|décembre) *(20[0-9]{2})",
param)
add_date = False
if param_date and param_date.group(1):
text = text.replace(
param_date.group(0), "date=%s %s" %
(param_date.group(2), param_date.group(3)))
# NB : ne fait rien si un paramètre date est mal renseigné
# mais que la correction à appliquer est inconnue, pour éviter
# d'ajouter une fausse date.
# ex : quelqu'un ajoute 'date=octobr 2015' après recherche
# mais le script tourne en janvier 2016 donc le bot
# effacerait l'ancienne date pour rajouter une date
# incorrecte.
#pywikibot.output("add_date = %s" % unicode(add_date))
#pywikibot.output("str_params = %s" % unicode(str_params))
#pywikibot.output("re_params = %s" % unicode(re_params))
if add_date:
# Permet de rechercher le modèle (avec ses paramètres) dans le texte
#pywikibot.output(u"{ *{ *([%s%s]%s%s) *} *}" % (titre_modele[0].lower(), titre_modele[0].upper(), titre_modele[1:].replace(' ', '[ _]'), unicode(re_params)))
str_re_titre_modele = u"{ *{ *([%s%s]%s%s) *} *}" % (
titre_modele[0].lower(), titre_modele[0].upper(),
titre_modele[1:].replace(' ', '[ _]+'), unicode(re_params))
(timestamp, id) = self.find_add(page, modele)
date = timestamp.strftime(u"%B %Y")
#print "found the add : %s (%i) -> %s" % (timestamp, id, date)
if date is None:
pywikibot.output("Error, date is None!")
return
#now = datetime.datetime.now()
#date = now.strftime(u"%B %Y")
match_modele = re.search(str_re_titre_modele, text)
if not match_modele:
pywikibot.output("An error occurred: no match for %s" %
str_re_titre_modele)
return
#pywikibot.output(match_modele.group(1))
new = u"{{%s|date=%s}}" % (match_modele.group(1),
date.decode('utf-8'))
#pywikibot.output(new)
text = re.sub(str_re_titre_modele, new, text)
#text = text.replace(match_modele.group(0), new)
modeles_modifies_str += u"[[Modèle:%(modele)s|{{%(modele)s}}]], " % {
'modele': titre_modele
}
count_modeles_modifies += 1
modeles_modifies_str = modeles_modifies_str[
0:-2] # Enlever le ', ' en trop
if count_modeles_modifies == 1:
comment = u"Bot: Ajout du paramètre 'date' dans le modèle %s" % modeles_modifies_str
else:
comment = u"Bot: Ajout du paramètre 'date' dans les modèles %s" % modeles_modifies_str
pywikibot.showDiff(page.get(), text)
if old_text != page.get():
pywikibot.output("Page was changed since retrived! Aborting.")
return
elif not debug and page.canBeEdited():
page.put(text, comment=comment)
def fix_1_double_redirect(self, redir_name):
redir = pywikibot.Page(self.site, redir_name)
# Show the title of the page we're working on.
# Highlight the title in purple.
pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<" %
redir.title())
newRedir = redir
redirList = [] # bookkeeping to detect loops
while True:
redirList.append(
u'%s:%s' %
(newRedir.site.lang, newRedir.title(withSection=False)))
try:
targetPage = newRedir.getRedirectTarget()
except pywikibot.IsNotRedirectPage:
if len(redirList) == 1:
pywikibot.output(u'Skipping: Page %s is not a redirect.' %
redir.title(asLink=True))
break # do nothing
elif len(redirList) == 2:
pywikibot.output(
u'Skipping: Redirect target %s is not a redirect.' %
newRedir.title(asLink=True))
break # do nothing
else:
pass # target found
except pywikibot.SectionError:
pywikibot.warning(
u"Redirect target section %s doesn't exist." %
newRedir.title(asLink=True))
except pywikibot.CircularRedirect as e:
pywikibot.exception(e)
pywikibot.output(u"Skipping %s." % newRedir)
break
except pywikibot.BadTitle as e:
# str(e) is in the format 'BadTitle: [[Foo]]'
pywikibot.warning(
u'Redirect target %s is not a valid page title.' %
str(e)[10:])
break
except pywikibot.NoPage:
if len(redirList) == 1:
pywikibot.output(u'Skipping: Page %s does not exist.' %
redir.title(asLink=True))
break
else:
if self.getOption('always'):
pywikibot.output(
u"Skipping: Redirect target %s doesn't exist." %
newRedir.title(asLink=True))
break # skip if automatic
else:
pywikibot.warning(
u"Redirect target %s doesn't exist." %
newRedir.title(asLink=True))
except pywikibot.ServerError:
pywikibot.output(u'Skipping due to server error: '
u'No textarea found')
break
else:
pywikibot.output(u' Links to: %s.' %
targetPage.title(asLink=True))
if targetPage.site != self.site:
pywikibot.warning(
u'redirect target (%s) is on a different site.' %
targetPage.title(asLink=True))
if self.getOption('always'):
break # skip if automatic
try:
mw_msg = targetPage.site.mediawiki_message(
'wikieditor-toolbar-tool-redirect-example')
except KeyError:
pass
else:
if targetPage.title() == mw_msg:
pywikibot.output(
u"Skipping toolbar example: Redirect source is "
u"potentially vandalized.")
break
# watch out for redirect loops
if redirList.count(u'%s:%s' %
(targetPage.site.lang,
targetPage.title(withSection=False))):
pywikibot.warning(
u'Redirect target %s forms a redirect loop.' %
targetPage.title(asLink=True))
break # FIXME: doesn't work. edits twice!
try:
content = targetPage.get(get_redirect=True)
except pywikibot.SectionError:
content_page = pywikibot.Page(
targetPage.site,
targetPage.title(withSection=False))
content = content_page.get(get_redirect=True)
if i18n.twhas_key(
targetPage.site.lang,
'redirect-broken-redirect-template') and \
i18n.twhas_key(targetPage.site.lang,
'redirect-remove-loop'):
pywikibot.output(u"Tagging redirect for deletion")
# Delete the two redirects
content = i18n.twtranslate(
targetPage.site.lang,
'redirect-broken-redirect-template'
) + "\n" + content
summ = i18n.twtranslate(targetPage.site.lang,
'redirect-remove-loop')
targetPage.put(content, summ)
redir.put(content, summ)
break
else: # redirect target found
if targetPage.isStaticRedirect():
pywikibot.output(
u" Redirect target is STATICREDIRECT.")
pass
else:
newRedir = targetPage
continue
try:
oldText = redir.get(get_redirect=True)
except pywikibot.BadTitle:
pywikibot.output(u"Bad Title Error")
break
oldlink = self.site.redirectRegex().search(oldText).group(1)
if "#" in oldlink and targetPage.section() is None:
sectionlink = oldlink[oldlink.index("#"):]
targetlink = pywikibot.Page(self.site,
targetPage.title() +
sectionlink).title(asLink=True,
textlink=True)
else:
targetlink = targetPage.title(asLink=True, textlink=True)
text = self.site.redirectRegex().sub(
'#%s %s' % (self.site.redirect(), targetlink), oldText, 1)
if redir.title() == targetPage.title() or text == oldText:
pywikibot.output(u"Note: Nothing left to do on %s" %
redir.title(asLink=True))
break
summary = i18n.twtranslate(self.site, 'redirect-fix-double',
{'to': targetPage.title(asLink=True)})
pywikibot.showDiff(oldText, text)
if self.user_confirm(u'Do you want to accept the changes?'):
try:
redir.put(text, summary)
except pywikibot.LockedPage:
pywikibot.output(u'%s is locked.' % redir.title())
except pywikibot.SpamfilterError as error:
pywikibot.output(
u"Saving page [[%s]] prevented by spam filter: %s" %
(redir.title(), error.url))
except pywikibot.PageNotSaved as error:
pywikibot.output(u"Saving page [[%s]] failed: %s" %
(redir.title(), error))
except pywikibot.NoUsername:
pywikibot.output(
u"Page [[%s]] not saved; sysop privileges required." %
redir.title())
except pywikibot.Error as error:
pywikibot.output(
u"Unexpected error occurred trying to save [[%s]]: %s"
% (redir.title(), error))
break
def main():
config.use_mwparserfromhell = False
locale.setlocale(locale.LC_ALL, 'fr_FR.utf-8')
db = False
global test
global dry
dry = False # À activer seulement pour les tests
test = False # À activer seulement pour tester le script sur une seule page
ns_test = False
recurse_test = False
for arg in pywikibot.handleArgs():
if arg == "-dry":
dry = True
pywikibot.output(u'(dry is ON)')
elif arg[0:6] == "-test:":
test = True
titre_page_test = arg[6:]
elif arg[0:4] == "-ns:":
ns_test = True
namespaces_test_value = [int(i) for i in arg[4:].split(',')]
elif arg[0:9] == "-recurse:":
recurse_test = True
recurse_test_value = bool(arg[9:])
comment_modele = u"%(nombre_articles)i articles) (Bot: Mise à jour de la liste des articles récents (%(precision_pages)s)"
site = pywikibot.Site()
titre_modele = u"Articles récents"
modele = pywikibot.Page(site, titre_modele, ns=10)
gen = pagegenerators.ReferringPageGenerator(modele,
onlyTemplateInclusion=True)
matchDebut1 = u"<!-- Ce tableau est créé automatiquement par un robot. Articles Récents DEBUT -->"
matchFin1 = u"\n<!-- Ce tableau est créé automatiquement par un robot. Articles Récents FIN -->"
matchDebut2 = u"<!-- Ce tableau est créé automatiquement par un robot. Articles Récents Liste DEBUT -->"
matchFin2 = u"\n<!-- Ce tableau est créé automatiquement par un robot. Articles Récents Liste FIN -->"
if test:
pywikibot.output(u'(test is ON)')
gen = [pywikibot.Page(site, titre_page_test)]
if ns_test:
pywikibot.output(u'(ns_test is ON)')
for main_page in gen:
try:
comment = comment_modele
pywikibot.output(
u"\n========================\nTraitement de %s\n========================"
% main_page.title())
text = main_page.get()
#####################
### Récupération des informations sur la page
#####################
templates = textlib.extract_templates_and_params_regex(text)
template_in_use = None
for tuple in templates:
if tuple[0] != u'Articles récents':
continue
else:
template_in_use = tuple[1]
break
if not template_in_use:
pywikibot.output(
u"Aucun modèle {{Articles récents}} détecté sur la page %s"
% main_page.title())
continue
titre_categorie = check_and_return_parameter(
template_in_use, u'catégorie')
if not titre_categorie:
continue
cat = pywikibot.Category(site, titre_categorie)
nbMax = check_and_return_parameter(template_in_use, 'nbMax', 10)
try:
nbMax = int(nbMax)
except:
pywikibot.output(u'Erreur : nbMax incorrect')
continue
namespaces = check_and_return_parameter(template_in_use,
'namespaces', '0')
namespaces = namespaces.split(',')
try:
namespaces = [int(k) for k in namespaces]
except:
pywikibot.output(
u'Erreur : des namespaces spécifiés ne sont pas des entiers'
)
continue
recurse = check_and_return_parameter(template_in_use, 'recurse',
'0')
if recurse.lower().strip() in ('oui', '1'):
recurse = True
else:
recurse = False
delai_creation = check_and_return_parameter(
template_in_use, 'delai', '0')
try:
delai_creation = int(delai_creation)
except:
pywikibot.output(u'Erreur : delai incorrect')
continue
format_date = check_and_return_parameter(template_in_use,
u'date') or None
if format_date:
try:
test_date = datetime.datetime.now()
test_date.strftime(format_date)
except:
format_date = None
pywikibot.output(u'Erreur : format de date incorrect')
puce = check_and_return_parameter(template_in_use, 'puces', '#')
listeRecents = text[(text.index(matchDebut1) +
len(matchDebut1)):text.index(matchFin1)]
# Permet d'enlever le premier élément (vide) de la liste
listeRecents = listeRecents.split('\n%s ' % puce)[1:]
listeRecents_old = [page for page in listeRecents]
listeRecents = list()
dico_dates_presentes = {}
for recent in listeRecents_old:
r = re.search(u"(\[\[.*\]\]) ?(\(.+\))?", recent)
if r:
listeRecents.append(r.group(1))
if r.group(2):
dico_dates_presentes[r.group(1)] = r.group(2)[1:-1]
else:
pass
text = re.sub(
re.compile(u"%s.*%s" % (matchDebut2, matchFin2), re.S),
u"%s%s" % (matchDebut2, matchFin2), text)
#####################
# Au cas où il n'y aurait aucune nouvelle page mais
# une ou des pages ayant été supprimée(s)
exception_maj = False
# Pour préciser le résumé d'édition
precisions_comment = u""
pywikibot.output('stade 0')
#####################
### Vérification des pages récentes actuelles (en cas de suppression)
#####################
for titre_article in listeRecents:
try:
page = pywikibot.Page(
site, re.sub(u"\[\[(.*)\]\]", "\\1", titre_article
)) # Pour enlever les crochets : [[…]].
# Si la page existe toujours et n'est pas une
# redirection, on la laisse dans la liste…
page.get()
if format_date and not dico_dates_presentes.has_key(
titre_article) and find_date(page, cat):
# Date trouvée alors qu'elle n'y était pas.
exception_maj = True
dico_dates_presentes[titre_article] = find_date(
page, cat).strftime(format_date)
except pywikibot.NoPage:
pywikibot.output(u"La page %s n'existe plus." %
page.title(asLink=True))
pywikibot.output(
u"Suppression de la page %s de la liste listeRecents" %
page.title(asLink=True))
precisions_comment += (u"; - %s" % titre_article)
listeRecents.remove(titre_article)
# On force la mise à jour de la page, même si aucun nouvel article
# récent n'est trouvé.
exception_maj = True
except pywikibot.IsRedirectPage:
pywikibot.output(
u"La page %s n'est plus qu'une redirection." %
page.title(asLink=True))
try:
nouvelle_page = page.getRedirectTarget()
pywikibot.output(
u"Modification du titre la page %s (renommée en %s)"
% (page.title(asLink=True),
nouvelle_page.title(asLink=True,
withSection=False)))
precisions_comment += (
u"; - %s ; + %s" %
(titre_article,
nouvelle_page.title(asLink=True,
withSection=False)))
if not nouvelle_page.title(
asLink=True,
withSection=False) in listeRecents:
listeRecents[listeRecents.index(
titre_article)] = nouvelle_page.title(
asLink=True, withSection=False)
else:
pywikibot.output(
u"La page destination était déjà présente dans la liste"
)
listeRecents.pop(listeRecents.index(titre_article))
# On force la mise à jour de la page, même si aucun nouvel article
# récent n'est trouvé.
exception_maj = True
except:
pywikibot.output(
u"an error occured (CircularRedirect?)")
#except KeyboardInterrupt:
# pywikibot.stopme()
except:
try:
pywikibot.output(
u"Erreur inconnue lors du traitement de la page %s"
% page.title(asLink=True))
except:
pywikibot.output(
u"Erreur inconnue lors du traitement d'une page")
else:
# Si pas d'erreur : on passe à la page suivante
continue
if precisions_comment:
precisions_comment = precisions_comment[
2:] # Pour supprimer le '; '
#####################
#####################
### Recherches des articles nouveaux
#####################
precisions_comment2 = u""
# Récupération de la dernière mise à jour de la page par le bot
db = _mysql.connect(
host='tools-db',
db='s51245__totoazero',
read_default_file="/data/project/totoazero/replica.my.cnf")
results = db.query(
'SELECT last FROM maj_articles_recents WHERE page="%s"' %
main_page.title().replace('"', '\\"').encode('utf-8'))
results = db.store_result()
result = results.fetch_row(maxrows=0)
pywikibot.output(("last check was " + str(result)))
if result:
first_passage = False
t = result[0][0]
timestamp = pywikibot.Timestamp.strptime(
t, "%Y-%m-%d %H:%M:%S")
# Permet de ne générer que la liste des articles ajoutés à la
# catégorie après la dernière modification de la page
# contenant le modèle {{Articles récents}}.
#list_new.extend([page for page in site.categorymembers(cat, starttime=timestamp, sortby='timestamp', namespaces=[0])])
list_new = [
page for page in cat.articles(starttime=timestamp,
sortby='timestamp',
namespaces=namespaces,
recurse=recurse)
]
list_new.reverse()
else: # nouvelle page, premier passage du bot
first_passage = True
timestamp = main_page.editTime()
if delai_creation > 0:
timestamp -= datetime.timedelta(hours=delai_creation)
# Génération de la première liste, pour éviter si possible de
# laisser la page vide.
list_new = [page for page in cat.newest_pages(total=nbMax)]
# TODO : mieux ?
#list_new = [page for page in cat.articles(sortby='timestamp', namespaces=namespaces, recurse=recurse)]
pywikibot.output('stade 2')
now = datetime.datetime.now()
# NB : exception_maj peut être passer à True si un article
# a été supprimé de la catégorie.
if len(list_new) == 0 and not exception_maj:
# Inutile d'aller plus loin s'il n'y a aucun nouvel article.
end_page(main_page, now, first_passage)
continue
# Liste des pages pour requête SQL sur base frwiki_p
list_new_str = '("'
list_new_str += '", "'.join([
page.title(asLink=False, underscore=True).replace('"', '\\"')
for page in list_new
])
list_new_str += '")'
pywikibot.output(list_new_str)
# Fonctionne uniquement avec les pages du ns 0 pour le moment
frwiki_p = _mysql.connect(
host='frwiki.analytics.db.svc.eqiad.wmflabs',
db='frwiki_p',
read_default_file="/data/project/totoazero/replica.my.cnf")
pywikibot.output(
'SELECT page_title, page_id FROM page where page_title IN %s AND page_namespace=0'
% list_new_str.encode('utf-8'))
results = frwiki_p.query(
'SELECT page_title, page_id FROM page where page_title IN %s AND page_namespace=0'
% list_new_str.encode('utf-8'))
results = frwiki_p.store_result()
result = results.fetch_row(maxrows=0)
pywikibot.output(result)
dico_result = {}
for tuple in result:
title = tuple[0]
id = tuple[1]
dico_result[title] = id
pywikibot.output(dico_result)
dico_timestamp = {}
pywikibot.output('stade 3')
frwiki_p = _mysql.connect(
host='frwiki.analytics.db.svc.eqiad.wmflabs',
db='frwiki_p',
read_default_file="/data/project/totoazero/replica.my.cnf")
for key in dico_result:
id = dico_result[key]
pywikibot.output(
'SELECT cl_from, cl_timestamp FROM categorylinks WHERE cl_from = %s AND cl_to = "%s"'
% (id.encode('utf-8'),
cat.title(asLink=False,
underscore=True,
withNamespace=False).encode('utf-8')))
results = frwiki_p.query(
'SELECT cl_from, cl_timestamp FROM categorylinks WHERE cl_from = %s AND cl_to = "%s"'
% (id.encode('utf-8'),
cat.title(asLink=False,
underscore=True,
withNamespace=False).encode('utf-8')))
results = frwiki_p.store_result()
result = results.fetch_row(maxrows=0)
if result:
dico_timestamp[key.decode(
'utf-8')] = pywikibot.Timestamp.strptime(
result[0][1], "%Y-%m-%d %H:%M:%S")
else:
pywikibot.output(u"pas de date trouvée pour %s" %
key.decode('utf-8'))
pywikibot.output(dico_timestamp)
# Permet de mettre les nouvelles pages comme des titres :
# nécessaires plus loin !
list_new = [page.title(asLink=True) for page in list_new]
# Permet de récupérer des infos sur la catégorie.
# NB : Si ralentit le script, l'item cat_info['pages']
# correspondant au nombre de pages contenues
# dans la catégorie doit pouvoir être remplacé
# par len(listeCategorie) + len(list_new).
cat_info = site.categoryinfo(cat)
pywikibot.output(cat_info)
pywikibot.output('stade 4')
list_new_old = list()
list_new_old.extend(list_new)
pywikibot.output('delai_creation is %s' % delai_creation)
#for titre_page in list_new_old:
# print titre_page
for titre_page in list_new_old:
# NB : titre_page est du type [[Nom de la page]]
pywikibot.output("----------")
pywikibot.output(u"Page récemment ajoutée : %s" % titre_page)
if not titre_page in listeRecents:
if delai_creation:
# Délai imposé (en heures) depuis la création de l'article,
# au-delà duquel l'article récemment ajouté à la catégorie
# ne doit pas figurer dans la liste.
# Exemple : delai_creation = 24
# => le bot liste uniquement les articles créés il y
# a moins de 24h.
page = pywikibot.Page(site, titre_page[2:-2])
# NB : date_creation et date_plus_petite_requise
# sont du type pywikibot.Timestamp
date_creation = page.getVersionHistory()[-1][1]
pywikibot.output(date_creation)
if delai_creation > 0:
date_plus_petite_requise = pywikibot.Timestamp.now(
) - datetime.timedelta(hours=delai_creation)
elif delai_creation == -1:
# 'timestamp' a été défini plus haut comme étant la date de dernière
# édition du bot sur la page.
date_plus_petite_requise = timestamp
pywikibot.output(date_plus_petite_requise)
if date_plus_petite_requise > date_creation:
pywikibot.output(u"Vérification du délai : Non")
pywikibot.output(
u"La page ne satisfait pas le délai depuis la création imposé."
)
list_new.remove(titre_page)
continue
else:
pywikibot.output(u"Vérification du délai : OK")
precisions_comment2 += (u"; + %s" % titre_page)
else:
# Si l'article se trouve déjà dans la liste listeRecents
# il est inutile de le rajouter à nouveau.
list_new.remove(titre_page)
pywikibot.output(
u"L'article était déjà présent sur la page.")
# Re-vérification pour voir si list_new contient toujours
# au moins une page.
if len(list_new) == 0 and not exception_maj:
# Inutile d'aller plus loin s'il n'y a aucun nouvel article.
pywikibot.output('Nothing left.')
continue
# Re-vérification pour voir si list_new contient toujours
# au moins une page.
if len(list_new) == 0 and not exception_maj:
# Inutile d'aller plus loin s'il n'y a aucun nouvel article.
end_page(main_page, now, first_passage)
continue
if precisions_comment: # Si precisions_comment contient déjà des infos (suppression de pages)
precisions_comment += precisions_comment2
else:
precisions_comment = precisions_comment2[
2:] # Pour supprimer le '; '
pywikibot.output('stade 5')
# Pour compléter le résumé d'édition
comment = comment % {
'nombre_articles': cat_info['pages'],
'precision_pages': precisions_comment
}
#####################
### Création de la liste des articles récents
#####################
liste_nouveaux_recents = list()
liste_nouveaux_recents.extend(list_new)
# Si le nombre d'articles nouveaux est strictement au nombre maximum
# d'articles récents qui doivent figurer.
if len(liste_nouveaux_recents) < nbMax:
i = 0
while len(liste_nouveaux_recents) != nbMax:
if len(listeRecents) < i + 1:
# Dans le cas où la liste listeRecents ne contiendrait pas
# assez d'éléments.
break
liste_nouveaux_recents.append(listeRecents[i])
i += 1
if i == len(listeRecents
): # Pourrait provoquer une erreur de longueur
break
elif len(liste_nouveaux_recents) > nbMax:
liste_nouveaux_recents = liste_nouveaux_recents[0:(nbMax - 1)]
# La liste liste_nouveaux_recents contient désormais
# nbMax articles récents exactement
pywikibot.output('stade 6')
liste_nouveaux_recents_string = u"<!-- Ce tableau est créé automatiquement par un robot. Articles Récents DEBUT -->"
for titre_article in liste_nouveaux_recents:
liste_nouveaux_recents_string += u'\n%s %s' % (puce,
titre_article)
if format_date and dico_timestamp.has_key(
titre_article[2:-2].replace(' ', '_')):
pywikibot.output('stade 6-1')
pywikibot.output(
dico_timestamp[titre_article[2:-2].replace(
' ', '_')].strftime(format_date))
try:
liste_nouveaux_recents_string += (
' (' + dico_timestamp[titre_article[2:-2].replace(
' ',
'_')].strftime(format_date).decode('utf-8') +
')')
except:
try:
liste_nouveaux_recents_string += (
' (' +
dico_timestamp[titre_article[2:-2].replace(
' ', '_')].strftime(format_date) + ')')
except:
raise "erreur au stade 6-1"
elif dico_dates_presentes.has_key(titre_article):
pywikibot.output('stade 6-2')
pywikibot.output(dico_dates_presentes[titre_article])
try:
liste_nouveaux_recents_string += (
' (' + dico_dates_presentes[titre_article] + ')')
except: # UnicodeEncodeError:
try:
liste_nouveaux_recents_string += (
' (' +
dico_dates_presentes[titre_article].decode(
'utf-8') + ')')
except:
raise "erreur au stade 6-2"
liste_nouveaux_recents_string += u"\n<!-- Ce tableau est créé automatiquement par un robot. Articles Récents FIN -->"
#####################
#####################
### Mise à jour du contenu de la page
#####################
new_text = text
pywikibot.output('stade 7')
# Mise à jour de la liste des articles récents (listeRecents)
new_text = re.sub(
re.compile(u'%s.*%s' % (matchDebut1, matchFin1), re.S),
liste_nouveaux_recents_string, new_text)
pywikibot.output(new_text)
pywikibot.output(u'Commentaire: %s' % comment)
if not dry:
main_page.put(new_text, comment=comment)
end_page(main_page, now, first_passage)
else:
pywikibot.showDiff(main_page.get(), new_text)
#####################
except Exception, myexception:
pywikibot.output("Erreur lors du traitement de la page %s" %
main_page.title(asLink=True))
_errorhandler.handle(
myexception,
level='warning',
addtags={'page': main_page.title(asLink=True)})