def get_all_sites(self):
dummypage = Page(self.wiki, 'Benutzer:THEbotIT/dummy')
dummypage.text = '{{subst:NUMBEROFARTICLES}}'
dummypage.save('get_new_number')
del dummypage
dummypage = Page(self.wiki, 'Benutzer:THEbotIT/dummy')
return '{}'.format(dummypage.text)
def process(day):
"""
one day bot processing
arguments:
day -- python date format
"""
if params.verbose:
print("processing Journal des recréations ({day})".format(day=format_date(day)))
start = to_date(day)
end = to_date(day+ONE_DAY)
result = "\n== {} ==\n".format(format_date(day))
for i,page in enumerate(creation_log(start,end),1):
if params.verbose:
print (i,page["timestamp"])
dl = deletelog(page["title"])
if dl:
r = ("* {{{{a-court|{title}}}}} <small>([[{pas}|PàS]])</small> supprimé le {date} recréé par {{{{u|{user}}}}} \n"
.format(title = wiki_param(page["title"]) ,
pas = wiki_param("Discussion:"+page["title"]+"/Suppression"),
user = wiki_param(page["user"]),
date = format_date(from_date(dl["timestamp"]))))
if params.verbose:
print(r)
result += r
page = Page(Site(), params.prefix+"/"+format_date(day,skip_day=True))
try:
result = page.get()+result
except NoPage:
pass
page.put(result,comment="Journal des recréations ({day})".format(day=format_date(day)))
def task(self): # pragma: no cover
page = Page(self.wiki, "Benutzer:THE IT/Werkstatt")
new_text = self.replace_in_page(page.text)
page.text = new_text
if not self.debug:
page.save("Time is ticking")
return True
def get_wiki_bearbeitungen(self):
dummypage = Page(self.wiki, 'Benutzer:THEbotIT/dummy')
dummypage.text = '{{subst:NUMBEROFEDITS}}'
dummypage.save('get_new_number')
del dummypage
dummypage = Page(self.wiki, 'Benutzer:THEbotIT/dummy')
return dummypage.text
class WsStatus(CanonicalBot):
def __init__(self, wiki, debug):
CanonicalBot.__init__(self, wiki, debug)
self.text = None
self.stat_page = None
def task(self):
if self.debug:
lemma = 'Benutzer:THEbotIT/' + self.bot_name
else:
lemma = 'WS:Statistik'
self.load_text_from_site(lemma)
# self.new_row(str(RowSeitenstatistik(self.wiki, self.logger)), 'SEITENSTATISTIK')
self.new_row(str(RowBearbeitungsstand(self.wiki, self.logger)), 'BEARBEITUNGSSTAND')
self.save_text_to_site()
return True
def new_row(self, row, placeholder):
self.text = re.sub('<!--BOT:{}-->'.format(placeholder), '<!--BOT:{}-->\n{}'
.format(placeholder, row), self.text)
def load_text_from_site(self, lemma):
self.logger.info('Load text from {}'.format(lemma))
self.stat_page = Page(self.wiki, lemma)
self.text = self.stat_page.text
def save_text_to_site(self):
self.stat_page.text = self.text
self.stat_page.save('Statistik wurde aktualisiert.', botflag=True)
def finish_task(self):
if self.data:
if not self.debug:
page = Page(self.wiki, "RE:Wartung:Strukturfehler")
page.text = page.text + self._build_entry()
page.save("Neue Fehlermeldungen", botflag=True)
super().finish_task()
def find_sub_templates(
lookingfor: str, page: Page, wholeword: bool, matchcase: bool
):
found_templates = []
if page.isRedirectPage():
page = page.getRedirectTarget()
pagetext = page.text
if not matchcase:
pagetext = pagetext.lower()
lookingfor = lookingfor.lower()
if wholeword:
pattern = re.compile(r'\b' + re.escape(lookingfor) + r'\b')
if pattern.search(pagetext):
found_templates.append(page)
elif lookingfor in pagetext:
found_templates.append(page)
for sub_template in page.templates(content=True):
if sub_template.isRedirectPage():
sub_template = sub_template.getRedirectTarget()
text = sub_template.text if matchcase else sub_template.text.lower()
if wholeword:
# noinspection PyUnboundLocalVariable
if pattern.search(text):
found_templates.append(sub_template)
elif lookingfor in text:
found_templates.append(sub_template)
# Remove duplicate templates
return {f.title(): f for f in found_templates}.values()
def history(self, fertig, korrigiert, unkorrigiert):
page = Page(self.wiki, "Benutzer:THEbotIT/" + self.bot_name)
temp_text = page.text
composed_text = "".join(["|-\n", "|", self.timestamp.start_of_run.strftime("%Y%m%d-%H%M"),
"||", str(unkorrigiert[1]), "||", str(unkorrigiert[0]),
"||", str(int(unkorrigiert[0] / unkorrigiert[1])),
"||", str(korrigiert[1]), "||", str(korrigiert[0]),
"||", str(int(korrigiert[0] / korrigiert[1])),
"||", str(fertig[1]), "||", str(fertig[0]),
"||", str(int(fertig[0] / fertig[1])),
"\n<!--new line-->"])
temp_text = re.sub("<!--new line-->", composed_text, temp_text)
page.text = temp_text
page.save("RE Statistik aktualisiert", botflag=True)
def make_magazines(self, dictionary_of_magazines_by_year):
for idx_year, year in enumerate(dictionary_of_magazines_by_year):
magazines = dictionary_of_magazines_by_year[year]
self.logger.debug(f"make_mag_year {idx_year + 1}/"
f"{len(dictionary_of_magazines_by_year)}")
for idx_mag, magazine in enumerate(magazines):
self.logger.debug("make_mag_mag {idx}/{len} ... issue:{year}/{mag}"
.format(idx=idx_mag + 1, len=len(magazines),
year=year, mag=magazine))
if year == "1986" and magazine == "31":
self.logger.warning("There is magazine 1986, 31, "
"this is special, no creating here")
continue
if self.debug:
lemma = Page(self.wiki, "Benutzer:THEbotIT/Test")
else:
lemma = Page(self.wiki, f"Die Gartenlaube ({year})/Heft {int(magazine):d}")
new_text = self.make_magazine(year, magazine)
if new_text:
if hash(new_text.strip()) != hash(lemma.text.strip()):
self.logger.debug("Print [[Die Gartenlaube ({year})/Heft {magazine}]]."
.format(year=year, magazine=magazine))
if lemma.text != '':
lemma.text = new_text
lemma.save("Automatische Aktualisierung des Heftes", botflag=True)
else:
lemma.text = new_text
lemma.save("automatische Hefterstellung", botflag=True)
else:
self.logger.debug("Keine Änderung im Text ({year}/{magazine})."
.format(year=year, magazine=magazine))
def task(self): # pragma: no cover
regex = re.compile(r"\n\|PND=")
searcher = PetScan()
searcher.add_yes_template("ADBDaten")
searcher.add_positive_category("ADB:Ohne GND-Link")
lemma_list = searcher.run()
for lemma in lemma_list:
page = Page(self.wiki, lemma["title"])
temp_text = page.text
if regex.search(temp_text):
self.logger.info(f"change {lemma['title']}")
temp_text = regex.sub("\n|GND=", temp_text)
page.text = temp_text
page.save("PND -> GND", botflag=True)
return True
def task(self):
lemma_list = self._run_searcher()
self._build_database(lemma_list)
if self.debug:
dump = Page(self.wiki, f"Benutzer:THEbotIT/{self.bot_name}")
else:
dump = Page(self.wiki, "Liste der Autoren")
old_text = dump.text
new_text = self._convert_to_table()
if new_text[150:] != old_text[150:]: # compare all but the date
dump.text = new_text
dump.save("Die Liste wurde auf den aktuellen Stand gebracht.", botflag=True)
else:
self.logger.info("Heute gab es keine Änderungen, "
"daher wird die Seite nicht überschrieben.")
return True
def task(self): # pragma: no cover
error_task = ERROTask(wiki=self.wiki, debug=False, logger=self.logger)
for lemma in self.search_pages():
page = Page(self.wiki, lemma["title"])
temp_text = page.text
try:
temp_text = self.convert_all(temp_text)
page.text = temp_text
re_page = RePage(page)
if not self.debug:
re_page.save("Entfernen veralteter Vorlagen.")
except (ReDatenException, ValueError):
error = traceback.format_exc().splitlines()[-1]
error_task.task(lemma["title"], error)
error_task.finish_task()
if self.search_pages():
return False
return True
def task(self):
lemma_list = self._search()
for idx, lemma in enumerate(lemma_list):
page = Page(self.wiki, title='Index:{}'.format(lemma['title']))
self.logger.info('{}/{}:{}'.format(idx, len(lemma_list), page))
match = regex_picture.search(page.text)
if match:
self.logger.info(match.group(1))
temp = re.sub('\|\d{2,3}px', '', match.group(1))
if not re.search('thumb', match.group(1)):
temp = temp + '|thumb'
self.logger.info(temp)
if temp == match.group(1):
self.logger.info('nothing to do here.')
continue
temp = '|BILD=[[{}]]'.format(temp)
temp_text = regex_picture.sub(temp, page.text)
page.text = temp_text
page.save(botflag=True, summary='set thumb as parameter')
return True
def check_page(self, pagename):
"""Check one page."""
pywikibot.output("\nChecking %s" % pagename)
sys.stdout.flush()
page1 = Page(self.original, pagename)
txt1 = page1.text
if self.options.dest_namespace:
dest_ns = int(self.options.dest_namespace)
else:
dest_ns = None
for site in self.sites:
if dest_ns is not None:
page2 = Page(site, page1.title(withNamespace=False), dest_ns)
pywikibot.output("\nCross namespace, new title: %s"
% page2.title())
else:
page2 = Page(site, pagename)
if page2.exists():
txt2 = page2.text
else:
txt2 = ''
if str(site) in config.replicate_replace:
txt_new = multiple_replace(txt1,
config.replicate_replace[str(site)])
if txt1 != txt_new:
pywikibot.output(
'NOTE: text replaced using config.sync_replace')
pywikibot.output('%s %s %s' % (txt1, txt_new, txt2))
txt1 = txt_new
if txt1 != txt2:
pywikibot.output("\n %s DIFFERS" % site)
self.differences[site].append(pagename)
if self.options.replace:
page2.text = txt1
page2.save(self.put_message(site))
else:
sys.stdout.write('.')
sys.stdout.flush()
def generate_overviews(self):
"""Create page on wikis with overview of bot results."""
for site in self.sites:
sync_overview_page = Page(site,
'User:%s/sync.py overview' % site.user())
output = "== Pages that differ from original ==\n\n"
if self.differences[site]:
output += "".join('* [[:%s]]\n' % l for l in
self.differences[site])
else:
output += "All important pages are the same"
output += "\n\n== Admins from original that are missing here ==\n\n"
if self.user_diff[site]:
output += "".join('* %s\n' % l.replace('_', ' ') for l in
self.user_diff[site])
else:
output += "All users from original are also present on this wiki"
pywikibot.output(output)
sync_overview_page.text = output
sync_overview_page.save(self.put_message(site))
def user_page_the_it(self, korrigiert):
status_string = []
color = make_html_color(20e6, 22e6, korrigiert[0])
status_string.append(f"<span style=\"background:#FF{color}{color}\">{korrigiert[0]}</span>")
color = make_html_color(5.0e3, 5.25e3, korrigiert[1])
status_string.append(f"<span style=\"background:#FF{color}{color}\">{korrigiert[1]}</span>")
list_of_lemmas = self.petscan(["RE:Teilkorrigiert", "RE:Korrigiert"],
["RE:Unkorrigiert", "RE:Unvollständig"])
date_page = Page(self.wiki, list_of_lemmas[0]["title"])
date_of_first = str(date_page.oldest_revision.timestamp)[0:10]
gap = datetime.now() - datetime.strptime(date_of_first, "%Y-%m-%d")
color = make_html_color(3 * 365, 3.5 * 365, gap.days)
status_string.append(f"<span style=\"background:#FF{color}{color}\">{date_of_first}</span>")
user_page = Page(self.wiki, "Benutzer:THE IT/Werkstatt")
temp_text = user_page.text
temp_text = re.sub(r"<!--RE-->.*<!--RE-->",
f"<!--RE-->{' ■ '.join(status_string)}<!--RE-->",
temp_text)
user_page.text = temp_text
user_page.save("todo RE aktualisiert")
def task(self):
if self.debug: # activate for debug purpose
lemma = "Benutzer:THEbotIT/" + self.bot_name
else:
lemma = "Die Gartenlaube"
page = Page(self.wiki, lemma)
temp_text = page.text
alle = self.petscan([])
fertig = self.petscan(["Fertig"])
korrigiert = self.petscan(["Korrigiert"])
unkorrigiert = self.petscan(["Unkorrigiert"])
articles = self.petscan([], article=True, not_categories=["Die Gartenlaube Hefte"])
temp_text = self.projektstand(temp_text, alle, fertig, korrigiert, unkorrigiert, articles)
temp_text = self.alle_seiten(temp_text, alle)
temp_text = self.korrigierte_seiten(temp_text, korrigiert)
temp_text = self.fertige_seiten(temp_text, fertig)
for year in range(1853, 1900):
temp_text = self.year(year, temp_text)
page.text = temp_text
page.save("Ein neuer Datensatz wurde eingefügt.", botflag=True)
return True
def load_text_from_site(self, lemma):
self.logger.info('Load text from {}'.format(lemma))
self.stat_page = Page(self.wiki, lemma)
self.text = self.stat_page.text
def send_log_to_wiki(self):
wiki_log_page = f"Benutzer:THEbotIT/Logs/{self.bot_name}"
page = Page(self.wiki, wiki_log_page)
page.text += self.logger.create_wiki_log_lines()
page.save(f"Update of Bot {self.bot_name}", botflag=True)
def process(day):
"""
one day bot processing
arguments:
day -- python date format
"""
if params.verbose:
print("processing Journal des recréations ({day})".format(day=format_date(day)))
start = to_date(day)
end = to_date(day+ONE_DAY)
result = "\n\n== {} ==\n".format(format_date(day))
comment = []
for i,page in enumerate(creation_log(start,end),1):
gras = ''
date = ''
if params.verbose:
print (i,page["timestamp"])
dl = deletelog(page["title"])
if dl:
page_pas = Page(Site(), "Discussion:" + page["title"] + "/Suppression")
if page_pas.isRedirectPage():
page_pas = page_pas.getRedirectTarget()
if page_pas.exists() and re.search(r'article supprimé', page_pas.get(), re.I):
if re.search(r'\{\{ ?article supprimé[^\}]*\d{1,2} (\S* \d{4}) à', page_pas.get(), re.I):
date = u' de %s' % re.search(r'\{\{ ?article supprimé[^\}]*\d{1,2} (\S* \d{4}) à', page_pas.get(), re.I).group(1)
comment.append(u'[[%s]] (malgré [[%s|PàS]]%s)' % (page["title"], page_pas.title(), date))
gras = "'''"
r = (u"* {g}{{{{a-court|{title}}}}} <small>([[{pas}|PàS]])</small> supprimé le {date} puis recréé par {{{{u|{user}}}}}{g} \n"
.format(title = wiki_param(page["title"]),
pas = page_pas.title(),
user = wiki_param(page["user"]),
date = format_date(from_date(dl["timestamp"])),
g = gras))
if params.verbose:
print(r)
result += r
page = Page(Site(), params.prefix + u'/' + format_date(day, skip_day=True))
try:
result = page.get() + result
except NoPage:
result = u'{{mise à jour bot|Zérobot}}' + result
if comment: comment.insert(0, '')
page.put(result,comment="Journal des recréations ({day}) ".format(day=format_date(day)) + ' - '.join(comment))
import sys
sys.path.append('../../')
import re
from pywikibot import Page, Site
from tools.petscan import PetScan
wiki = Site()
searcher = PetScan()
searcher.add_positive_category("RE:Korrigiert")
lemma_list = searcher.run()
list_for_pfaerrich = []
for idx_lem, lemma in enumerate(lemma_list):
print(idx_lem)
page = Page(wiki, lemma['title'])
version_history = page.fullVersionHistory()[::-1]
size_all_changes = 0
for idx_rev, revision in enumerate(version_history):
user = revision.user
if user == 'Pfaerrich':
if idx_rev > 0:
size_prev = len(version_history[idx_rev-1].text)
else:
size_prev = 0
size_all_changes += abs(len(version_history[idx_rev].text) - size_prev)
korrigiert_flag = False
if size_all_changes > 0:
for version in page.getVersionHistory():
if version.user == 'Pfaerrich':
if re.search('orrigiert', version.comment):
def save_if_changed(page: Page, text: str, change_msg: str):
if text.rstrip() != page.text:
page.text = text
page.save(change_msg, botflag=True)
# -*- coding: utf-8 -*-
__author__ = 'eso'
import sys
sys.path.append('../../')
import re
from pywikibot import Page, Site
from tools.petscan import PetScan
wiki = Site()
searcher = PetScan()
searcher.add_yes_template('Biel')
lemma_list = searcher.run()
for idx, lemma in enumerate(lemma_list):
print(idx, len(lemma_list), lemma['title'])
link_page = Page(wiki, lemma['title'])
temp_text = link_page.text
if re.search('\{\{Biel\|', temp_text):
temp_text = re.sub('\{\{Biel\|1240647\}\}', '{{Bielefeld|1240647}}', temp_text)
temp_text = re.sub('\{\{Biel\|590504\}\}', '{{Bielefeld|590504}}', temp_text)
temp_text = re.sub('\{\{Biel\|1732676\}\}', '{{Bielefeld|1732676}}', temp_text)
temp_text = re.sub('\{\{Biel\|548435\}\}', '{{Bielefeld|548435}}', temp_text)
temp_text = re.sub('\{\{Biel\|32920\}\}', '{{Bielefeld|32920}}', temp_text)
if link_page.text != temp_text:
link_page.text = temp_text
link_page.save(botflag=True, summary='Biel -> Bielefeld')
