def get_template_parameters_from_article(self, article, templates,
template_redirects):
legal_chars = r'[ %!"$&\'()*,\-.0-9:;?@A-Z^_`a-z~\x80-\xFF]'
article_parameters = set()
inner_template_re = re.compile(r'\{\{[^}]+\}\}', re.I | re.MULTILINE)
parameter_re = re.compile(r'\|\s*(' + legal_chars + r'+)\s*=',
re.I | re.MULTILINE)
article_text = wikitools.Page(self.wiki,
article).getWikiText().decode('utf-8')
for template in templates:
template_content = self.grab_template(article_text,
template_redirects)
if not template_content:
continue
for match in inner_template_re.finditer(template_content[2:]):
template_redirects = legal_chars + '+'
inner_template = self.grab_template(template_content[2:],
template_redirects)
if inner_template:
template_content = re.sub(re.escape(inner_template), '',
template_content)
break
if template_content:
for match in parameter_re.finditer(template_content):
article_parameter = match.group(1).strip()
article_parameters.add(article_parameter)
return article_parameters
def get_input():
page = wikitools.Page(wiki,
'Wikidata:Database reports/Missing links/Input')
text = page.getWikiText()
for line in text.splitlines():
if not line.startswith(('#', '<')) and line.endswith('wiki'):
yield line.strip()
def change_status(pagename):
page = wikitools.Page(wiki, "Page:" + pagename, followRedir=True)
print page
logging.info("Editing " + "https://ta.wikisource.org/wiki/" + page.title)
content = page.getWikiText()
list_of_words = content.split()
wiki_username_string = 'user='******'"' + wiki_username + '"'
# print wiki_username_string
for n, i in enumerate(list_of_words):
if "user="******"1"',
'pagequality level="3"')
page.edit(text=new_content,
summary="[[பகுப்பு:மெய்ப்புப் பார்க்கப்பட்டவை]]")
logging.info("Changed level 1 to 3")
def run(db, first, second):
title = base + '/{0} not in {1}'.format(first, second)
page = wikitools.Page(wiki, title)
text = mk_report(db, first, second)
text = header.format(first, second, replag(db)) + text
page.edit(text, summary='Bot: Updating database report', bot=1)
return '\n*[[{0}|Pages with "{1}" but not "{2}"]]'.format(
title, get_label(db, first), get_label(db, second))
def write_page(self, title, text, comment):
if self.online and self.live:
wikipage = wikitools.Page(self.site, title)
text = text.encode('utf8')
wikipage.edit(text=text, summary=comment)
print(u'Changed Page %s.' % title)
else:
print(u'Would have changed page %s if live (%s)' % (title, comment))
def postSection(bothmonth_data):
page = wikitools.Page(wiki, page_namespace + page_title)
page_text = section_template % (bothmonth_data[1], bothmonth_data[0])
# page_text = page_text.encode('utf-8')
page.edit(page_text,
section="new",
sectiontitle="Page views",
summary="HostBot is updating monthly metrics on page views",
bot=1)
def delete_page(pagename):
page = wikitools.Page(wiki, pagename, followRedir=True)
logging.info("deleting " + "https://ta.wikisource.org/wiki/" + page.title)
page.delete(reason=u"தரவுப் பிழைக்காக இந்தப் பக்கம் நீக்கப்பட்டது")
logging.info("deleted " + pagename)
def clearCheckinList():
page_path = page_namespace + sub_domain
page = wikitools.Page(wiki, page_path)
page.edit(
checkin_template,
section=1,
summary=
"HostBot is automatically clearing the list of recently [[WP:Teahouse/Host_checkin|checked in]] hosts. Profiles in [[WP:Teahouse/Host_breakroom]] will be move to [[WP:Teahouse/Host_landing]]",
bot=1)
def postSection(datetime, count, metrics):
page = wikitools.Page(wiki, page_namespace + page_title)
page_text = section % (datetime, count, metrics)
# page_text = page_text.encode('utf-8')
page.edit(
page_text,
section="new",
sectiontitle="Hosts",
summary="HostBot is updating monthly metrics on host participation",
bot=1)
def messageUsers():
for name in recipients:
page_title = page_namespace + name
page = wikitools.Page(wiki, page_title)
page.edit(
tmplt,
section="new",
summary=
"Automatic reminder to participate in [[Grants_Advisory_Committee|GAC]] activities",
bot=1)
def moveReactiveProfiles(all_reactive_profiles):
landing_string = page_namespace + host_landing
removed = wikitools.Page(wiki, landing_string)
removed_profiles = host_landing_template % '\n'.join(all_reactive_profiles)
removed_profiles = removed_profiles.encode('utf-8')
removed.edit(
removed_profiles,
section=2,
summary=
"HostBot is automatically moving profiles of recently active hosts from [[WP:Teahouse/Host_breakroom]]",
bot=1)
def main():
page = wikitools.Page(wiki, base)
db = oursql.connect(db='wikidatawiki_p',
host="sql-s5",
read_default_file=os.path.expanduser("~/.my.cnf"),
charset=None,
use_unicode=False)
text = 'This report spans multiple subpages. It was last run at <onlyinclude>~~~~~</onlyinclude>.'
text += run(db, 'P17', 'P30') #country w/o continent
text += run(db, 'P21', 'P107') #sex w/o entity type
page.edit(text, summary='Bot: Updating database report', bot=1)
def moveInactiveProfiles(all_inactive_profiles):
breakroom_string = page_namespace + host_breakroom
removed = wikitools.Page(wiki, breakroom_string)
removed_profiles = breakroom_template % '\n'.join(all_inactive_profiles)
removed_profiles = removed_profiles.encode('utf-8')
removed.edit(
removed_profiles,
section=2,
summary=
"HostBot is automatically moving profiles for currently inactive hosts from [[WP:Teahouse/Host_landing]]",
bot=1)
def postSection(cur_datetime, q_tot_count, bothmonth_data):
page = wikitools.Page(wiki, page_namespace + page_title)
page_text = questions_section % (cur_datetime, q_tot_count,
bothmonth_data[1], bothmonth_data[0])
# page_text = page_text.encode('utf-8')
page.edit(
page_text,
section="new",
sectiontitle="Profiles",
summary="HostBot is updating monthly metrics on profile creation",
bot=1)
def returnActiveProfiles(all_active_profiles):
landing_string = page_namespace + host_landing
returned = wikitools.Page(wiki, landing_string)
returned_profiles = host_landing_template % '\n'.join(all_active_profiles)
returned_profiles = returned_profiles.encode('utf-8')
returned.edit(
returned_profiles,
section=1,
summary=
"HostBot is automatically moving profiles for currently inactive hosts to [[WP:Teahouse/Host_breakroom]]",
bot=1)
def main():
page = wikitools.Page(wiki, 'Wikidata:Database reports/Missing links')
db = oursql.connect(db='wikidatawiki_p',
host="wikidatawiki-p.rrdb.toolserver.org",
read_default_file=os.path.expanduser("~/.my.cnf"),
charset=None,
use_unicode=False)
sites = get_input()
report_text = report_template
for site in sites:
report = mk_report(db, site)
repl = replag(db)
text = template.format(repl, site=site) + report + footer
rep_page = wikitools.Page(
wiki, 'Wikidata:Database reports/Missing links/' + site)
rep_page.edit(text, summary='Bot: Updating database report', bot=1)
report_text += '\n*[[Wikidata:Database reports/Missing links/{0}|{0}]]'.format(
site)
#print '----'
#print text
page.edit(report_text, summary='Bot: Updating database report', bot=1)
def returnReorderedProfiles(all_profiles):
report_title = page_namespace + page_section
report = wikitools.Page(wiki, report_title)
profiles = page_template % '\n'.join(all_profiles)
profiles = profiles.encode('utf-8')
report.edit(
profiles,
section=1,
summary=
"HostBot is moving new hosts and highly active hosts to the top, in random order",
bot=1)
def updateFeaturedHosts(featured_profiles):
i = 1
for profile in featured_profiles:
report_title = page_namespace + page_section % i
report = wikitools.Page(wiki, page_namespace + page_section % i)
report_text = report_template % (profile[0], profile[1])
i += 1
report.edit(
report_text,
summary=
"Automatic update of [[Wikipedia:Teahouse/Host/Featured|featured host gallery]] by [[User:HostBot|HostBot]]",
bot=1)
def main():
page = wikitools.Page(wiki, 'Wikidata:Database reports/Popular properties')
db = oursql.connect(db='wikidatawiki_p',
host="sql-s5",
read_default_file=os.path.expanduser("~/.my.cnf"),
charset=None,
use_unicode=False)
report = mk_report(db)
repl = replag(db)
text = template.format(repl) + report + footer
#print '----'
#print text
page.edit(text, summary='Bot: Updating database report', bot=1)
def get_page(self, title):
if self.online:
wikipage = wikitools.Page(self.site, title)
if wikipage.isRedir():
return None
try:
text = wikipage.getWikiText()
text = unicode(text, 'utf-8')
return self.page_class(title=title, text=text)
except (wikitools.NoPage):
return None
else:
raise StandardError('Not online')
def find_page_size(pagename):
page = wikitools.Page(wiki, pagename, followRedir=True)
logging.info("Reading " + "https://ta.wikisource.org/wiki/" + page.title)
metadata = page.getHistory(content=False)
size = metadata[0]['size']
if size <= 1000:
data_file.write("#[[" + pagename + "]]" + " ~ " + str(size) + "\n")
logging.info("page size for " + pagename + " is " + str(size))
def move_page(original_pagename, new_pagename):
page = wikitools.Page(wiki, original_pagename, followRedir=True)
logging.info("Editing " + "https://" + wikisource_language +
".wikisource.org/wiki/" + page.title)
page.move(new_pagename, reason="Moved page", noredirect=True)
# page.edit(text = new_content,summary = "உரிமப்பக்கத்திற்குரிய தரவைப் பதிவேற்றியது")
# print page.getWikiText()
logging.info("moved " + original_pagename + " to " + new_pagename)
def postSection(month_data, section):
print section % (str(month_data[0]), str(month_data[1]), str(
month_data[0]), str(month_data[2]), str(month_data[1]))
page = wikitools.Page(wiki, page_namespace + page_title)
page_text = section % (str(month_data[0]), str(
month_data[1]), str(month_data[0]), str(
month_data[2]), str(month_data[1]))
# page_text = page_text.encode('utf-8')
page.edit(
page_text,
summary=
"HostBot is adding the monthly automated metrics intro section for %s/%s"
% (str(month_data[0]), str(month_data[1])),
bot=1)
def add_license_content(pagename):
page = wikitools.Page(wiki, "Page:" + pagename, followRedir=True)
print page
logging.info("Editing " + "https://ta.wikisource.org/wiki/" + page.title)
new_content = """
<big><center>'''உலகளாவிய பொதுக் கள உரிமம் (CC0 1.0)'''</center></big>
இது சட்ட ஏற்புடைய உரிமத்தின் சுருக்கம் மட்டுமே. முழு உரையை https://creativecommons.org/publicdomain/zero/1.0/legalcode என்ற முகவரியில் காணலாம்.
'''பதிப்புரிமை அற்றது'''
இந்த ஆக்கத்துடன் தொடர்புடையவர்கள், உலகளளாவிய பொதுப் பயன்பாட்டுக்கு என பதிப்புரிமைச் சட்டத்துக்கு உட்பட்டு, தங்கள் அனைத்துப் பதிப்புரிமைகளையும் விடுவித்துள்ளனர்.
நீங்கள் இவ்வாக்கத்தைப் படியெடுக்கலாம்; மேம்படுத்தலாம்; பகிரலாம்; வேறு வடிவமாக மாற்றலாம்; வணிகப் பயன்களும் அடையலாம். இவற்றுக்கு நீங்கள் ஒப்புதல் ஏதும் கோரத் தேவையில்லை.
<center>'''***'''</center>
இது, உலகத் தமிழ் விக்கியூடகச் சமூகமும் ( https://ta.wikisource.org ), தமிழ் இணையக் கல்விக் கழகமும் ( http://tamilvu.org ) இணைந்த கூட்டுமுயற்சியில், பதிவேற்றிய நூல்களில் ஒன்று. இக்கூட்டு முயற்சியைப் பற்றி, https://ta.wikisource.org/s/4kx என்ற முகவரியில் விரிவாகக் காணலாம்.
:::[[File:CC-Zero-badge.svg|100px|left]] [[File:CC-logo.svg|100px|center]]
<big><center>'''Universal (CC0 1.0) Public Domain Dedication'''</center></big>
This is a human readable summary of the legal code found at https://creativecommons.org/publicdomain/zero/1.0/legalcode
'''No Copyright'''
The person who associated a work with this deed has dedicated the work to the public domain by waiving all of his or her rights to the work worldwide under copyright law including all related and neighboring rights, to the extent allowed by law.
You can copy, modify, distribute and perform the work even for commercial purposes, all without asking permission.
<center>'''***'''</center/>
This book is uploaded as part of the collaboration between Global Tamil Wikimedia Community ( https://ta.wikisource.org ) and Tamil Virtual Academy ( http://tamilvu.org ). More details about this collaboration can be found at https://ta.wikisource.org/s/4kx.
"""
page.edit(text=new_content, summary="added license content")
logging.info("Added license content")
def add_license_content(pagename):
page = wikitools.Page(wiki, "Page:" + pagename, followRedir=True)
print page
logging.info("Editing " + "https://ta.wikisource.org/wiki/" + page.title)
new_content = """{{nop}}
[[File:""" + str(pagename.split("/1")[0]) + """|center|240px]]
{{nop}}
"""
page.edit(text=new_content, summary="அட்டைப்படம் சேர்க்கப்பட்டது")
logging.info("Added cover image")
def messageUsers():
for name in recipients:
page_title = page_namespace + name
page = wikitools.Page(wiki, page_title)
try:
page.edit(message_template, section="new", sectiontitle="== Individual Engagement Grant proposals due 15 February 2013 ==", summary="Automatic reminder to complete a submitted [[Grants:IEG|Individual Engagement Grant Proposal]]", bot=1)
try: #update the db to show that this user has been reminded
cursor.execute('UPDATE ieg_proposals SET pc_reminded = 1 WHERE pc_username = "******"' % (name,))
conn.commit()
except:
logging.info('UPDATE: Could not update reminded status for User:'******' at ' + curtime)
continue
except:
logging.info('REMIND: Reminder to User:'******' failed at to send at ' + curtime)
continue
def talkpageCheck(guest, header):
skip_test = False
try:
tp = wikitools.Page(wiki, 'User talk:' + guest)
contents = unicode(tp.getWikiText(), 'utf8')
for template in skip_templates:
if template in contents:
skip_test = True
allowed = allow_bots(contents, settings.username)
if not allowed:
skip_test = True
except:
logging.info('Guest ' + guest + ' failed on talkpageCheck ' + curtime)
return skip_test
def inviteGuests(cursor):
for invitee in invite_list:
host = select_host(curHosts)
invite_title = page_namespace + invitee
invite_page = wikitools.Page(wiki, invite_title)
invite_text = invite_template % (host, host, '|signature=~~~~')
invite_text = invite_text.encode('utf-8')
try:
invite_page.edit(invite_text, section="new", sectiontitle="== {{subst:PAGENAME}}, you are invited to the Teahouse ==", summary="Automatic invitation to visit [[WP:Teahouse]] sent by [[User:HostBot|HostBot]]", bot=1)
except:
logging.info('Guest ' + invitee + ' failed on invitation ' + curtime)
continue
try:
# invitee = MySQLdb.escape_string(invitee)
cursor.execute('''update jmorgan.th_up_invitees set invite_status = 1, hostbot_invite = 1, hostbot_personal = 1 where user_name = %s ''', (invitee,))
conn.commit()
except UnicodeDecodeError:
logging.info('Guest ' + invitee + ' failed on invite db update due to UnicodeDecodeError ' + curtime)
continue
def __init__(self, title, description, xml_file, page_class, site, username, live=True, online=True, memory_file_name=None):
self.title = title
self.live = live
self.log_frequency = 20
self.online = online
if online:
self.site = site
self.description = description
self.words = []
self.xml_file = xml_file
self.page_class = page_class
self.memory = FixMemory(memory_file_name)
self.username = username
if self.online:
self.wikilogpage = wikitools.Page(site, u'User:%s/Log' % self.username)
self.log_page = LogPage(title=self.wikilogpage.title, text=self.wikilogpage.getWikiText()).parse()
self.site.login(self.username)
else:
self.log_page = LogPage(title=u'Bot Log Page', text='').parse()
self.log_section = None
def compute_fixed_snippets(cfg):
log.info('computing fixed snippets for %s' % cfg.lang_code)
live_db = chdb.init_db(cfg.lang_code)
stats_db = chdb.init_stats_db()
# Load snippets that have been clicked in the past few hours
to_ts = datetime.datetime.today()
from_ts = to_ts - datetime.timedelta(hours=3)
page_title_to_snippets = stats_db.execute_with_retry(
load_pages_and_snippets_to_process, cfg.lang_code, from_ts, to_ts)
if not page_title_to_snippets:
log.info('No pages to process!')
return
log.info('Will reparse pages: %r' % page_title_to_snippets.keys())
# Now fetch and parse the pages and check which snippets are gone
wiki = wikitools.wiki.Wiki('https://' + cfg.wikipedia_domain +
'/w/api.php')
wiki.setUserAgent('citationhunt (https://tools.wmflabs.org/citationhunt)')
parser = snippet_parser.create_snippet_parser(wiki, cfg)
for page_title, snippet_to_ts in page_title_to_snippets.items():
page = wikitools.Page(wiki, page_title)
snippets = parser.extract(page.getWikiText())
# FIXME Duplicated logic with parse_live.py :(
for sec, snips in snippets:
for sni in snips:
id = mkid(d(page.title) + sni)
snippet_to_ts.pop(id, None)
for snippet_id, clicked_ts in snippet_to_ts.items():
log.info(snippet_id)
stats_db.execute_with_retry_s(
'INSERT IGNORE INTO fixed VALUES (%s, %s, %s)', clicked_ts,
snippet_id, cfg.lang_code)
live_db.close()
stats_db.close()
return 0
