def review(site, token, page):
"""Review the latest revision of the given page."""
revid = page.latest_revision_id
request = Request(site=site,
action="review",
token=token,
revid=revid)
request.submit()
def RecentChanges(limit):
limitString = limit
time_format = "%Y-%m-%dT%H:%M:%SZ"
limit = datetime.datetime.fromtimestamp(time.mktime(time.strptime(limitString, time_format)))
current = datetime.datetime.now()
list = set()
params = {
'action' :'query',
'list' :'recentchanges',
'rcprop' :'timestamp|title',
'rclimit' :'100',
'rcnamespace': 0
}
while current > limit:
textDate = current.strftime(time_format)
params['rcstart'] = textDate
req = Request(**params)
qr = req.submit()
try: current = datetime.datetime.fromtimestamp(time.mktime(time.strptime(qr['query-continue']['recentchanges']['rcstart'], time_format)))
except KeyError:
for elem in qr['query']['recentchanges']:
list.add(elem['title'])
break
else:
for elem in qr['query']['recentchanges']:
list.add(elem['title'])
return list
def RecentChanges(limit):
limitString = limit
time_format = "%Y-%m-%dT%H:%M:%SZ"
limit = datetime.datetime.fromtimestamp(
time.mktime(time.strptime(limitString, time_format)))
current = datetime.datetime.now()
list = set()
params = {
'action': 'query',
'list': 'recentchanges',
'rcprop': 'timestamp|title',
'rclimit': '100',
'rcnamespace': 0
}
while current > limit:
textDate = current.strftime(time_format)
params['rcstart'] = textDate
req = Request(**params)
qr = req.submit()
try:
current = datetime.datetime.fromtimestamp(
time.mktime(
time.strptime(
qr['query-continue']['recentchanges']['rcstart'],
time_format)))
except KeyError:
for elem in qr['query']['recentchanges']:
list.add(elem['title'])
break
else:
for elem in qr['query']['recentchanges']:
list.add(elem['title'])
return list
def submit(self, parameters):
""" submit the request with the given parameters
Args:
parameters(list): the parameters to use for the SMW API request
Returns:
dict: the submit result"""
request = Request(site=self.site, parameters=parameters)
return request.submit()
def pageCounter(language):
#returns number of entries for a language
params = {
'action': 'expandtemplates',
'text': '{{PAGESINCAT:%s (indeks)|R}}' % language,
}
req = Request(**params)
qr = req.submit()
print(qr['expandtemplates']['*'])
def pageCounter(language):
#returns number of entries for a language
params = {
'action' :'expandtemplates',
'text' :'{{PAGESINCAT:%s (indeks)|R}}' % language,
}
req = Request(**params)
qr = req.submit()
print(qr['expandtemplates']['*'])
def convert_title(title):
req = Request(site=site, parameters={
'action': 'query',
'titles': title,
"redirects": 1,
"converttitles": 1
})
data = req.submit()
new_title = list(data['query']['pages'].values())[0]['title']
return new_title
def converttitle(title):
r = Request(site=site,
parameters={
'action': 'query',
'titles': title,
'redirects': 1,
'converttitles': 1
})
data = r.submit()
return list(data['query']['pages'].values())[0]['title']
def submit(self, parameters):
""" submit the request with the given parameters
Args:
parameters(list): the parameters to use for the SMW API request
Returns:
dict: the submit result"""
if not "Request" in sys.modules:
from pywikibot.data.api import Request
request = Request(site=self.site, parameters=parameters)
return request.submit()
def converttitle(site, title):
r = Request(site=site, parameters={
'action': 'query',
'titles': title,
'redirects': 1,
'converttitles': 1
})
data = r.submit()
page = list(data['query']['pages'].values())[0]
if 'missing' in page:
return None
return page['title'].replace(' ', '_')
def flagLastRev(site, revid, comment=''):
token = site.getToken(sysop=False)
params = {
'site': site,
'action': 'review',
'revid': revid,
'token': token,
'flag_accuracy': 1,
'comment': comment,
}
req = Request(**params)
query = req.submit()
def flagLastRev(site, revid, comment=''):
token = site.getToken(sysop=False)
params = {
'site': site,
'action': 'review',
'revid': revid,
'token': token,
'flag_accuracy': 1,
'comment': comment,
}
req = Request(**params)
query = req.submit()
def converttitle(title):
r = Request(site=site, parameters={
'action': 'query',
'titles': title,
"redirects": 1,
"converttitles": 1
})
data = r.submit()
newtitle = list(data['query']['pages'].values())[0]['title']
checkAfdTemplate(newtitle)
return newtitle
def _load_pages(self, titles):
"""Load a chunk of pages from the API."""
def _get_rev(page):
try:
return page["revisions"][0]["slots"]["main"]["content"]
except (KeyError, IndexError):
return ""
req = Request(self._bot.site, parameters={
"action": "query", "prop": "revisions", "rvprop": "content",
"rvslots": "main", "formatversion": "2", "titles": "|".join(titles)
})
data = req.submit()
return [(page["title"], _get_rev(page))
for page in data["query"]["pages"]]
def transclusions(pages):
from pywikibot.data.api import Request
parameters = {
'action': 'query',
'generator': 'templates',
'prop': 'revisions',
'titles': '|'.join([p.title for p in pages])
}
# Use POST in case the list of titles would cause a very long URL for GET
request = Request(_site, use_get=False, parameters=parameters)
# TODO Error handling
raw_query_data = request.submit()
return _title_and_revision_dict(raw_query_data)
def get_orlist(site=DEFAULT_SITE, namespace="0|6|10|14|100|828", redirects="nonredirects"):
"""Get list of oldreviewed pages."""
request = Request(site=site,
action="query",
list="oldreviewedpages",
ornamespace=namespace,
orfilterredir=redirects,
orlimit="5000")
result = []
while True:
answer = request.submit()
result += [page["title"] for page in answer["query"]["oldreviewedpages"]]
if "query-continue" in answer:
request["orstart"] = answer["query-continue"]["oldreviewedpages"]["orstart"]
else:
break
return result
def converttitle(title):
oldtitle = title
r = Request(site=site,
parameters={
'action': 'query',
'titles': title,
'redirects': 1,
'converttitles': 1
})
data = r.submit()
title = list(data['query']['pages'].values())[0]['title']
mode = []
if 'redirects' in data['query']: # 重定向
mode.append('redirects')
if 'converted' in data['query']: # 繁簡轉換
mode.append('converted')
if 'normalized' in data['query']: # 命名空間
mode.append('normalized')
if 'redirects' not in mode:
page = pywikibot.Page(site, title)
if not page.exists():
mode.append('vfd_on_source')
if page.exists() and (
page.content_model != 'wikitext' or page.namespace().id == 8
or re.search(
r'{{\s*([vaictumr]fd|Copyvio)', page.text, flags=re.I)):
mode.append('vfd_on_source')
else:
page = pywikibot.Page(site, oldtitle)
if page.exists() and (
page.content_model != 'wikitext' or page.namespace().id == 8
or re.search(
r'{{\s*([vaictumr]fd|Copyvio)', page.text, flags=re.I)):
mode.append('vfd_on_source')
page = pywikibot.Page(site, title)
if page.exists() and (
page.content_model != 'wikitext' or page.namespace().id == 8
or re.search(
r'{{\s*([vaictumr]fd|Copyvio)', page.text, flags=re.I)):
mode.append('vfd_on_target')
if 'vfd_on_source' not in mode and 'vfd_on_target' not in mode:
mode.append('no_vfd')
return {'title': title, 'mode': mode}
def get_pages_categories(pagelist, site=DEFAULT_SITE, limit=500):
"""
For every page from the list get list of categories and return
{page: [categories]}
dictionary.
"""
result = dict.fromkeys(pagelist, [])
kwargs = {
"action": "query",
"prop": "categories",
"cllimit": "5000"
}
for idx in range(0, len(pagelist), limit):
kwargs["titles"] = "|".join(pagelist[idx:idx+limit])
request = Request(site=site, **kwargs)
while True:
answer = request.submit()
# Wikipedia API can return page list in non-canonical form!
# At least when there are two possible canonical forms for one namespace
# (for instance, "Участник" – "Участница" in Russian Wikipedia).
# This query will normalize them and we need to handle it.
denormalize = {}
if "normalized" in answer["query"]:
for fix in answer["query"]["normalized"]:
denormalize[fix["to"]] = fix["from"]
for value in answer["query"]["pages"].values():
title = value["title"]
if title in denormalize:
title = denormalize[title]
if "categories" in value:
cats = [cat["title"] for cat in value["categories"]]
result[title] = result[title] + cats
if "query-continue" in answer:
request["clcontinue"] = answer["query-continue"]["categories"]["clcontinue"]
continue
break
return result
def active_and_future_campaigns():
from pywikibot.data.api import Request
from pywikibot import Timestamp
parameters = {
'action': 'query',
'list': 'centralnoticeactivecampaigns',
'cnacincludefuture': ''
}
request = Request(_site, parameters=parameters)
# TODO Error handling
raw_query_data = request.submit()
raw_campaigns = (
raw_query_data['query']['centralnoticeactivecampaigns']['campaigns'])
# Convert start and end to datetime objects
for c in raw_campaigns:
c['start'] = Timestamp.fromtimestampformat(c['start'])
c['end'] = Timestamp.fromtimestampformat(c['end'])
return raw_campaigns
def get_ds_alert_hits(start_date: datetime.datetime,
end_date: datetime.datetime) -> Iterator[DsAlert]:
# url = "https://en.wikipedia.org/w/api.php"
params = {
"action": "query",
"list": "abuselog",
"format": "json",
"formatversion": 2,
"aflstart": start_date.isoformat(),
"aflend": end_date.isoformat(),
"afldir": "newer",
"aflfilter": 602,
"afllimit": "max",
"aflprop": "user|title|result|timestamp|details|revid",
"continue": "",
}
for i in range(100):
logger.debug(i)
# res = session.get(url, params=params)
# res.raise_for_status()
# raw_data = res.json()
req = Request(site=site, parameters=params, use_get=True)
raw_data = req.submit()
# breakpoint()
for hit in raw_data["query"]["abuselog"]:
if hit["result"] == "tag":
for alert in parse_alert_data(hit):
yield alert
if raw_data.get("continue"):
logger.debug(f"Continue: {raw_data['continue']}")
params.update(raw_data["continue"])
else:
break
else:
# flask.abort(400)
logger.warning("Too many API queries!")
def get_pages_categories(pagelist, site=DEFAULT_SITE, limit=500):
"""
For every page from the list get list of categories and return
{page: [categories]}
dictionary.
"""
result = dict.fromkeys(pagelist, [])
kwargs = {"action": "query", "prop": "categories", "cllimit": "5000"}
for idx in range(0, len(pagelist), limit):
kwargs["titles"] = "|".join(pagelist[idx:idx + limit])
request = Request(site=site, **kwargs)
while True:
answer = request.submit()
# Wikipedia API can return page list in non-canonical form!
# At least when there are two possible canonical forms for one namespace
# (for instance, "Участник" – "Участница" in Russian Wikipedia).
# This query will normalize them and we need to handle it.
denormalize = {}
if "normalized" in answer["query"]:
for fix in answer["query"]["normalized"]:
denormalize[fix["to"]] = fix["from"]
for value in answer["query"]["pages"].values():
title = value["title"]
if title in denormalize:
title = denormalize[title]
if "categories" in value:
cats = [cat["title"] for cat in value["categories"]]
result[title] = result[title] + cats
if "query-continue" in answer:
request["clcontinue"] = answer["query-continue"]["categories"][
"clcontinue"]
continue
break
return result
if args.debug:
print('Found {} users with block log'.format(len(result)))
for row in result:
user_name = row[0].decode().replace('_', ' ')
r = Request(site=site,
parameters={
'action': 'query',
'format': 'json',
'list': 'logevents',
'letype': 'block',
'lestart': BASETIME.isoformat(),
'leend': DAYS_400_AGO.isoformat(),
'letitle': 'User:'******'lelimit': 'max'
})
data = r.submit()
unblocktime = None
# print(user_name)
for logevent in data['query']['logevents']:
if logevent['action'] == 'unblock':
unblocktime = pywikibot.Timestamp.fromISOformat(
logevent['timestamp'])
else:
if 'expiry' in logevent['params']:
expiry = pywikibot.Timestamp.fromISOformat(
logevent['params']['expiry'])
else:
expiry = DAYS_AFTER
if unblocktime and unblocktime < expiry:
expiry = unblocktime
start = pywikibot.Timestamp.fromISOformat(logevent['timestamp'])