def add_qualifier(self, item, claim, qual, summary=None):
"""
Check if a qualifier is present at the given claim, otherwise add it.
Known issue: This will qualify an already referenced claim
this must therefore be tested for before.
@param item: itemPage to check
@param claim: Claim to check
@param qual: Qualifier to check
@param summary: summary to append to auto-generated edit summary
"""
if not qual:
raise pywikibot.Error(
'Cannot call add_qualifier() without a qualifier.')
# check if already present
if self.has_qualifier(qual, claim):
return False
q_claim = self.make_simple_claim(qual.prop, qual.itis)
try:
claim.addQualifier(q_claim, summary=summary) # writes to database
pywikibot.output('Adding qualifier {0} to {1} in {2}'.format(
qual.prop, claim.getID(), item))
return True
except pywikibot.data.api.APIError as e:
if e.code == 'modification-failed':
pywikibot.output('modification-failed error: '
'qualifier to {0} to {1} in {2}'.format(
qual.prop, claim.getID(), item))
return False
else:
raise pywikibot.Error('Something went very wrong trying to '
'add a qualifier: {}'.format(e))
def main(*args):
"""Run the bot from the command line and handle any arguments."""
# handle arguments
rows = None
add_new = True
cursor = None
cache_max_age = 0
for arg in pywikibot.handle_args(args):
option, sep, value = arg.partition(':')
if option == '-rows':
if helpers.is_pos_int(value):
rows = int(value)
else:
raise pywikibot.Error(usage)
elif option == '-new':
if value.lower() in ('t', 'true'):
add_new = True
elif value.lower() in ('f', 'false'):
add_new = False
else:
raise pywikibot.Error(usage)
elif option == '-cursor':
cursor = value
elif option == '-wdq_cache':
cache_max_age = int(value)
painting_gen = get_painting_generator(rows=rows, cursor=cursor)
paintings_bot = PaintingsBot(painting_gen, INVNO_P, cache_max_age)
paintings_bot.add_new = add_new
paintings_bot.run()
def review(revisionID, comment='', getagain=False):
"""
This function will review the given revision ID with the given comment.
If getagain is set, it will get a new token (not necessary since it will
try to get a new token automatically if the old one is broken)
"""
# print('review will start id: %s - %s' % (revisionID, getagain))
predata = {
'action': 'review',
'format': 'xml',
'revid': str(revisionID),
'token': pywikibot.getSite().getToken(getagain=getagain),
'comment': comment,
}
address = pywikibot.getSite().family.apipath(pywikibot.getSite().lang)
data = api.postForm(pywikibot.getSite(), address, predata)
if data.find('review result=\"Success\"') == -1:
if not data.find('Invalid token') == -1:
if getagain:
raise pywikibot.Error('Invalid Token')
else:
review(revisionID, comment='', getagain=True)
else:
raise pywikibot.Error('Review unsuccessful %s' % data)
def get_single_entry(uuid):
"""Retrieve the data on a single kulturnav entry.
Raises an pywikibot.Error if:
* @graph is not a key in the json response
* a non-json response is received
@param uuid: the uuid for the target item
@type uuid: str
@return: the entry object
@rtype: dict
@raise: pywikibot.Error
"""
query_url = 'http://kulturnav.org/%s?format=application/ld%%2Bjson'
item_url = query_url % uuid
try:
record_page = urllib2.urlopen(item_url)
json_data = json.loads(record_page.read())
except ValueError as e:
raise pywikibot.Error('Error loading KulturNav item at '
'%s with error %s' % (item_url, e))
if json_data.get(u'@graph'):
return json_data
else:
raise pywikibot.Error('No @graph in KulturNav reply at '
'%s\n data: %s' % (item_url, json_data))
def get_single_painting(item):
"""Retrieve the data on a single painting.
Raises an pywikibot.Error if a non-json response is received or if the
query status is not success.
@param item: an item entry from the search results
@type item: dict
@return: the painting object
@rtype: dict
@raise: pywikibot.Error
"""
url = 'http://europeana.eu/api/v2/record/%s.json?wskey=' + \
config.APIKEY + \
'&profile=full'
item_url = url % item.get('id').lstrip('/')
# retrieve and load the data
try:
api_page = urllib2.urlopen(item_url)
json_data = json.loads(api_page.read())
api_page.close()
except ValueError as e:
raise pywikibot.Error('Error loading Europeana item at '
'%s with error %s' % (item_url, e))
# check that it worked
if json_data.get(u'success'):
return json_data
else:
raise pywikibot.Error('Error loading Europeana item at '
'%s with data:\n%s' % (item_url, json_data))
def undo(pagetitle, revisionOld, revisionNew, comment = '', getagain = False):
""""
This function will try to undo the page to the old revision ID
If getagain is set, it will get a new token (not necessary since it will
try to get a new token automatically if the old one is broken)
"""
predata = {
'action' : 'edit',
'format' : 'xml',
'title' : pagetitle,
'undo' : str(revisionOld),
'undoafter' : str(revisionNew),
'token' : pywikibot.getSite().getToken(),
'summary' : comment,
}
address = pywikibot.getSite().family.apipath(pywikibot.getSite().lang)
data = api.postForm(pywikibot.getSite(), address, predata=predata)
dom = minidom.parseString(data.encode('utf8'))
# print('undo data posted, page %s' % pagetitle)
error = dom.getElementsByTagName('error')
if not len(error) ==0:
if not data.find('Invalid token') == -1:
if getagain: raise pywikibot.Error('Invalid Token')
else: undo(pagetitle, revisionOld, revisionNew, comment = '', getagain = True)
else: raise pywikibot.Error('%s : %s' % (error[0].getAttribute('code'), error[0].getAttribute('info')))
edit = dom.getElementsByTagName('edit')
result = edit[0].getAttribute('result')
newrevid = edit[0].getAttribute('newrevid')
return result, newrevid
def validate_sources(self):
"""Validate the sources of a Reference."""
sources = self.get_all_sources()
if not sources:
raise pywikibot.Error(
'You tried to create a reference without any sources')
if not all(isinstance(s, pywikibot.Claim) for s in sources):
raise pywikibot.Error(
'You tried to create a reference with a non-Claim source')
def match_claim(self, claims, qualifiers, force):
"""
Determine which claim is the best match for some given qualifiers.
Selection is done in the following order:
0. no claims: None selected
1. many exact matches: raise error (means duplicates on Wikidata)
2. one exact: select this
3. many has_all: raise error (likely means duplicates on Wikidata)
4. one has_all: select this
5. if many claims: None selected
6. if only one claim: see below
6.1. if claim unsourced and unqualified: select this
6.2. if sourced and unqualified and force: select this
6.3. else: None selected
@param claims: claims to attempt matching against
@type claims: list of pywikibot.Claims
@param qualifiers: qualifiers to match with
@type qualifiers: list of Qualifier
@param force: if sourced (but unqualified) claims should get qualified
@type force: bool
@return: Matching claim or None
@rtype pywikibot.Claim|None
"""
# case 0
if not claims:
return None
# close matches for qualifiers case 1-4
exact_matches = []
has_all_matches = []
for claim in claims:
exact, has_all = self.has_all_qualifiers(qualifiers, claim)
if exact:
exact_matches.append(claim)
if has_all:
has_all_matches.append(claim)
if exact_matches:
if len(exact_matches) > 1:
raise pywikibot.Error("Multiple identical claims")
return exact_matches[0]
elif has_all_matches:
if len(has_all_matches) > 1:
raise pywikibot.Error("Multiple semi-identical claims")
return has_all_matches[0]
# weak matches case 5-6
if len(claims) > 1:
return None
claim = claims[0]
if not claim.sources and not claim.qualifiers:
return claim
elif not claim.qualifiers and force:
return claim
else:
return None
def create_new_painting(self, painting, painting_id, europeana_url, uri):
"""Create a new painting item and return it.
@param painting: information object for the painting
@type painting: dict
@param painting_id: the common (older) id of the painting in the
Nationalmuseum collection
@type painting_id: str
@param europeana_url: reference url for Europeana
@type europeana_url: str
@param uri: reference uri at nationalmuseum.se
@type uri: str
@return: the created painting item
@rtype: pywikibot.ItemPage
"""
data = {'labels': {}, 'descriptions': {}}
data['labels'] = make_labels(painting)
data['descriptions'] = make_descriptions(painting)
if not data['descriptions']:
return
# print data
# create new empty item and request Q-number
summary = u'%s: Creating new item with data from %s' % (EDIT_SUMMARY,
europeana_url)
painting_item = None
try:
painting_item = self.wd.make_new_item(data, summary)
except pywikibot.data.api.APIError as e:
if e.code == u'modification-failed':
# disambiguate and try again
for lang, content in data['descriptions'].iteritems():
disambiguation = content['value'] + u' (%s)' % painting_id
data['descriptions'][lang]['value'] = disambiguation
try:
painting_item = self.wd.make_new_item(data, summary)
except pywikibot.data.api.APIError as e:
if e.code == u'modification-failed':
pywikibot.output(u'modification-failed error: '
u'skipping %s' % uri)
return
else:
raise pywikibot.Error(u'Error during item creation: '
u'%s' % e)
else:
raise pywikibot.Error(u'Error during item creation: %s' % e)
return painting_item
def query_to_lookup(query, item_label='item', value_label='value',
props=None):
"""
Fetch sparql result and return it as a lookup table for wikidata id.
If props are not provided the returned dict simply consists of
value_label:item_label pairs. If props are provided the returned dict
becomes value_label:{'wd':item_label, other props}
:param item_label: the label of the selected wikidata id
:param value_label: the label of the selected lookup key
:param props: dict of other properties to save from the results using
the format label_in_sparql:key_in_output.
:return: dict
"""
wdqs = sparql.SparqlQuery()
result = wdqs.select(query, full_data=True)
lookup = {}
for entry in result:
if entry[value_label] in lookup:
raise pywikibot.Error('Non-unique value in lookup')
key = str(entry[value_label])
qid = entry[item_label].getID()
if not props:
lookup[key] = qid
else:
lookup[key] = {'wd': qid}
for prop, label in props.items():
if entry[prop] and not entry[prop].type:
entry[prop] = repr(entry[prop])
lookup[key][label] = entry[prop]
return lookup
def run(self):
template_image = i18n.translate(self.site, template_to_the_image)
template_user = i18n.translate(self.site, template_to_the_user)
except_text_translated = i18n.translate(self.site, except_text)
summary = i18n.translate(self.site, comment, fallback=True)
if not all(
[template_image, template_user, except_text_translated, comment]):
raise pywikibot.Error(
u'This script is not localized for %s site.' % self.site)
self.summary = summary
generator = pagegenerators.UnusedFilesGenerator(total=1000,
site=self.site) #500
generator = pagegenerators.PreloadingGenerator(generator)
for image in generator:
tituloImagem = image.title()
if not image.exists():
pywikibot.output(u"File '%s' does not exist (see bug 69133)." %
image.title())
continue
print(tituloImagem)
imagemTexto = image.text
if (except_text_translated not in image.getImagePageHtml()
and u'http://' not in imagemTexto):
if template_image in image.text:
pywikibot.output(u"%s done already" %
image.title(asLink=True))
continue
#self.append_text(image, u"\n\n" + template_image)
categorias = list(image.categories())
if not categorias:
print(tituloImagem + " deletado")
image.delete(u"Imagem não utilizada", False, True)
else:
print("Há categorias!\nPulando " + tituloImagem)
'''uploader = image.getFileVersionHistory().pop(0)['user']
def add_start_end_qualifiers(statement, startVal, endVal):
"""
Add start/end qualifiers to a statement if non-None, or return None.
@param statement: The statement to decorate
@type statement: WD.Statement
@param startVal: An ISO date string for the starting point
@type startVal: basestring or None
@param endVal: An ISO date string for the end point
@type endVal: basestring or None
@return: A statement decorated with start/end qualifiers
@rtype: WD.Statement, or None
"""
if not isinstance(statement, WD.Statement):
raise pywikibot.Error('Non-statement recieved: %s' % statement)
if statement.isNone():
return None
# add qualifiers
quals = []
if startVal:
quals.append(
WD.Qualifier(
P=START_P,
itis=iso_to_WbTime(startVal)))
if endVal:
quals.append(
WD.Qualifier(
P=END_P,
itis=iso_to_WbTime(endVal)))
for q in quals:
statement.addQualifier(q)
return statement
def _decompose_page(self):
"""Split Proofread Page text in header, body and footer.
Raises:
exception Error: the page is not formatted according to ProofreadPage
extension.
"""
if not self.text: # Property force page text loading.
self._create_empty_page()
return
open_queue = list(self.p_open.finditer(self._text))
close_queue = list(self.p_close.finditer(self._text))
len_oq = len(open_queue)
len_cq = len(close_queue)
if (len_oq != len_cq) or (len_oq < 2 or len_cq < 2):
raise pywikibot.Error('ProofreadPage %s: invalid format' %
self.title(asLink=True))
f_open, f_close = open_queue[0], close_queue[0]
self._full_header = FullHeader(
self._text[f_open.end():f_close.start()])
l_open, l_close = open_queue[-1], close_queue[-1]
self._footer = self._text[l_open.end():l_close.start()]
self._body = self._text[f_close.end():l_open.start()]
def add_start_end_qualifiers(statement, start_val, end_val):
"""
Add start/end qualifiers to a statement if non-None, or return None.
@param statement: The statement to decorate
@type statement: Statement
@param start_val: An ISO date string for the starting point
@type start_val: basestring or None
@param end_val: An ISO date string for the end point
@type end_val: basestring or None
@return: A statement decorated with start/end qualifiers
@rtype: Statement, or None
"""
if not isinstance(statement, Statement):
raise pywikibot.Error('Non-statement recieved: {}'.format(statement))
if statement.is_none():
return None
# add qualifiers
quals = []
if start_val:
quals.append(
Qualifier(prop=helpers.START_P,
itis=helpers.iso_to_WbTime(start_val)))
if end_val:
quals.append(
Qualifier(prop=helpers.END_P, itis=helpers.iso_to_WbTime(end_val)))
for q in quals:
statement.add_qualifier(q)
return statement
def treat(self, page):
"""Process one ProofreadPage page.
@param page: page to be treated.
@type page: ProofreadPage
@raises: pywikibot.Error
"""
if not isinstance(page, ProofreadPage):
raise pywikibot.Error('Page {} must be a ProofreadPage object.'
.format(page))
summary = self.getOption('summary')
if page.exists():
old_text = page.text
else:
old_text = ''
if self.getOption('ocr'):
_body = self._get_ocr(page)
if _body is None:
pywikibot.output('No OCR found. Skipping {}'
.format(page.title(as_link=True)))
return
page.body = _body
if (page.exists() and
not (self.getOption('ocr') and self.getOption('force'))):
pywikibot.output('Page {} already exists, not adding!'
.format(page))
else:
self.userPut(page, old_text, page.text, summary=summary,
show_diff=self.getOption('showdiff'))
def run(self):
"""Start the bot."""
template_image = i18n.translate(self.site,
template_to_the_image)
template_user = i18n.translate(self.site,
template_to_the_user)
summary = i18n.translate(self.site, comment, fallback=True)
if not all([template_image, template_user, comment]):
raise pywikibot.Error(u'This script is not localized for %s site.'
% self.site)
self.summary = summary
generator = pagegenerators.UnusedFilesGenerator(site=self.site)
generator = pagegenerators.PreloadingGenerator(generator)
for image in generator:
if not image.exists():
pywikibot.output(u"File '%s' does not exist (see bug 69133)."
% image.title())
continue
# Use fileUrl() and fileIsShared() to confirm it is local media
# rather than a local page with the same name as shared media.
if (image.fileUrl() and not image.fileIsShared() and
u'http://' not in image.text):
if template_image in image.text:
pywikibot.output(u"%s done already"
% image.title(asLink=True))
continue
self.append_text(image, u"\n\n" + template_image)
uploader = image.getFileVersionHistory().pop(0)['user']
user = pywikibot.User(image.site, uploader)
usertalkpage = user.getUserTalkPage()
msg2uploader = template_user % {'title': image.title()}
self.append_text(usertalkpage, msg2uploader)
def treat(self, page):
"""Process one ProofreadPage page.
@param page: page to be treated.
@type page: ProofreadPage
@raises: pywikibot.Error
"""
if not isinstance(page, ProofreadPage):
raise pywikibot.Error('Page %s must be a ProofreadPage object.'
% page)
summary = self.getOption('summary')
if page.exists():
old_text = page.text
else:
old_text = ''
if self.getOption('ocr'):
page.body = page.ocr()
if (page.exists() and
not (self.getOption('ocr') and self.getOption('force'))):
pywikibot.output('Page %s already exists, not adding!' % page)
else:
self.userPut(page, old_text, page.text, summary=summary,
show_diff=self.getOption('showdiff'))
def sanitize_wdqs_result(data):
"""
Strip url component out of wdqs results.
Source: deprecated wdqsLookup.py.
Strip out http://www.wikidata.org/entity/
For dicts it is assumed that it is the key which should be sanitized
@param data: data to sanitize
@type data: str, or list of str or dict
@return: sanitized data
@rtype: list of str
"""
if helpers.is_str(data):
return data.split('/')[-1]
elif isinstance(data, list):
for i, d in enumerate(data):
data[i] = d.split('/')[-1]
return data
if isinstance(data, dict):
new_data = dict()
for k, v in data.items():
new_data[k.split('/')[-1]] = v
return new_data
else:
raise pywikibot.Error('sanitize_wdqs_result() requires a string, dict '
'or a list of strings. Not a %s' % type(data))
def create_new_place_id_item(self, data):
"""
Create a new place_id item with some basic info and return it.
:param data: dict of data for a single object
:return: pywikibot.ItemPage
"""
labels = helpers.convert_language_dict_to_json(self.make_labels(data),
typ='labels')
desc = helpers.convert_language_dict_to_json(
self.make_descriptions(data), typ='descriptions')
id_claim = self.wd.make_simple_claim(self.place_id_p,
data.get('place_id'))
item_data = {
"labels": labels,
"descriptions": desc,
"claims": [
id_claim.toJSON(),
]
}
try:
return self.wd.make_new_item(item_data, EDIT_SUMMARY)
except pywikibot.data.api.APIError as e:
raise pywikibot.Error('Error during item creation: {:s}'.format(e))
def treat(self, page):
"""Process one ProofreadPage page.
@param page: page to be treated.
@type page: ProofreadPage
@raises: pywikibot.Error
"""
print("processing page %s" % page.title())
if not isinstance(page, ProofreadPage):
raise pywikibot.Error('Page %s must be a ProofreadPage object.' %
page)
if page.exists():
print("page exists")
# return
# page.get()
print("page status is %s " % page.status)
page.header = u"{{rh|2|''THE POPULAR SCIENCE MONTHLY.''}}"
page.footer = u'\n{{smallrefs}}'
# page.text = " bot experiment"
# page.save()
#sql = "SELECT transcode from table WHERE page = %s ORDER BY seq" % pagenum
#conn = sqlite3.connect('example.db')
#c = conn.cursor()
#text = ""
#for data in c.execute(sql):
# text = text + data['transcode']
# return text
page.body = """{{c|{{x-larger|A BID FOR FORTUNE.}}}}
def iso_to_wbtime(date):
"""
Convert ISO date string into WbTime object.
Given an ISO date object (1922-09-17Z or 2014-07-11T08:14:46Z)
this returns the equivalent WbTime object
@param item: An ISO date string
@type item: basestring
@return: The converted result
@rtype: pywikibot.WbTime
"""
date = date[:len('YYYY-MM-DD')].split('-')
if len(date) == 3 and all(is_int(x) for x in date):
# 1921-09-17Z or 2014-07-11T08:14:46Z
d = int(date[2])
if d == 0:
d = None
m = int(date[1])
if m == 0:
m = None
return pywikibot.WbTime(year=int(date[0]), month=m, day=d)
elif len(date) == 1 and is_int(date[0][:len('YYYY')]):
# 1921Z
return pywikibot.WbTime(year=int(date[0][:len('YYYY')]))
elif (len(date) == 2
and all(is_int(x) for x in (date[0], date[1][:len('MM')]))):
# 1921-09Z
m = int(date[1][:len('MM')])
if m == 0:
m = None
return pywikibot.WbTime(year=int(date[0]), month=m)
# once here all interpretations have failed
raise pywikibot.Error('An invalid ISO-date string received: ' % date)
def run(self):
template_image = i18n.translate(self.site, template_to_the_image)
template_user = i18n.translate(self.site, template_to_the_user)
except_text_translated = i18n.translate(self.site, except_text)
summary = i18n.translate(self.site, comment, fallback=True)
if not all(
[template_image, template_user, except_text_translated, comment]):
raise pywikibot.Error(
u'This script is not localized for %s site.' % self.site)
self.summary = summary
generator = pagegenerators.UnusedFilesGenerator(site=self.site)
generator = pagegenerators.PreloadingGenerator(generator)
for image in generator:
if not image.exists():
pywikibot.output(u"File '%s' does not exist (see bug 69133)." %
image.title())
continue
if (except_text_translated.encode('utf-8')
not in image.getImagePageHtml()
and u'http://' not in image.text):
if template_image in image.text:
pywikibot.output(u"%s done already" %
image.title(asLink=True))
continue
self.append_text(image, u"\n\n" + template_image)
uploader = image.getFileVersionHistory().pop(0)['user']
user = pywikibot.User(image.site, uploader)
usertalkpage = user.getUserTalkPage()
msg2uploader = template_user % {'title': image.title()}
self.append_text(usertalkpage, msg2uploader)
def add_qualifier(self, qual, force=False):
"""
Add qualifier to the statement if not None or already present.
Returns self to allow chaining.
@param qual: the qualifier to add
@type qual: Qualifier|None
@param force: whether qualifier should be added even to already
sourced items
@type force: bool
@rtype Statement
"""
# test input
if qual is None:
# simply skip any action
return self
elif not isinstance(qual, Qualifier):
raise pywikibot.Error(
'add_qualifier was called with something other '
'than a Qualifier|None object: {}'.format(qual))
# register qualifier
self._quals.add(qual)
if force:
self.force = True
return self
def addReference(self, item, claim, ref, summary=None):
"""
Add a reference if not already present.
If a source contains the claims in WD.Reference AND additional claims,
it will not be sourced.
@param item: the item on which all of this happens
@param claim: the pywikibot.Claim to be sourced
@param ref: the WD.Reference to add
@param summary: summary to append to auto-generated edit summary
"""
# check if any of the sources are already present
# note that this can be in any of its references
if ref is None:
return False
if any(self.hasRef(source.getID(), source.getTarget(), claim)
for source in ref.source_test):
return False
try:
# writes to database
claim.addSources(ref.get_all_sources(), summary=summary)
pywikibot.output('Adding reference claim to %s in %s' %
(claim.getID(), item))
return True
except pywikibot.data.api.APIError as e:
if e.code == 'modification-failed':
pywikibot.output('modification-failed error: '
'ref to %s in %s' % (claim.getID(), item))
return False
else:
raise pywikibot.Error(
'Something went very wrong trying to add a source: %s' % e)
def add_new_claim(self, prop, statement, item, ref, summary=None):
"""
Add a claim or source it if already existing.
Known caveats:
* Will source a claim with no qualifiers
* Will not source a claim with only one of the qualifiers
@param prop: property id, with "P" prefix
@type prop: basestring
@param statement: target statement for the claim
@type statement: Statement
@param item: the item being checked
@type item: pywikibot.ItemPage
@param ref: reference to add to the claim, overrides ref embedded in
statement.
@type ref: Reference|None
@param summary: summary to append to auto-generated edit summary
@type summary: basestring|None
"""
claim = pywikibot.Claim(self.repo, prop)
summary = summary or self.edit_summary
# handle special cases
if statement.special:
claim.setSnakType(statement.itis)
prior_claims = self.has_special_claim(prop, statement.itis, item)
else:
claim.setTarget(statement.itis)
prior_claims = self.has_claim(prop, statement.itis, item)
# use ref embedded in statement unless external is explicitly provided
if not ref and statement.ref:
ref = statement.ref
# test reference (must be a Reference or explicitly missing)
if not isinstance(ref, Reference) and ref is not None:
raise pywikibot.Error('The provided reference was not a Reference '
'object. Crashing')
# determine which, if any, prior_claim to source
try:
matching_claim = self.match_claim(prior_claims, statement.quals,
statement.force)
except pywikibot.Error as e:
pywikibot.warning("Problem adding {0} claim to {1}: {2}".format(
prop, item, e))
return
if matching_claim:
for qual in statement.quals:
self.add_qualifier(item, matching_claim, qual, summary=summary)
self.add_reference(item, matching_claim, ref, summary=summary)
else:
item.addClaim(claim, summary=summary)
pywikibot.output('Adding {0} claim to {1}'.format(prop, item))
for qual in statement.quals:
self.add_qualifier(item, claim, qual, summary=summary)
self.add_reference(item, claim, ref, summary=summary)
def purge_links(self, pages=None, page_titles=None, forcelinkupdate=True,
verbose=False):
"""
Trigger a purge (and link update) for the given pages.
These are done in batches to reduce number of api calls.
Timeout settings are temporarily changed to deal with the unusually
long response time for a purge request and the method implements its
own throttling to ensure it respects rate limits.
:param pages: A list of pywkibot.Page to look at
:param page_titles: A list of pages to look at, incl. namespace prefix.
Used only if pages is not provided.
:param forcelinkupdate: if links table should be updated
:param verbose: output to log after every purge
:return: bool (if errors were encountered)
"""
if not pages:
if not page_titles:
raise pywikibot.Error(
'purge_links() requires a list of either pages titles or '
'pywikibot.Page objects.')
pages = [pywikibot.Page(self.site, title) for title in page_titles]
batch_size = 30
rate_limit = 65 # default limit is 30 edits per 60 seconds
max_timeout = 300
original_size = len(pages)
# bump timeout
old_timeout = pwb_config.socket_timeout
pwb_config.socket_timeout = max_timeout
requestparams = {}
if forcelinkupdate:
requestparams['forcelinkupdate'] = True
count = 0
while True:
batch = pages[:batch_size]
pages = pages[batch_size:]
pre_timepoint = time.time()
result = self.site.purgepages(batch, **requestparams)
count += len(batch)
if verbose:
pywikibot.output(u'Purging: %d/%d' % (count, original_size))
if not result:
pywikibot.output(u'Some trouble occurred while purging')
if pages:
# ensure the rate limit is respected
duration = time.time() - pre_timepoint
time.sleep(max(0, (rate_limit - duration)))
else:
break
# reset timeout
pwb_config.socket_timeout = old_timeout
def run(self):
if not self.requester:
raise pywikibot.Error("No requester is set")
else:
self.summary = "[[Commons:Bots|Bot]]: Uploading files from Flickr per request by [[User:%s|%s]]" \
% (self.requester, self.requester)
if not self.uploadByURL:
self.ensureSubDir()
self.parsePages()
def getOption(self, option):
"""
Get the current value of an option.
@param option: key defined in Bot.availableOptions
"""
try:
return self.options.get(option, self.availableOptions[option])
except KeyError:
raise pywikibot.Error(u'%s is not a valid bot option.' % option)
def edit(self, text, jumpIndex=None, highlight=None):
"""
Call the editor and thus allows the user to change the text.
Halts the thread's operation until the editor is closed.
@param text: the text to be edited
@type text: unicode
@param jumpIndex: position at which to put the caret
@type jumpIndex: int
@param highlight: each occurence of this substring will be highlighted
@type highlight: unicode
@return: the modified text, or None if the user didn't save the text
file in his text editor
@rtype: unicode or None
"""
text = self.convertLinebreaks(text)
if config.editor:
tempFilename = '%s.%s' % (tempfile.mktemp(),
config.editor_filename_extension)
with codecs.open(tempFilename,
'w',
encoding=config.editor_encoding) as tempFile:
tempFile.write(text)
creationDate = os.stat(tempFilename).st_mtime
command = self.command(tempFilename, text, jumpIndex)
os.system(command)
lastChangeDate = os.stat(tempFilename).st_mtime
if lastChangeDate == creationDate:
# Nothing changed
return None
else:
with codecs.open(tempFilename,
'r',
encoding=config.editor_encoding) as temp_file:
newcontent = temp_file.read()
os.unlink(tempFilename)
return self.restoreLinebreaks(newcontent)
try:
import gui # noqa
except ImportError as e:
raise pywikibot.Error(
'Could not load GUI modules: %s\nNo editor available.\n'
'Set your favourite editor in user-config.py "editor", '
'or install python packages tkinter and idlelib, which '
'are typically part of Python but may be packaged separately '
'on your platform.\n' % e)
return self.restoreLinebreaks(
pywikibot.ui.editText(text,
jumpIndex=jumpIndex,
highlight=highlight))
def __init__(self, **kwargs):
"""Initializer."""
self.availableOptions.update({
'nouserwarning': False # do not warn uploader
})
super(UnusedFilesBot, self).__init__(**kwargs)
self.template_image = i18n.translate(self.site, template_to_the_image)
self.template_user = i18n.translate(self.site, template_to_the_user)
if not (self.template_image and
(self.template_user or self.getOption('nouserwarning'))):
raise pywikibot.Error(
'This script is not localized for {0} site.'.format(self.site))