def display_references(self):
"""
Display pages which links the current page, sorted per namespace.
Number of pages to display per namespace is provided by:
- self.getOption('isorphan')
"""
refs = self.current_page.ref_table
if refs:
total = sum(len(v) for v in refs.values())
pywikibot.warning('There are %d pages who link to %s.'
% (total, self.current_page))
else:
return
show_n_pages = self.getOption('isorphan')
width = len(max((ns.canonical_prefix() for ns in refs), key=len))
for ns in sorted(refs):
n_pages_in_ns = len(refs[ns])
plural = '' if n_pages_in_ns == 1 else 's'
ns_name = ns.canonical_prefix() if ns != ns.MAIN else 'Main:'
ns_id = '[{0}]'.format(ns.id)
pywikibot.output(
' {0!s:<{width}} {1:>6} {2:>10} page{pl}'.format(
ns_name, ns_id, n_pages_in_ns, width=width, pl=plural))
if show_n_pages: # do not show marker if 0 pages are requested.
for page in islice_with_ellipsis(refs[ns], show_n_pages):
pywikibot.output(' {0!s}'.format(page.title()))
def run(self):
for page in self.generator:
if self.getOption('purge'):
pywikibot.output(u'Page %s%s purged'
% (page.title(asLink=True),
"" if page.purge() else " not"))
continue
try:
# get the page, and save it using the unmodified text.
# whether or not getting a redirect throws an exception
# depends on the variable self.touch_redirects.
page.get(get_redirect=self.getOption('redir'))
page.save("Pywikibot touch script")
except pywikibot.NoPage:
pywikibot.error(u"Page %s does not exist."
% page.title(asLink=True))
except pywikibot.IsRedirectPage:
pywikibot.warning(u"Page %s is a redirect; skipping."
% page.title(asLink=True))
except pywikibot.LockedPage:
pywikibot.error(u"Page %s is locked."
% page.title(asLink=True))
except pywikibot.PageNotSaved:
pywikibot.error(u"Page %s not saved."
% page.title(asLink=True))
def load_values(self, replacements):
"""
Load the dataset specific replacements into self.variables.
Does not accept variables other than those in self.variables.
Automatically handles lat_int and lon_int.
@param replacements: Dictionary with target variable as key and
replacement SQL a value. e.g. {"adm0": "'ad'", "lat": "`lat`"}
"""
required_fields = ('country', 'lang')
if not all(required in replacements for required in required_fields):
raise ValueError(
"All of the required fields '{}' must be replaced".format(
"','".join(required_fields)))
if 'lat' not in replacements:
self.variables['lat_int'] = None
if 'lon' not in replacements:
self.variables['lon_int'] = None
for variable in self.variables:
if variable in replacements:
value = replacements[variable]
if not isinstance(value, VariableType):
raise ValueError(
"All variables must be encoded through VariableType, "
"'{}' was not".format(variable))
self.variables[variable] = value.format()
for target in replacements:
if target not in self.variables:
pywikibot.warning(
"Unrecognized variable in {table}: {variable}".format(
table=self.table, variable=target))
def treat_page(self):
page = self.current_page
text = page.text
done_replacements = []
quickly = self.getOption('quick') is True
start = time.clock()
if self.own_generator:
text = self.currentrule.apply(page.text, done_replacements)
if page.text == text:
if quickly:
pywikibot.output('Typo not found, not fixing another typos '
'in quick mode')
return
else:
self.replaced += 1
for rule in self.typoRules:
if self.own_generator and rule == self.currentrule: # __eq__
continue
if rule.matches(page.title()):
continue
if quickly and rule.needsDecision():
continue
text = rule.apply(text, done_replacements)
stop = time.clock()
if quickly and stop - start > 15:
pywikibot.warning('Other typos exceeded 15s, skipping')
break
self.put_current(
text, summary='oprava překlepů: %s' % ', '.join(done_replacements))
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
local_args = pywikibot.handle_args(args)
start = None
force = False
# Parse command line arguments
for arg in local_args:
if arg.startswith('-start:'):
start = arg[7:]
elif arg == '-force':
force = True
else:
pywikibot.warning(
u'argument "%s" not understood; ignoring.' % arg)
bot = StatesRedirectBot(start, force)
bot.run()
def group_missing_commonscat_by_source(commonscats, withoutCommonscat,
countryconfig):
"""Identify all unused images and group them by source page and id."""
missing_commonscat = {}
for catSortKey in sorted(commonscats.keys()):
try:
monumentId = common.get_id_from_sort_key(
catSortKey, withoutCommonscat)
except ValueError:
pywikibot.warning(u'Got value error for {0}'.format(catSortKey))
continue
if monumentId in withoutCommonscat:
try:
source_link = common.get_source_link(
withoutCommonscat.get(monumentId),
countryconfig.get('type'))
if source_link not in missing_commonscat:
missing_commonscat[source_link] = []
except ValueError:
pywikibot.warning(
u'Could not find source page for {0} ({1})'.format(
monumentId, withoutCommonscat.get(monumentId)))
continue
missing_commonscat[source_link].append(
(commonscats.get(catSortKey), monumentId))
return missing_commonscat
def testApiMethods(self):
"""Test generic ApiSite methods"""
self.assertType(mysite.logged_in(), bool)
self.assertType(mysite.logged_in(True), bool)
self.assertType(mysite.userinfo, dict)
self.assertType(mysite.is_blocked(), bool)
self.assertType(mysite.messages(), bool)
self.assertType(mysite.has_right("edit"), bool)
self.assertFalse(mysite.has_right("nonexistent_right"))
self.assertType(mysite.has_group("bots"), bool)
self.assertFalse(mysite.has_group("nonexistent_group"))
try:
self.assertType(mysite.is_blocked(True), bool)
self.assertType(mysite.has_right("edit", True), bool)
self.assertFalse(mysite.has_right("nonexistent_right", True))
self.assertType(mysite.has_group("bots", True), bool)
self.assertFalse(mysite.has_group("nonexistent_group", True))
except pywikibot.NoUsername:
pywikibot.warning(
"Cannot test Site methods for sysop; no sysop account configured.")
for msg in ("1movedto2", "about", "aboutpage", "aboutsite",
"accesskey-n-portal"):
self.assertTrue(mysite.has_mediawiki_message(msg))
self.assertType(mysite.mediawiki_message(msg), basestring)
self.assertFalse(mysite.has_mediawiki_message("nosuchmessage"))
self.assertRaises(KeyError, mysite.mediawiki_message, "nosuchmessage")
self.assertType(mysite.getcurrenttimestamp(), basestring)
self.assertType(mysite.siteinfo, dict)
self.assertType(mysite.case(), basestring)
ver = mysite.live_version()
self.assertType(ver, tuple)
self.assertTrue(all(isinstance(ver[i], int) for i in (0, 1)))
self.assertType(ver[2], basestring)
def saveToCache(self, q, data):
"""
Save data from a query to a cache file, if enabled.
No return value.
"""
if self.cacheMaxAge <= 0:
return
# we have to use our own query string, as otherwise we may
# be able to find the cache file again if there are e.g.
# whitespace differences
cacheFile = self.getCacheFilename(q)
if os.path.exists(cacheFile) and not os.path.isfile(cacheFile):
return
if not os.path.exists(self.cacheDir):
os.makedirs(self.cacheDir)
with open(cacheFile, 'wb') as f:
try:
pickle.dump(data, f)
except IOError:
pywikibot.warning(u"Failed to write cache file %s" % cacheFile)
def readFromCache(self, queryStr):
"""
Load the query result from the cache, if possible.
Returns None if the data is not there or if it is too old.
"""
if self.cacheMaxAge <= 0:
return None
cacheFile = self.getCacheFilename(queryStr)
if os.path.isfile(cacheFile):
mtime = os.path.getmtime(cacheFile)
now = time.time()
if ((now - mtime) / 60) < self.cacheMaxAge:
with open(cacheFile, 'rb') as f:
try:
data = pickle.load(f)
except pickle.UnpicklingError:
pywikibot.warning(u"Couldn't read cached data from %s"
% cacheFile)
data = None
return data
return None
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
options = {}
for arg in pywikibot.handle_args(args):
if arg == '-replacealways':
options['replace'] = True
options['replacealways'] = True
elif arg == '-hash':
options['use_hash'] = True
elif arg == '-autonomous':
pywikibot.warning(u"The '-autonomous' argument is DEPRECATED,"
u" use '-always' instead.")
options['always'] = True
elif arg.startswith('-'):
if arg[1:] in ('always', 'replace', 'replaceloose', 'replaceonly'):
options[arg[1:]] = True
bot = NowCommonsDeleteBot(**options)
bot.run()
def getOpenStreetMap(latitude, longitude):
"""
Get the result from https://nominatim.openstreetmap.org/reverse .
@rtype: list of tuples
"""
result = []
gotInfo = False
parameters = urlencode({'lat': latitude, 'lon': longitude, 'accept-language': 'en'})
while not gotInfo:
try:
page = fetch('https://nominatim.openstreetmap.org/reverse?format=xml&%s' % parameters)
et = xml.etree.ElementTree.fromstring(page.content)
gotInfo = True
except IOError:
pywikibot.output(u'Got an IOError, let\'s try again')
time.sleep(30)
except socket.timeout:
pywikibot.output(u'Got a timeout, let\'s try again')
time.sleep(30)
validParts = [u'hamlet', u'village', u'city', u'county', u'country']
invalidParts = [u'path', u'road', u'suburb', u'state', u'country_code']
addressparts = et.find('addressparts')
for addresspart in addressparts.getchildren():
if addresspart.tag in validParts:
result.append(addresspart.text)
elif addresspart.tag in invalidParts:
pywikibot.output(u'Dropping %s, %s' % (addresspart.tag, addresspart.text))
else:
pywikibot.warning('%s, %s is not in addressparts lists'
% (addresspart.tag, addresspart.text))
return result
def site_rc_listener(site, total=None):
"""Yield changes received from EventStream.
@param site: the Pywikibot.Site object to yield live recent changes for
@type site: Pywikibot.BaseSite
@param total: the maximum number of changes to return
@type total: int
@return: pywikibot.comms.eventstream.rc_listener configured for given site
"""
if isinstance(EventSource, Exception):
warning('sseclient is required for EventStreams;\n'
'install it with "pip install sseclient"\n')
# fallback to old rcstream method
# NOTE: this will be deprecated soon
from pywikibot.comms.rcstream import rc_listener
return rc_listener(
wikihost=site.hostname(),
rchost=site.rcstream_host(),
rcport=site.rcstream_port(),
rcpath=site.rcstream_path(),
total=total,
)
stream = EventStreams(stream='recentchange', site=site)
stream.set_maximum_items(total)
stream.register_filter(server_name=site.hostname())
return stream
def setLinkDead(self, url, error, page, day):
"""Add the fact that the link was found dead to the .dat file."""
self.semaphore.acquire()
now = time.time()
if url in self.historyDict:
timeSinceFirstFound = now - self.historyDict[url][0][1]
timeSinceLastFound = now - self.historyDict[url][-1][1]
# if the last time we found this dead link is less than an hour
# ago, we won't save it in the history this time.
if timeSinceLastFound > 60 * 60:
self.historyDict[url].append((page.title(), now, error))
# if the first time we found this link longer than x day ago
# (default is a week), it should probably be fixed or removed.
# We'll list it in a file so that it can be removed manually.
if timeSinceFirstFound > 60 * 60 * 24 * day:
# search for archived page
try:
archiveURL = get_archive_url(url)
except Exception as e:
pywikibot.warning(
'get_closest_memento_url({0}) failed: {1}'.format(
url, e))
archiveURL = None
if archiveURL is None:
archiveURL = weblib.getInternetArchiveURL(url)
if archiveURL is None:
archiveURL = weblib.getWebCitationURL(url)
self.log(url, error, page, archiveURL)
else:
self.historyDict[url] = [(page.title(), now, error)]
self.semaphore.release()
def read_file_content(self, file_url=None):
"""Return name of temp file in which remote file is saved."""
if not file_url:
file_url = self.url
pywikibot.warning("file_url is not given. "
"Set to self.url by default.")
pywikibot.output(u'Reading file %s' % file_url)
resume = False
rlen = 0
_contents = None
dt = 15
uo = URLopener()
retrieved = False
while not retrieved:
if resume:
pywikibot.output(u"Resume download...")
uo.addheader('Range', 'bytes=%s-' % rlen)
infile = uo.open(file_url)
if 'text/html' in infile.info().getheader('Content-Type'):
pywikibot.output(u"Couldn't download the image: "
"the requested URL was not found on server.")
return
content_len = infile.info().getheader('Content-Length')
accept_ranges = infile.info().getheader('Accept-Ranges') == 'bytes'
if resume:
_contents += infile.read()
else:
_contents = infile.read()
infile.close()
retrieved = True
if content_len:
rlen = len(_contents)
content_len = int(content_len)
if rlen < content_len:
retrieved = False
pywikibot.output(
u"Connection closed at byte %s (%s left)"
% (rlen, content_len))
if accept_ranges and rlen > 0:
resume = True
pywikibot.output(u"Sleeping for %d seconds..." % dt)
time.sleep(dt)
if dt <= 60:
dt += 15
elif dt < 360:
dt += 60
else:
pywikibot.log(
u"WARNING: length check of retrieved data not possible.")
handle, tempname = tempfile.mkstemp()
with os.fdopen(handle, "wb") as t:
t.write(_contents)
return tempname
def modeletodic(modele):
""" Transforme un chaine "modèle" en tableau
convention : r[0] est le nom du modèle
"""
r = {}
m = RE_Modele.search(modele)
if m:
chaine = m.group(1).replace('\n', '')
chaine = RE_Comment.sub(r'', chaine)
chaine = RE_Pipe.sub(r'[[\2]]', chaine) #pour le split après
pos = 0
for l in chaine.split("|"):
if pos == 0:
r[0] = l.strip()
pos += 1
else:
a = l.split("=")
b = a[0].strip()
if len(a) > 1:
r[b] = a[1].strip()
else:
r[pos] = b
pos +=1
else:
pywikibot.warning("BeBot.modeletodic() ; il a été impossible de lire le modèle suivant\n%s" % modele)
return r
def readFromCache(self, queryStr):
"""
Check if we have cached this data recently enough, read it
if we have. Returns None if the data is not there or if it is
too old
"""
if self.cacheMaxAge <= 0:
return None
cacheFile = self.getCacheFilename(queryStr)
if os.path.isfile(cacheFile):
mtime = os.path.getmtime(cacheFile)
now = time.time()
if ((now - mtime) / 60) < self.cacheMaxAge:
try:
data = pickle.load(open(cacheFile, 'r'))
except pickle.UnpicklingError:
pywikibot.warning(u"Couldn't read cached data from %s"
% cacheFile)
data = None
return data
return None
def loadTypos(self):
pywikibot.output('Loading typo rules')
self.typoRules = []
if self.typos_page_name is None:
self.typos_page_name = 'Wikipedie:WPCleaner/Typo'
typos_page = pywikibot.Page(self.site, self.typos_page_name)
if not typos_page.exists():
# todo: feedback
return
content = typos_page.get()
load_all = self.load_all is True
for template, fielddict in textlib.extract_templates_and_params(
content, remove_disabled_parts=False, strip=False):
if template.lower() == 'typo':
try:
rule = TypoRule.newFromParameters(fielddict, self.site)
except IncompleteTypoRuleException as exc:
pywikibot.warning(exc.message) # pwb.exception?
except InvalidExpressionException as exc:
if 'fixed-width' not in exc.message:
pywikibot.warning('Invalid %s %s: %s' % (
exc.aspect, fielddict['1'], exc.message))
else:
rule.id = self.top_id
self.top_id += 1
if load_all or not rule.needsDecision():
self.typoRules.append(rule)
pywikibot.output('%s typo rules loaded' % len(self.typoRules))
return self.typoRules
def __init__(self, url, urlEncoding=None, description=u'',
useFilename=None, keepFilename=False,
verifyDescription=True, ignoreWarning=False,
targetSite=None, uploadByUrl=False, aborts=[], chunk_size=0):
"""
Constructor.
@param url: path to url or local file (deprecated), or list of urls or
paths to local files.
@type url: string (deprecated) or list
@param description: Description of file for its page. If multiple files
are uploading the same description is used for every file.
@type description: string
@param useFilename: Specify title of the file's page. If multiple
files are uploading it asks to change the name for second, third,
etc. files, otherwise the last file will overwrite the other.
@type useFilename: string
@param keepFilename: Set to True to keep original names of urls and
files, otherwise it will ask to enter a name for each file.
@type keepFilename: bool
@param verifyDescription: Set to False to not proofread the description.
@type verifyDescription: bool
@param ignoreWarning: Set this to True to upload even if another file
would be overwritten or another mistake would be risked. Set it to
an array of warning codes to selectively ignore specific warnings.
@type ignoreWarning: bool or list
@param targetSite: Set the site to upload to. If target site is not
given it's taken from user-config.py.
@type targetSite: object
@param aborts: List of the warning types to abort upload on. Set to True
to abort on any warning.
@type aborts: bool or list
@param chunk_size: Upload the file in chunks (more overhead, but
restartable) specified in bytes. If no value is specified the file
will be uploaded as whole.
@type chunk_size: integer
@deprecated: Using upload_image() is deprecated, use upload_file() with
file_url param instead
"""
self.url = url
if isinstance(self.url, basestring):
pywikibot.warning("url as string is deprecated. "
"Use an iterable instead.")
self.urlEncoding = urlEncoding
self.description = description
self.useFilename = useFilename
self.keepFilename = keepFilename
self.verifyDescription = verifyDescription
self.ignoreWarning = ignoreWarning
self.aborts = aborts
self.chunk_size = chunk_size
if config.upload_to_commons:
self.targetSite = targetSite or pywikibot.Site('commons',
'commons')
else:
self.targetSite = targetSite or pywikibot.Site()
self.targetSite.login()
self.uploadByUrl = uploadByUrl
def get_sd_template(self):
"""Look for speedy deletion template and return it.
@return: A valid speedy deletion template.
@rtype: str or None
"""
if self.getOption('delete') and not self.site.logged_in(sysop=True):
sd = self.getOption('sdtemplate')
if not sd and i18n.twhas_key(self.site,
'redirect-broken-redirect-template'):
sd = i18n.twtranslate(self.site,
'redirect-broken-redirect-template')
# TODO: Add bot's signature if needed (Bug: T131517)
# check whether template exists for this site
title = None
if sd:
template = extract_templates_and_params_regex_simple(sd)
if template:
title = template[0][0]
page = pywikibot.Page(self.site, title, ns=10)
if page.exists():
return sd
pywikibot.warning(
'No speedy deletion template {0}available.'
''.format('"{0}" '.format(title) if title else ''))
return None
def main():
always = False
for arg in pywikibot.handleArgs():
if arg == '-always':
always = True
mysite = pywikibot.Site()
# If anything needs to be prepared, you can do it here
template_image = i18n.translate(pywikibot.Site(),
template_to_the_image)
template_user = i18n.translate(pywikibot.Site(),
template_to_the_user)
except_text_translated = i18n.translate(pywikibot.Site(), except_text)
if not(template_image and template_user and except_text_translated):
pywikibot.warning(u'This script is not localized for %s site.' % mysite)
return
generator = pagegenerators.UnusedFilesGenerator()
generator = pagegenerators.PreloadingGenerator(generator)
for image in generator:
if (except_text_translated.encode('utf-8')
not in image.getImagePageHtml() and
u'http://' not in image.text):
pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
% image.title())
if template_image in image.text:
pywikibot.output(u"%s done already"
% image.title(asLink=True))
continue
appendtext(image, u"\n\n" + template_image, always)
uploader = image.getFileVersionHistory().pop(0)['user']
user = pywikibot.User(mysite, uploader)
usertalkpage = user.getUserTalkPage()
msg2uploader = template_user % {'title': image.title()}
appendtext(usertalkpage, msg2uploader, always)
def add_mbid_claim_to_item(self, item, mbid):
"""
Adds a claim with pid `pid` with value `mbid` to `item` and call `donefunc`
with `mbid` to signal the completion.
:type pid: str
:type mbid: str
:type item: pywikibot.ItemPage
"""
claim = wp.Claim(const.WIKIDATA_DATASITE, self.property_id)
claim.setTarget(mbid)
wp.debug(u"Adding property {pid}, value {mbid} to {title}".format
(pid=self.property_id, mbid=mbid, title=item.title()),
layer="")
if wp.config.simulate:
wp.output("Simulation, no property has been added")
return
try:
item.addClaim(claim, True)
except wp.UserBlocked as e:
wp.error("I have been blocked")
exit(1)
except wp.Error as e:
wp.warning(e)
return
else:
wp.debug("Adding the source Claim", layer="")
claim.addSources([const.MUSICBRAINZ_CLAIM, const.RETRIEVED_CLAIM], bot=True)
self.donefunc(mbid)
def treat_page(self):
"""Process one page."""
page = self.current_page
text = page.get()
for url in weblinksIn(text):
ignoreUrl = False
for ignoreR in ignorelist:
if ignoreR.match(url):
ignoreUrl = True
if not ignoreUrl:
# Limit the number of threads started at the same time. Each
# thread will check one page, then die.
while threading.activeCount() >= config.max_external_links:
time.sleep(config.retry_wait)
thread = LinkCheckThread(page, url, self.history,
self.HTTPignore, self.day)
# thread dies when program terminates
thread.setDaemon(True)
try:
thread.start()
except threading.ThreadError:
pywikibot.warning(
"Can't start a new thread.\nPlease decrease "
"max_external_links in your user-config.py or use\n"
"'-max_external_links:' option with a smaller value. "
"Default is 50.")
raise
def main(*args):
generator = None
local_args = pywikibot.handle_args(args)
site = pywikibot.Site()
if str(site) != "commons:commons":
pywikibot.warning("The script has not been tested on sites other that "
"commons:commons.")
gen_factory = pagegenerators.GeneratorFactory(site)
for local_arg in local_args:
if gen_factory.handleArg(local_arg):
continue
arg, sep, value = local_arg.partition(':')
if arg in ('-showcats',):
options[arg[1:]] = True
else:
raise ValueError('Unknown argument: ' + local_arg)
generator = gen_factory.getCombinedGenerator(gen=generator)
if not generator:
pywikibot.bot.suggest_help(missing_generator=True)
else:
pregenerator = pagegenerators.PreloadingGenerator(generator)
for i, page in enumerate(pregenerator):
if page.exists():
log = handle_page(page)
pywikibot.output('\n'.join(log))
pywikibot.output("")
def treat_page(self):
commons = pywikibot.Site(code = u'commons', fam = u'commons')
today = datetime.date.today()
# fileTemplate = pywikibot.Page(commons, u'Template:Potd filename')
# captionTemplate = pywikibot.Page(commons, u'Template:Potd description') # (Potd page, POTD description)
filePage = pywikibot.Page(commons, u'Template:Potd/%s' % today.isoformat())
file = get_template_parameter_value(filePage, u'Potd filename', u'1')
# TODO: use languages instead of lang
captionPage = pywikibot.Page(commons, u'Template:Potd/%s (%s)'
% (today.isoformat(), self.current_page.site.lang))
if self.current_page.site.lang != u'en' and not captionPage.exists():
pywikibot.warning(u'%s does not exist' % captionPage.title(asLink=True))
# try en instead
captionPage = pywikibot.Page(commons, u'Template:Potd/%s (en)' % today.isoformat())
caption = get_template_parameter_value(captionPage, u'Potd description', u'1')
# TODO: Complete caption parsing to fix links (if not an interwiki then make it an interwiki to Commons)
caption = re.sub(r"\[\[([^:])", r"[[:\1", caption, flags=re.UNICODE) # Force links to start with ':'
caption = re.sub(r"\[\[(:Category:)", r"[[:c\1", caption, flags=re.UNICODE | re.IGNORECASE) # Make category links interwiki links
# TODO: Use [[d:Q4608595]] to get the local {{Documentation}}
doc = u'Documentation'
if file != u'':
summary = u'Updating Commons picture of the day'
if caption != u'':
summary = summary + u', [[:c:%s|caption attribution]]' % captionPage.title()
else:
summary = summary + u', failed to parse caption'
pywikibot.error(u'Failed to parse parameter 1 from {{Potd description}} on %s'
% captionPage.title(asLink=True))
self.put_current(u'<includeonly>{{#switch:{{{1|}}}|caption=%s|#default=%s}}</includeonly><noinclude>\n{{%s}}</noinclude>'
% (caption, file, doc), summary=summary, minor=False)
else:
pywikibot.error(u'Failed to parse parameter 1 from {{Potd filename}} on %s'
% filePage.title(asLink=True))
def __init__(self, site, name, parameters, aliases=None, subst=False):
self._name = name
if not aliases:
aliases = []
elif not subst:
aliases = list(aliases) + [name]
else:
name = 'subst:' + name
if parameters:
name += '|' + parameters
self.template = '{{' + name + '}}'
self._names = frozenset(aliases)
template_ns = site.namespaces[10]
# TODO: Add redirects to self.names too
if not pywikibot.Page(site, self._name, template_ns.id).exists():
raise ValueError('Orphan template "{0}" does not exist on '
'"{1}".'.format(self._name, site))
for name in self._names:
if not pywikibot.Page(site, name, template_ns.id).exists():
pywikibot.warning('Orphan template alias "{0}" does not exist '
'on "{1}"'.format(name, site))
self.regex = re.compile(
r'\{\{(?:' + ':|'.join(template_ns) + '|)(' +
'|'.join(re.escape(name) for name in self._names) +
r')[\|\}]', re.I)
def check_user_exists(self):
"""
Check that the username exists on the site.
@see: U{https://www.mediawiki.org/wiki/API:Users}
@raises NoUsername: Username doesn't exist in user list.
"""
# convert any Special:BotPassword usernames to main account equivalent
main_username = self.username
if '@' in self.username:
warn(
'When using BotPasswords it is recommended that you store '
'your login credentials in a password_file instead. See '
'{}/BotPasswords for instructions and more information.'
.format(__url__))
main_username = self.username.partition('@')[0]
try:
data = self.site.allusers(start=main_username, total=1)
user = next(iter(data))
except pywikibot.data.api.APIError as e:
if e.code == 'readapidenied':
pywikibot.warning('Could not check user %s exists on %s'
% (main_username, self.site))
return
else:
raise
if user['name'] != main_username:
# Report the same error as server error code NotExists
raise NoUsername("Username '%s' does not exist on %s"
% (main_username, self.site))
def testTkdialog(self):
"""Test Tk dialog."""
try:
box = Tkdialog('foo', 'tests/data/MP_sounds.png', 'MP_sounds.png')
box.show_dialog()
except ImportError as e:
pywikibot.warning(e)
def load(self, page):
"""
Loads the given page, does some changes, and saves it.
"""
try:
# Load the page
text = page.get()
except pywikibot.NoPage:
if self.create:
pywikibot.output(u"Page %s doesn't exist yet; creating."
% (page.title(asLink=True)))
text = ''
else:
pywikibot.output(u"Page %s does not exist; skipping."
% page.title(asLink=True))
except pywikibot.IsRedirectPage as arg:
redirTarget = pywikibot.Page(self.site, arg.args[0])
if self.follow_redirects:
text = redirTarget.get()
else:
pywikibot.warning(u"Page %s is a redirect to %s; skipping."
% (page.title(asLink=True),
redirTarget.title(asLink=True)))
else:
return text
def status(self):
"""Return Proofread Page status."""
try:
return self.site.proofread_levels[self.ql]
except KeyError:
pywikibot.warning('Not valid status set for {0!s}: quality level = {1!s}'.format(self.title(asLink=True), self.ql))
return None
def treat(self, page):
"""
Load the given page, make the required changes, and save it.
@param page: the page to treat
@type page: pywikibot.Page
"""
self.current_page = page
page.get()
wikicode = mwparserfromhell.parse(page.text)
for el in wikicode.ifilter_tags():
if el.tag in self.getOption('tags'):
try:
result = self.process_lines(el.contents)
except PoemError as e:
pywikibot.warning(e)
continue
if result:
lines, numbering, lines_count = result
if lines != el.contents:
scheme = self.detect_numbering(numbering, lines_count)
if scheme:
el.contents = lines
for attr, val in scheme.items():
el.add(attr, val)
if self.getOption('preferred'):
# change the tag name to the preferred form
el.tag = self.getOption('preferred')
else:
pywikibot.warning(u'a reliable line numbering scheme could not be obtained')
newtext = unicode(wikicode)
summary = self.getOption('summary')
if not summary:
summary = i18n.translate(page.site, self.summary, fallback=True)
self.userPut(page, page.text, newtext, comment=summary)
def main(*args):
"""Process command line arguments and perform task."""
# imagerecat.initLists()
photoset = u'' # public (popular photos), full (all photos), user ID number
start_id = u''
end_id = u''
addCategory = u''
autonomous = False
totalPhotos = 0
uploadedPhotos = 0
# Do we mark the images as reviewed right away?
if config.panoramio['review']:
panoramioreview = config.panoramio['review']
else:
panoramioreview = False
# Set the Panoramio reviewer
if config.panoramio['reviewer']:
reviewer = config.panoramio['reviewer']
elif 'commons' in config.sysopnames['commons']:
reviewer = config.sysopnames['commons']['commons']
elif 'commons' in config.usernames['commons']:
reviewer = config.usernames['commons']['commons']
else:
reviewer = u''
# Should be renamed to overrideLicense or something like that
override = u''
local_args = pywikibot.handle_args(args)
for arg in local_args:
if arg.startswith('-set'):
if len(arg) == 4:
photoset = pywikibot.input(u'What is the set?')
else:
photoset = arg[5:]
elif arg.startswith('-start_id'):
if len(arg) == 9:
start_id = pywikibot.input(
u'What is the id of the photo you want to start at?')
else:
start_id = arg[10:]
elif arg.startswith('-end_id'):
if len(arg) == 7:
end_id = pywikibot.input(
u'What is the id of the photo you want to end at?')
else:
end_id = arg[8:]
elif arg == '-panoramioreview':
panoramioreview = True
elif arg.startswith('-reviewer'):
if len(arg) == 9:
reviewer = pywikibot.input(u'Who is the reviewer?')
else:
reviewer = arg[10:]
elif arg.startswith('-override'):
if len(arg) == 9:
override = pywikibot.input(u'What is the override text?')
else:
override = arg[10:]
elif arg.startswith('-addcategory'):
if len(arg) == 12:
addCategory = pywikibot.input(
u'What category do you want to add?')
else:
addCategory = arg[13:]
elif arg == '-autonomous':
autonomous = True
if photoset:
site = pywikibot.Site()
if site != pywikibot.Site('commons', 'commons'):
pywikibot.warning(
'Using {0} instead of Wikimedia Commons'.format(site))
for photoInfo in getPhotos(photoset, start_id, end_id):
photoInfo = getLicense(photoInfo)
# time.sleep(10)
uploadedPhotos += processPhoto(photoInfo, panoramioreview,
reviewer, override, addCategory,
autonomous, site=site)
totalPhotos += 1
else:
usage()
pywikibot.output(u'Finished running')
pywikibot.output(u'Total photos: ' + str(totalPhotos))
pywikibot.output(u'Uploaded photos: ' + str(uploadedPhotos))
def replace_links(self, text, linkedPage, targetPage):
"""Replace all source links by target."""
mysite = pywikibot.Site()
linktrail = mysite.linktrail()
# make a backup of the original text so we can show the changes later
linkR = re.compile(
r'\[\[(?P<title>[^\]\|#]*)(?P<section>#[^\]\|]*)?'
r'(\|(?P<label>[^\]]*))?\]\](?P<linktrail>' + linktrail + ')')
curpos = 0
# This loop will run until we have finished the current page
while True:
m = linkR.search(text, pos=curpos)
if not m:
break
# Make sure that next time around we will not find this same hit.
curpos = m.start() + 1
# T283403
try:
is_interwikilink = mysite.isInterwikiLink(m.group('title'))
except ValueError:
pywikibot.exception()
continue
# ignore interwiki links, links in the disabled area
# and links to sections of the same page
if (m.group('title').strip() == ''
or is_interwikilink
or isDisabled(text, m.start())):
continue
actualLinkPage = pywikibot.Page(targetPage.site, m.group('title'))
# Check whether the link found is to page.
try:
actualLinkPage.title()
except InvalidTitleError:
pywikibot.exception()
continue
if actualLinkPage != linkedPage:
continue
# The link looks like this:
# [[page_title|link_text]]trailing_chars
page_title = m.group('title')
link_text = m.group('label')
if not link_text:
# or like this: [[page_title]]trailing_chars
link_text = page_title
if m.group('section') is None:
section = ''
else:
section = m.group('section')
if section and targetPage.section():
pywikibot.warning(
'Source section {} and target section {} found. '
'Skipping.'.format(section, targetPage))
continue
trailing_chars = m.group('linktrail')
if trailing_chars:
link_text += trailing_chars
# remove preleading ":"
if link_text[0] == ':':
link_text = link_text[1:]
if link_text[0].isupper() or link_text[0].isdigit():
new_page_title = targetPage.title()
else:
new_page_title = first_lower(targetPage.title())
# remove preleading ":"
if new_page_title[0] == ':':
new_page_title = new_page_title[1:]
if ((new_page_title == link_text and not section)
or self.opt.overwrite):
newlink = '[[{}]]'.format(new_page_title)
# check if we can create a link with trailing characters instead of
# a pipelink
elif (len(new_page_title) <= len(link_text)
and (firstcap(link_text[:len(new_page_title)])
== firstcap(new_page_title))
and re.sub(re.compile(linktrail), '',
link_text[len(new_page_title):]) == ''
and not section):
newlink = '[[{}]]{}'.format(link_text[:len(new_page_title)],
link_text[len(new_page_title):])
else:
newlink = '[[{}{}|{}]]'.format(new_page_title,
section, link_text)
text = text[:m.start()] + newlink + text[m.end():]
continue
return text
def fix_1_double_redirect(self):
"""Treat one double redirect."""
newRedir = redir = self.current_page
redirList = [] # bookkeeping to detect loops
while True:
redirList.append('{0}:{1}'
.format(newRedir.site.lang,
newRedir.title(with_section=False)))
try:
targetPage = newRedir.getRedirectTarget()
except pywikibot.IsNotRedirectPage:
if len(redirList) == 2:
pywikibot.output(
'Skipping: Redirect target {0} is not a redirect.'
.format(newRedir.title(as_link=True)))
break # do nothing
except pywikibot.SectionError:
pywikibot.warning(
"Redirect target section {0} doesn't exist."
.format(newRedir.title(as_link=True)))
except (pywikibot.CircularRedirect,
pywikibot.InterwikiRedirectPage,
pywikibot.UnsupportedPage,
RuntimeError):
pywikibot.exception()
pywikibot.output('Skipping {0}.'.format(newRedir))
break
except pywikibot.BadTitle as e:
# str(e) is in the format 'BadTitle: [[Foo]]'
pywikibot.warning(
'Redirect target {0} is not a valid page title.'
.format(str(e)[10:]))
break
except pywikibot.NoPage:
if self.getOption('always'):
pywikibot.output(
"Skipping: Redirect target {} doesn't exist."
.format(newRedir.title(as_link=True)))
break # skip if automatic
else:
pywikibot.warning(
"Redirect target {} doesn't exist."
.format(newRedir.title(as_link=True)))
except pywikibot.ServerError:
pywikibot.output('Skipping due to server error: '
'No textarea found')
break
else:
pywikibot.output(
' Links to: {0}.'
.format(targetPage.title(as_link=True)))
try:
mw_msg = targetPage.site.mediawiki_message(
'wikieditor-toolbar-tool-redirect-example')
except KeyError:
pass
else:
if targetPage.title() == mw_msg:
pywikibot.output(
'Skipping toolbar example: Redirect source is '
'potentially vandalized.')
break
# watch out for redirect loops
if redirList.count('{0}:{1}'.format(
targetPage.site.lang,
targetPage.title(with_section=False))):
pywikibot.warning(
'Redirect target {0} forms a redirect loop.'
.format(targetPage.title(as_link=True)))
break # FIXME: doesn't work. edits twice!
if self.getOption('delete'):
# Delete the two redirects
# TODO: Check whether pages aren't vandalized
# and (maybe) do not have a version history
self.delete_redirect(targetPage,
'redirect-remove-loop')
self.delete_redirect(redir, 'redirect-remove-loop')
break
else: # redirect target found
if targetPage.isStaticRedirect():
pywikibot.output(
' Redirect target is STATICREDIRECT.')
else:
newRedir = targetPage
continue
try:
oldText = redir.get(get_redirect=True)
except pywikibot.BadTitle:
pywikibot.output('Bad Title Error')
break
if self.is_repo and redir.namespace() == self.repo.item_namespace:
redir = pywikibot.ItemPage(self.repo, redir.title())
targetPage = pywikibot.ItemPage(self.repo, targetPage.title())
pywikibot.output('Fixing double item redirect')
redir.set_redirect_target(targetPage)
break
redir.set_redirect_target(targetPage, keep_section=True,
save=False)
summary = i18n.twtranslate(self.site, 'redirect-fix-double',
{'to': targetPage.title(as_link=True)}
)
self.userPut(redir, oldText, redir.text, summary=summary,
ignore_save_related_errors=True,
ignore_server_errors=True)
break
def main(*args):
"""Process command line arguments and invoke PatrolBot."""
# This factory is responsible for processing command line arguments
# that are also used by other scripts and that determine on which pages
# to work on.
usercontribs = None
gen = None
recentchanges = False
newpages = False
repeat = False
gen_factory = pagegenerators.GeneratorFactory()
options = {}
# Parse command line arguments
for arg in pywikibot.handle_args(args):
if arg.startswith('-ask'):
options['ask'] = True
elif arg.startswith('-autopatroluserns'):
options['autopatroluserns'] = True
elif arg.startswith('-repeat'):
repeat = True
elif arg.startswith('-newpages'):
newpages = True
elif arg.startswith('-recentchanges'):
recentchanges = True
elif arg.startswith('-usercontribs:'):
usercontribs = arg[14:]
elif arg.startswith('-versionchecktime:'):
versionchecktime = arg[len('-versionchecktime:'):]
options['versionchecktime'] = int(versionchecktime)
elif arg.startswith("-whitelist:"):
options['whitelist'] = arg[len('-whitelist:'):]
else:
generator = gen_factory.handleArg(arg)
if not generator:
if ':' in arg:
m = arg.split(':')
options[m[0]] = m[1]
site = pywikibot.Site()
site.login()
if usercontribs:
user = pywikibot.User(site, usercontribs)
if user.isAnonymous() or user.isRegistered():
pywikibot.output('Processing user: {}'.format(usercontribs))
else:
pywikibot.warning('User {} does not exist on site {}.'.format(
usercontribs, site))
# default behaviour
if not any((newpages, recentchanges, usercontribs)):
if site.family.name == 'wikipedia':
newpages = True
else:
recentchanges = True
bot = PatrolBot(**options)
if newpages or usercontribs:
pywikibot.output(u'Newpages:')
gen = site.newpages
feed = api_feed_repeater(gen,
delay=60,
repeat=repeat,
user=usercontribs,
namespaces=gen_factory.namespaces,
recent_new_gen=False)
bot.run(feed)
if recentchanges or usercontribs:
pywikibot.output(u'Recentchanges:')
gen = site.recentchanges
feed = api_feed_repeater(gen,
delay=60,
repeat=repeat,
namespaces=gen_factory.namespaces,
user=usercontribs)
bot.run(feed)
pywikibot.output(u'%d/%d patrolled' %
(bot.patrol_counter, bot.rc_item_counter))
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
gen = None
notitle = False
fmt = '1'
outputlang = None
page_get = False
base_dir = None
encoding = config.textfile_encoding
# Process global args and prepare generator args parser
local_args = pywikibot.handle_args(args)
genFactory = GeneratorFactory()
for arg in local_args:
if arg == '-notitle':
notitle = True
elif arg.startswith('-format:'):
fmt = arg[len('-format:'):]
fmt = fmt.replace(u'\\03{{', u'\03{{')
elif arg.startswith('-outputlang:'):
outputlang = arg[len('-outputlang:'):]
elif arg == '-get':
page_get = True
elif arg.startswith('-save'):
base_dir = arg.partition(':')[2] or '.'
elif arg.startswith('-encode:'):
encoding = arg.partition(':')[2]
else:
genFactory.handleArg(arg)
if base_dir:
base_dir = os.path.expanduser(base_dir)
if not os.path.isabs(base_dir):
base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir))
if not os.path.exists(base_dir):
pywikibot.output(u'Directory "%s" does not exist.' % base_dir)
choice = pywikibot.input_yn(
u'Do you want to create it ("No" to continue without saving)?')
if choice:
os.makedirs(base_dir, mode=0o744)
else:
base_dir = None
elif not os.path.isdir(base_dir):
# base_dir is a file.
pywikibot.warning(u'Not a directory: "%s"\n'
u'Skipping saving ...'
% base_dir)
base_dir = None
gen = genFactory.getCombinedGenerator()
if gen:
i = 0
for i, page in enumerate(gen, start=1):
if not notitle:
page_fmt = Formatter(page, outputlang)
pywikibot.stdout(page_fmt.output(num=i, fmt=fmt))
if page_get:
try:
pywikibot.output(page.text, toStdout=True)
except pywikibot.Error as err:
pywikibot.output(err)
if base_dir:
filename = os.path.join(base_dir, page.title(as_filename=True))
pywikibot.output(u'Saving %s to %s' % (page.title(), filename))
with open(filename, mode='wb') as f:
f.write(page.text.encode(encoding))
pywikibot.output(u"%i page(s) found" % i)
else:
pywikibot.showHelp()
def main(*args) -> None:
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: str
"""
password = None
logall = False
logout = False
oauth = False
autocreate = False
unknown_args = []
for arg in pywikibot.handle_args(args):
if arg.startswith('-pass'):
if len(arg) == 5:
password = pywikibot.input(
'Password for all accounts (no characters will be shown):',
password=True)
else:
password = arg[6:]
elif arg == '-all':
logall = True
elif arg == '-force':
pywikibot.output('To force a re-login, please delete the '
"revelant lines from '{0}' (or the entire file) "
'and try again.'.format(
join(config.base_dir, 'pywikibot.lwp')))
elif arg == '-logout':
logout = True
elif arg == '-oauth':
oauth = True
elif arg == '-autocreate':
autocreate = True
else:
unknown_args += [arg]
if pywikibot.bot.suggest_help(unknown_parameters=unknown_args):
return
if password is not None:
pywikibot.warning('The -pass argument is not implemented yet. See: '
'https://phabricator.wikimedia.org/T102477')
if logall:
namedict = config.usernames
else:
site = pywikibot.Site()
namedict = {site.family.name: {site.code: None}}
for family_name in namedict:
for lang in namedict[family_name]:
try:
site = pywikibot.Site(code=lang, fam=family_name)
if oauth:
_oauth_login(site)
continue
if logout:
site.logout()
else:
site.login(autocreate=autocreate)
user = site.user()
if user:
pywikibot.output('Logged in on {0} as {1}.'.format(
site, user))
else:
if logout:
pywikibot.output('Logged out of {0}.'.format(site))
else:
pywikibot.output('Not logged in on {0}.'.format(site))
except SiteDefinitionError:
pywikibot.output('{0}.{1} is not a valid site, '
'please remove it from your config'.format(
lang, family_name))
def __init__(self, generator, templates, **kwargs):
"""
Initializer.
@param generator: the pages to work on
@type generator: iterable
@param templates: a dictionary which maps old template names to
their replacements. If remove or subst is True, it maps the
names of the templates that should be removed/resolved to None.
@type templates: dict
"""
self.availableOptions.update({
'subst': False,
'remove': False,
'summary': None,
'addedCat': None,
})
Bot.__init__(self, generator=generator, **kwargs)
self.templates = templates
# get edit summary message if it's empty
if not self.getOption('summary'):
comma = self.site.mediawiki_message('comma-separator')
params = {'list': comma.join(self.templates.keys()),
'num': len(self.templates)}
if self.getOption('remove'):
self.options['summary'] = i18n.twtranslate(
self.site, 'template-removing', params)
elif self.getOption('subst'):
self.options['summary'] = i18n.twtranslate(
self.site, 'template-substituting', params)
else:
self.options['summary'] = i18n.twtranslate(
self.site, 'template-changing', params)
replacements = []
exceptions = {}
builder = textlib._MultiTemplateMatchBuilder(self.site)
for old, new in self.templates.items():
template_regex = builder.pattern(old)
if self.getOption('subst') and self.getOption('remove'):
replacements.append((template_regex,
r'{{subst:%s\g<parameters>}}' % new))
exceptions['inside-tags'] = ['ref', 'gallery', 'poem',
'pagelist', ]
elif self.getOption('subst'):
replacements.append((template_regex,
r'{{subst:%s\g<parameters>}}' % old))
exceptions['inside-tags'] = ['ref', 'gallery', 'poem',
'pagelist', ]
elif self.getOption('remove'):
separate_line_regex = re.compile(
r'^[*#:]* *{0} *\n'.format(template_regex.pattern),
re.DOTALL | re.MULTILINE)
replacements.append((separate_line_regex, ''))
spaced_regex = re.compile(
r' +{0} +'.format(template_regex.pattern),
re.DOTALL)
replacements.append((spaced_regex, ' '))
replacements.append((template_regex, ''))
else:
template = pywikibot.Page(self.site, new, ns=10)
if not template.exists():
pywikibot.warning('Template "{0}" does not exist.'
.format(new))
if not pywikibot.input_yn('Do you want to proceed anyway?',
default=False,
automatic_quit=False):
continue
replacements.append((template_regex,
r'{{%s\g<parameters>}}' % new))
super(TemplateRobot, self).__init__(
generator, replacements, exceptions,
always=self.getOption('always'),
addedCat=self.getOption('addedCat'),
summary=self.getOption('summary'))
def handle_args(args):
"""Process command line arguments.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: str
"""
mapping = {
# option: (attribute, value),
'-break': ('recursive', False),
'-nlog': ('makeWelcomeLog', False),
'-ask': ('confirm', True),
'-filter': ('filtBadName', True),
'-savedata': ('saveSignIndex', True),
'-random': ('randomSign', True),
'-sul': ('welcomeAuto', True),
'-quiet': ('quiet', True),
}
for arg in pywikibot.handle_args(args):
arg, _, val = arg.partition(':')
if arg == '-edit':
globalvar.attachEditCount = int(
val if val.isdigit() else pywikibot.input(
'After how many edits would you like to welcome new users?'
' (0 is allowed)'))
elif arg == '-timeoffset':
globalvar.timeoffset = int(
val if val.isdigit() else pywikibot.
input('Which time offset (in minutes) for new users would you '
'like to use?'))
elif arg == '-time':
globalvar.timeRecur = int(
val if val.isdigit() else pywikibot.
input('For how many seconds would you like to bot to sleep '
'before checking again?'))
elif arg == '-offset':
_handle_offset(val)
elif arg == '-file':
globalvar.randomSign = True
globalvar.signFileName = val or pywikibot.input(
'Where have you saved your signatures?')
elif arg == '-sign':
globalvar.defaultSign = val or pywikibot.input(
'Which signature to use?')
globalvar.defaultSign += timeselected
elif arg == '-limit':
globalvar.queryLimit = int(
val if val.isdigit() else pywikibot.
input('How many of the latest new users would you like to '
'load?'))
elif arg == '-numberlog':
globalvar.dumpToLog = int(
val if val.isdigit() else pywikibot.
input('After how many welcomed users would you like to update '
'the welcome log?'))
elif arg in mapping:
setattr(globalvar, *mapping[arg])
else:
pywikibot.warning('Unknown option "{}"'.format(arg))
def run(self):
"""Run the bot."""
for page in self.generator:
self.current_page = page
try:
text = page.text
except pywikibot.NoPage:
pywikibot.warning('Page %s does not exist?!' %
page.title(asLink=True))
continue
except pywikibot.IsRedirectPage:
pywikibot.output(u"Page %s is a redirect; skipping." %
page.title(asLink=True))
continue
except pywikibot.LockedPage:
pywikibot.warning('Page %s is locked?!' %
page.title(asLink=True))
continue
if page.isDisambig():
pywikibot.output(u"Page %s is a disambig; skipping." %
page.title(asLink=True))
continue
if self.site.sitename == 'wikipedia:en' and page.isIpEdit():
pywikibot.warning(
u"Page %s is edited by IP. Possible vandalized" %
page.title(asLink=True))
continue
if self.lacksReferences(text):
newText = self.addReferences(text)
try:
self.userPut(page,
page.text,
newText,
summary=self.comment)
except pywikibot.EditConflict:
pywikibot.warning('Skipping %s because of edit conflict' %
page.title(asLink=True))
except pywikibot.SpamfilterError as e:
pywikibot.warning(
u'Cannot change %s because of blacklist entry %s' %
(page.title(asLink=True), e.url))
except pywikibot.LockedPage:
pywikibot.warning('Skipping %s (locked page)' %
page.title(asLink=True))
def test_warning(self):
pywikibot.warning('warning')
self.assertEqual(newstdout.getvalue(), '')
self.assertEqual(newstderr.getvalue(), 'WARNING: warning\n')
def process_filename(self, file_url=None):
"""Return base filename portion of file_url."""
if not file_url:
file_url = self.url
pywikibot.warning("file_url is not given. "
"Set to self.url by default.")
always = self.getOption('always')
# Isolate the pure name
filename = file_url
# Filename may be either a URL or a local file path
if "://" in filename:
# extract the path portion of the URL
filename = urlparse(filename).path
filename = os.path.basename(filename)
if self.useFilename:
filename = self.useFilename
if not self.keepFilename:
pywikibot.output(
u"The filename on the target wiki will default to: %s" %
filename)
assert not always
newfn = pywikibot.input(
u'Enter a better name, or press enter to accept:')
if newfn != "":
filename = newfn
# FIXME: these 2 belong somewhere else, presumably in family
# forbidden characters are handled by pywikibot/page.py
forbidden = ':*?/\\' # to be extended
try:
allowed_formats = self.targetSite.siteinfo.get('fileextensions',
get_default=False)
except KeyError:
allowed_formats = []
else:
allowed_formats = [item['ext'] for item in allowed_formats]
# ask until it's valid
first_check = True
while True:
if not first_check:
if always:
filename = None
else:
filename = pywikibot.input('Enter a better name, or press '
'enter to skip the file:')
if not filename:
return None
first_check = False
ext = os.path.splitext(filename)[1].lower().strip('.')
# are any chars in forbidden also in filename?
invalid = set(forbidden) & set(filename)
if invalid:
c = "".join(invalid)
pywikibot.output('Invalid character(s): %s. Please try again' %
c)
continue
if allowed_formats and ext not in allowed_formats:
if always:
pywikibot.output('File format is not one of '
'[{0}]'.format(' '.join(allowed_formats)))
continue
elif not pywikibot.input_yn(
u"File format is not one of [%s], but %s. Continue?" %
(u' '.join(allowed_formats), ext),
default=False,
automatic_quit=False):
continue
potential_file_page = pywikibot.FilePage(self.targetSite, filename)
if potential_file_page.exists():
overwrite = self._handle_warning('exists')
if overwrite is False:
pywikibot.output(
"File exists and you asked to abort. Skipping.")
return None
if potential_file_page.canBeEdited():
if overwrite is None:
overwrite = not pywikibot.input_yn(
"File with name %s already exists. "
"Would you like to change the name? "
"(Otherwise file will be overwritten.)" % filename,
default=True,
automatic_quit=False)
if not overwrite:
continue
else:
break
else:
pywikibot.output(u"File with name %s already exists and "
"cannot be overwritten." % filename)
continue
else:
try:
if potential_file_page.fileIsShared():
pywikibot.output(
u"File with name %s already exists in shared "
"repository and cannot be overwritten." % filename)
continue
else:
break
except pywikibot.NoPage:
break
# A proper description for the submission.
# Empty descriptions are not accepted.
pywikibot.output(u'The suggested description is:\n%s' %
self.description)
# Description must be set and verified
if not self.description:
self.verifyDescription = True
while not self.description or self.verifyDescription:
if not self.description:
pywikibot.output(
u'\03{lightred}It is not possible to upload a file '
'without a summary/description.\03{default}')
assert not always
# if no description, default is 'yes'
if pywikibot.input_yn(u'Do you want to change this description?',
default=not self.description):
from pywikibot import editor as editarticle
editor = editarticle.TextEditor()
try:
newDescription = editor.edit(self.description)
except Exception as e:
pywikibot.error(e)
continue
# if user saved / didn't press Cancel
if newDescription:
self.description = newDescription
self.verifyDescription = False
return filename
def __init__(self,
url,
urlEncoding=None,
description=u'',
useFilename=None,
keepFilename=False,
verifyDescription=True,
ignoreWarning=False,
targetSite=None,
uploadByUrl=False,
aborts=[],
chunk_size=0,
**kwargs):
"""
Constructor.
@param url: path to url or local file (deprecated), or list of urls or
paths to local files.
@type url: string (deprecated) or list
@param description: Description of file for its page. If multiple files
are uploading the same description is used for every file.
@type description: string
@param useFilename: Specify title of the file's page. If multiple
files are uploading it asks to change the name for second, third,
etc. files, otherwise the last file will overwrite the other.
@type useFilename: string
@param keepFilename: Set to True to keep original names of urls and
files, otherwise it will ask to enter a name for each file.
@type keepFilename: bool
@param verifyDescription: Set to False to not proofread the description.
@type verifyDescription: bool
@param ignoreWarning: Set this to True to upload even if another file
would be overwritten or another mistake would be risked. Set it to
an array of warning codes to selectively ignore specific warnings.
@type ignoreWarning: bool or list
@param targetSite: Set the site to upload to. If target site is not
given it's taken from user-config.py.
@type targetSite: object
@param aborts: List of the warning types to abort upload on. Set to True
to abort on any warning.
@type aborts: bool or list
@param chunk_size: Upload the file in chunks (more overhead, but
restartable) specified in bytes. If no value is specified the file
will be uploaded as whole.
@type chunk_size: integer
@param always: Disables any input, requires that either ignoreWarning or
aborts are set to True and that the description is also set. It will
overwrite verifyDescription to False and keepFilename to True.
@type always: bool
@deprecated: Using upload_image() is deprecated, use upload_file() with
file_url param instead
"""
super(UploadRobot, self).__init__(**kwargs)
always = self.getOption('always')
if (always and ignoreWarning is not True and aborts is not True):
raise ValueError(
'When always is set to True, either ignoreWarning '
'or aborts must be set to True.')
if always and not description:
raise ValueError('When always is set to True the description must '
'be set.')
self.url = url
if isinstance(self.url, basestring):
pywikibot.warning("url as string is deprecated. "
"Use an iterable instead.")
self.urlEncoding = urlEncoding
self.description = description
self.useFilename = useFilename
self.keepFilename = keepFilename or always
self.verifyDescription = verifyDescription and not always
self.ignoreWarning = ignoreWarning
self.aborts = aborts
self.chunk_size = chunk_size
if config.upload_to_commons:
self.targetSite = targetSite or pywikibot.Site(
'commons', 'commons')
else:
self.targetSite = targetSite or pywikibot.Site()
self.targetSite.login()
self.uploadByUrl = uploadByUrl
def delete_1_broken_redirect(self, redir_name):
redir_page = pywikibot.Page(self.site, redir_name)
# Show the title of the page we're working on.
# Highlight the title in purple.
pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
% redir_page.title())
try:
targetPage = redir_page.getRedirectTarget()
except pywikibot.IsNotRedirectPage:
pywikibot.output(u'%s is not a redirect.' % redir_page.title())
except pywikibot.NoPage:
pywikibot.output(u'%s doesn\'t exist.' % redir_page.title())
else:
try:
targetPage.get()
except pywikibot.BadTitle as e:
pywikibot.warning(
u'Redirect target %s is not a valid page title.'
% str(e)[10:])
pass
except pywikibot.NoPage:
movedTarget = self.moved_page(targetPage)
if movedTarget:
if not movedTarget.exists():
# FIXME: Test to another move
pywikibot.output(u'Target page %s does not exist'
% (movedTarget))
elif redir_name == movedTarget.title():
pywikibot.output(u'Target page forms a redirect loop')
else:
pywikibot.output(u'%s has been moved to %s'
% (redir_page, movedTarget))
reason = i18n.twtranslate(self.site,
'redirect-fix-broken-moved',
{'to': movedTarget.title(
asLink=True)})
content = redir_page.get(get_redirect=True)
text = self.site.redirectRegex().sub(
'#%s %s' % (self.site.redirect(),
movedTarget.title(asLink=True,
textlink=True)),
content)
pywikibot.showDiff(content, text)
pywikibot.output(u'Summary - %s' % reason)
if self.user_confirm(
u'Redirect target %s has been moved to %s.\n'
u'Do you want to fix %s?'
% (targetPage, movedTarget, redir_page)):
try:
redir_page.text = text
redir_page.save(reason)
except pywikibot.NoUsername:
pywikibot.output(u"Page [[%s]] not saved; "
u"sysop privileges required."
% redir_page.title())
pass
except pywikibot.LockedPage:
pywikibot.output(u'%s is locked.'
% redir_page.title())
pass
elif self.getOption('delete') and self.user_confirm(
u'Redirect target %s does not exist.\n'
u'Do you want to delete %s?'
% (targetPage.title(asLink=True),
redir_page.title(asLink=True))):
reason = i18n.twtranslate(self.site,
'redirect-remove-broken')
if self.site.logged_in(sysop=True):
redir_page.delete(reason, prompt=False)
else:
assert targetPage.site == self.site, (
u'target page is on different site %s'
% targetPage.site)
if (self.has_valid_template(
'redirect-broken-redirect-template') and
i18n.twhas_key(targetPage.site,
'redirect-remove-broken')):
pywikibot.output(u"No sysop in user-config.py, "
u"put page to speedy deletion.")
content = redir_page.get(get_redirect=True)
# TODO: Add bot's signature if needed
# Not supported via TW yet
content = i18n.twtranslate(
targetPage.site,
'redirect-broken-redirect-template'
) + "\n" + content
redir_page.put(content, reason)
else:
pywikibot.output(
u'No speedy deletion template available')
except pywikibot.IsRedirectPage:
pywikibot.output(u"Redirect target %s is also a redirect! "
u"Won't delete anything."
% targetPage.title(asLink=True))
else:
# we successfully get the target page, meaning that
# it exists and is not a redirect: no reason to touch it.
pywikibot.output(
u'Redirect target %s does exist! Won\'t delete anything.'
% targetPage.title(asLink=True))
pywikibot.output(u'')
def fix_1_double_redirect(self, redir_name):
redir = pywikibot.Page(self.site, redir_name)
# Show the title of the page we're working on.
# Highlight the title in purple.
pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
% redir.title())
newRedir = redir
redirList = [] # bookkeeping to detect loops
while True:
redirList.append(u'%s:%s' % (newRedir.site.lang,
newRedir.title(withSection=False)))
try:
targetPage = newRedir.getRedirectTarget()
except pywikibot.IsNotRedirectPage:
if len(redirList) == 1:
pywikibot.output(u'Skipping: Page %s is not a redirect.'
% redir.title(asLink=True))
break # do nothing
elif len(redirList) == 2:
pywikibot.output(
u'Skipping: Redirect target %s is not a redirect.'
% newRedir.title(asLink=True))
break # do nothing
else:
pass # target found
except pywikibot.SectionError:
pywikibot.warning(
u"Redirect target section %s doesn't exist."
% newRedir.title(asLink=True))
except (pywikibot.CircularRedirect,
pywikibot.InterwikiRedirectPage) as e:
pywikibot.exception(e)
pywikibot.output(u"Skipping %s." % newRedir)
break
except pywikibot.BadTitle as e:
# str(e) is in the format 'BadTitle: [[Foo]]'
pywikibot.warning(
u'Redirect target %s is not a valid page title.'
% str(e)[10:])
break
except pywikibot.NoPage:
if len(redirList) == 1:
pywikibot.output(u'Skipping: Page %s does not exist.'
% redir.title(asLink=True))
break
else:
if self.getOption('always'):
pywikibot.output(
u"Skipping: Redirect target %s doesn't exist."
% newRedir.title(asLink=True))
break # skip if automatic
else:
pywikibot.warning(
u"Redirect target %s doesn't exist."
% newRedir.title(asLink=True))
except pywikibot.ServerError:
pywikibot.output(u'Skipping due to server error: '
u'No textarea found')
break
else:
pywikibot.output(
u' Links to: %s.'
% targetPage.title(asLink=True))
try:
mw_msg = targetPage.site.mediawiki_message(
'wikieditor-toolbar-tool-redirect-example')
except KeyError:
pass
else:
if targetPage.title() == mw_msg:
pywikibot.output(
u"Skipping toolbar example: Redirect source is "
u"potentially vandalized.")
break
# watch out for redirect loops
if redirList.count(u'%s:%s'
% (targetPage.site.lang,
targetPage.title(withSection=False))):
pywikibot.warning(
u'Redirect target %s forms a redirect loop.'
% targetPage.title(asLink=True))
break # FIXME: doesn't work. edits twice!
try:
content = targetPage.get(get_redirect=True)
except pywikibot.SectionError:
content_page = pywikibot.Page(
targetPage.site,
targetPage.title(withSection=False))
content = content_page.get(get_redirect=True)
if i18n.twhas_key(
targetPage.site.lang,
'redirect-broken-redirect-template') and \
i18n.twhas_key(targetPage.site.lang,
'redirect-remove-loop'):
pywikibot.output(u"Tagging redirect for deletion")
# Delete the two redirects
content = i18n.twtranslate(
targetPage.site.lang,
'redirect-broken-redirect-template'
) + "\n" + content
summ = i18n.twtranslate(targetPage.site.lang,
'redirect-remove-loop')
targetPage.put(content, summ)
redir.put(content, summ)
break
else: # redirect target found
if targetPage.isStaticRedirect():
pywikibot.output(
u" Redirect target is STATICREDIRECT.")
pass
else:
newRedir = targetPage
continue
try:
oldText = redir.get(get_redirect=True)
except pywikibot.BadTitle:
pywikibot.output(u"Bad Title Error")
break
oldlink = self.site.redirectRegex().search(oldText).group(1)
if "#" in oldlink and targetPage.section() is None:
sectionlink = oldlink[oldlink.index("#"):]
targetlink = pywikibot.Page(
self.site,
targetPage.title() + sectionlink
).title(asLink=True, allow_interwiki=False, textlink=True)
else:
targetlink = targetPage.title(asLink=True, allow_interwiki=False, textlink=True)
text = self.site.redirectRegex().sub(
'#%s %s' % (self.site.redirect(),
targetlink),
oldText, 1)
if redir.title() == targetPage.title() or text == oldText:
pywikibot.output(u"Note: Nothing left to do on %s"
% redir.title(asLink=True))
break
summary = i18n.twtranslate(self.site, 'redirect-fix-double',
{'to': targetPage.title(asLink=True, allow_interwiki=False)}
)
pywikibot.showDiff(oldText, text)
if self.user_confirm(u'Do you want to accept the changes?'):
try:
redir.put(text, summary)
except pywikibot.LockedPage:
pywikibot.output(u'%s is locked.' % redir.title())
except pywikibot.SpamfilterError as error:
pywikibot.output(
u"Saving page [[%s]] prevented by spam filter: %s"
% (redir.title(), error.url))
except pywikibot.PageNotSaved as error:
pywikibot.output(u"Saving page [[%s]] failed: %s"
% (redir.title(), error))
except pywikibot.NoUsername:
pywikibot.output(
u"Page [[%s]] not saved; sysop privileges required."
% redir.title())
except pywikibot.Error as error:
pywikibot.output(
u"Unexpected error occurred trying to save [[%s]]: %s"
% (redir.title(), error))
break
def run(self):
"""Run bot."""
def convert_from_bytes(total_bytes):
for unit in ['B', 'K', 'M', 'G', 'T']:
if abs(total_bytes) < 1024:
return str(total_bytes) + unit
total_bytes = float(format(total_bytes / 1024.0, '.2f'))
return str(total_bytes) + 'P'
pywikibot.output('Downloading dump from ' + self.opt.wikiname)
download_filename = '{wikiname}-{dumpdate}-{filename}'.format_map(
self.opt)
temp_filename = download_filename + '-' \
+ binascii.b2a_hex(urandom(8)).decode('ascii') + '.part'
file_final_storepath = os.path.join(self.opt.storepath,
download_filename)
file_current_storepath = os.path.join(self.opt.storepath,
temp_filename)
# https://wikitech.wikimedia.org/wiki/Help:Toolforge#Dumps
toolforge_dump_filepath = self.get_dump_name(self.opt.wikiname,
self.opt.filename,
self.opt.dumpdate)
# First iteration for atomic download with temporary file
# Second iteration for fallback non-atomic download
for non_atomic in range(2):
try:
if toolforge_dump_filepath:
pywikibot.output('Symlinking file from ' +
toolforge_dump_filepath)
if non_atomic:
if os.path.exists(file_final_storepath):
remove(file_final_storepath)
symlink(toolforge_dump_filepath, file_current_storepath)
else:
url = 'https://dumps.wikimedia.org/{}/{}/{}'.format(
self.opt.wikiname, self.opt.dumpdate,
download_filename)
pywikibot.output('Downloading file from ' + url)
response = fetch(url, stream=True)
if response.status_code != HTTPStatus.OK:
if response.status_code == HTTPStatus.NOT_FOUND:
pywikibot.output(
'File with name {filename!r}, from dumpdate '
'{dumpdate!r}, and wiki {wikiname!r} ({url}) '
"isn't available in the Wikimedia Dumps".
format(url=url, **self.opt))
else:
pywikibot.output(
HTTPStatus(response.status_code).description)
return
with open(file_current_storepath, 'wb') as result_file:
total = int(response.headers['content-length'])
if total == -1:
pywikibot.warning("'content-length' missing in "
'response headers')
downloaded = 0
parts = 50
display_string = ''
pywikibot.output('')
for data in response.iter_content(100 * 1024):
result_file.write(data)
if total <= 0:
continue
downloaded += len(data)
done = int(parts * downloaded / total)
display = map(convert_from_bytes,
(downloaded, total))
prior_display = display_string
display_string = '\r|{}{}|{}{}/{}'.format(
'=' * done, '-' * (parts - done), ' ' * 5,
*display)
# Add whitespace to cover up prior bar
display_string += ' ' * (
len(prior_display.rstrip()) -
len(display_string.rstrip()))
pywikibot.output(display_string, newline=False)
pywikibot.output('')
# Rename the temporary file to the target file
# if the download completes successfully
if not non_atomic:
replace(file_current_storepath, file_final_storepath)
break
except OSError:
pywikibot.exception()
try:
remove(file_current_storepath)
except OSError:
pywikibot.exception()
# If the atomic download fails, try without a temporary file
# If the non-atomic download also fails, exit the script
if non_atomic:
return
pywikibot.output('Cannot make temporary file, '
'falling back to non-atomic download')
file_current_storepath = file_final_storepath
pywikibot.output('Done! File stored as ' + file_final_storepath)
def run(self):
"""Run the bot."""
# validate L10N
try:
self.template_list = self.site.family.category_redirect_templates[
self.site.code]
except KeyError:
pywikibot.warning(u"No redirect templates defined for %s" %
self.site)
return
if not self.get_cat_title():
pywikibot.warning(u"No redirect category found for %s" % self.site)
return
user = self.site.user() # invokes login()
newredirs = []
l = time.localtime()
today = "%04d-%02d-%02d" % l[:3]
edit_request_page = pywikibot.Page(
self.site, u"User:%s/category edit requests" % user)
datafile = pywikibot.config.datafilepath("%s-catmovebot-data" %
self.site.dbName())
try:
with open(datafile, "rb") as inp:
record = cPickle.load(inp)
except IOError:
record = {}
if record:
with open(datafile + ".bak", "wb") as f:
cPickle.dump(record, f, protocol=config.pickle_protocol)
# regex to match soft category redirects
# TODO: enhance and use textlib._MultiTemplateMatchBuilder
# note that any templates containing optional "category:" are
# incorrect and will be fixed by the bot
template_regex = re.compile(
r"""{{\s*(?:%(prefix)s\s*:\s*)? # optional "template:"
(?:%(template)s)\s*\| # catredir template name
(\s*%(catns)s\s*:\s*)? # optional "category:"
([^|}]+) # redirect target cat
(?:\|[^|}]*)*}} # optional arguments 2+, ignored
""" % {
'prefix':
self.site.namespace(10).lower(),
'template':
"|".join(
item.replace(" ", "[ _]+") for item in self.template_list),
'catns':
self.site.namespace(14)
}, re.I | re.X)
self.check_hard_redirect()
comment = i18n.twtranslate(self.site, self.move_comment)
counts = {}
nonemptypages = []
redircat = pywikibot.Category(pywikibot.Link(self.cat_title,
self.site))
pywikibot.output(u"\nChecking %d category redirect pages" %
redircat.categoryinfo['subcats'])
catpages = set()
for cat in redircat.subcategories():
catpages.add(cat)
cat_title = cat.title(withNamespace=False)
if "category redirect" in cat_title:
self.log_text.append(u"* Ignoring %s" %
cat.title(asLink=True, textlink=True))
continue
if hasattr(cat, "_catinfo"):
# skip empty categories that don't return a "categoryinfo" key
catdata = cat.categoryinfo
if "size" in catdata and int(catdata['size']):
# save those categories that have contents
nonemptypages.append(cat)
if cat_title not in record:
# make sure every redirect has a record entry
record[cat_title] = {today: None}
try:
newredirs.append("*# %s -> %s" %
(cat.title(asLink=True, textlink=True),
cat.getCategoryRedirectTarget().title(
asLink=True, textlink=True)))
except pywikibot.Error:
pass
# do a null edit on cat
try:
cat.save()
except:
pass
# delete record entries for non-existent categories
for cat_name in record.keys():
if pywikibot.Category(self.site,
self.catprefix + cat_name) not in catpages:
del record[cat_name]
pywikibot.output(u"\nMoving pages out of %s redirected categories." %
len(nonemptypages))
for cat in pagegenerators.PreloadingGenerator(nonemptypages):
try:
if not cat.isCategoryRedirect():
self.log_text.append(u"* False positive: %s" %
cat.title(asLink=True, textlink=True))
continue
except pywikibot.Error:
self.log_text.append(u"* Could not load %s; ignoring" %
cat.title(asLink=True, textlink=True))
continue
cat_title = cat.title(withNamespace=False)
if not self.readyToEdit(cat):
counts[cat_title] = None
self.log_text.append(u"* Skipping %s; in cooldown period." %
cat.title(asLink=True, textlink=True))
continue
dest = cat.getCategoryRedirectTarget()
if not dest.exists():
self.problems.append("# %s redirects to %s" %
(cat.title(asLink=True, textlink=True),
dest.title(asLink=True, textlink=True)))
# do a null edit on cat to update any special redirect
# categories this wiki might maintain
try:
cat.save()
except:
pass
continue
if dest.isCategoryRedirect():
double = dest.getCategoryRedirectTarget()
if double == dest or double == cat:
self.log_text.append(
u"* Redirect loop from %s" %
dest.title(asLink=True, textlink=True))
# do a null edit on cat
try:
cat.save()
except:
pass
else:
self.log_text.append(
u"* Fixed double-redirect: %s -> %s -> %s" %
(cat.title(asLink=True, textlink=True),
dest.title(asLink=True, textlink=True),
double.title(asLink=True, textlink=True)))
oldtext = cat.text
# remove the old redirect from the old text,
# leaving behind any non-redirect text
oldtext = template_regex.sub("", oldtext)
newtext = (u"{{%(redirtemp)s|%(ncat)s}}" % {
'redirtemp': self.template_list[0],
'ncat': double.title(withNamespace=False)
})
newtext = newtext + oldtext.strip()
try:
cat.text = newtext
cat.save(
i18n.twtranslate(self.site,
self.dbl_redir_comment))
except pywikibot.Error as e:
self.log_text.append("** Failed: %s" % e)
continue
found, moved = self.move_contents(cat_title,
dest.title(withNamespace=False),
editSummary=comment)
if found is None:
self.log_text.append(u"* [[:%s%s]]: error in move_contents" %
(self.catprefix, cat_title))
elif found:
record[cat_title][today] = found
self.log_text.append(u"* [[:%s%s]]: %d found, %d moved" %
(self.catprefix, cat_title, found, moved))
counts[cat_title] = found
# do a null edit on cat
try:
cat.save()
except:
pass
with open(datafile, "wb") as f:
cPickle.dump(record, f, protocol=config.pickle_protocol)
self.log_text.sort()
self.problems.sort()
newredirs.sort()
comment = i18n.twtranslate(self.site, self.maint_comment)
self.log_page.text = (
u"\n== %i-%02i-%02iT%02i:%02i:%02iZ ==\n" % time.gmtime()[:6] +
u'\n'.join(self.log_text) +
u'\n* New redirects since last report:\n' + u'\n'.join(newredirs) +
u'\n' + u'\n'.join(self.problems) + u'\n' + self.get_log_text())
self.log_page.save(comment)
if self.edit_requests:
edit_request_page.text = (self.edit_request_text % {
'itemlist':
u"\n" + u"\n".join((self.edit_request_item % item)
for item in self.edit_requests)
})
edit_request_page.save(comment)
def main(*args):
"""
Process command line arguments and perform task.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: str
"""
# Loading the comments
global categoryToCheck, project_inserted
# always, define a generator to understand if the user sets one,
# defining what's genFactory
always = False
generator = False
show = False
moveBlockCheck = False
protectedpages = False
protectType = 'edit'
namespace = 0
# Process global args and prepare generator args parser
local_args = pywikibot.handle_args(args)
genFactory = pagegenerators.GeneratorFactory()
# Process local args
for arg in local_args:
option, sep, value = arg.partition(':')
if option == '-always':
always = True
elif option == '-move':
moveBlockCheck = True
elif option == '-show':
show = True
elif option in ('-protectedpages', '-moveprotected'):
protectedpages = True
if option == '-moveprotected':
protectType = 'move'
if value:
namespace = int(value)
else:
genFactory.handleArg(arg)
if config.mylang not in project_inserted:
pywikibot.output('Your project is not supported by this script.\n'
'You have to edit the script and add it!')
return
site = pywikibot.Site()
if protectedpages:
generator = site.protectedpages(namespace=namespace, type=protectType)
# Take the right templates to use, the category and the comment
TSP = i18n.translate(site, templateSemiProtection)
TTP = i18n.translate(site, templateTotalProtection)
TSMP = i18n.translate(site, templateSemiMoveProtection)
TTMP = i18n.translate(site, templateTotalMoveProtection)
TNR = i18n.translate(site, templateNoRegex)
TU = i18n.translate(site, templateUnique)
categories = i18n.translate(site, categoryToCheck)
commentUsed = i18n.twtranslate(site, 'blockpageschecker-summary')
if not generator:
generator = genFactory.getCombinedGenerator()
if not generator:
generator = []
pywikibot.output('Loading categories...')
# Define the category if no other generator has been set
for CAT in categories:
cat = pywikibot.Category(site, CAT)
# Define the generator
gen = pagegenerators.CategorizedPageGenerator(cat)
for pageCat in gen:
generator.append(pageCat)
pywikibot.output('Categories loaded, start!')
# Main Loop
if not genFactory.nopreload:
generator = pagegenerators.PreloadingGenerator(generator, groupsize=60)
for page in generator:
pagename = page.title(as_link=True)
pywikibot.output('Loading %s...' % pagename)
try:
text = page.text
except pywikibot.NoPage:
pywikibot.output("%s doesn't exist! Skipping..." % pagename)
continue
except pywikibot.IsRedirectPage:
pywikibot.output('{} is a redirect! Skipping...'.format(pagename))
if show:
showQuest(page)
continue
# FIXME: This check does not work :
# PreloadingGenerator cannot set correctly page.editRestriction
# (see bug T57322)
# if not page.has_permission():
# pywikibot.output(
# "%s is sysop-protected : this account can't edit "
# "it! Skipping..." % pagename)
# continue
restrictions = page.protection()
try:
editRestr = restrictions['edit']
except KeyError:
editRestr = None
if not page.has_permission():
pywikibot.output('%s is protected: '
"this account can't edit it! Skipping..." %
pagename)
continue
# Understand, according to the template in the page, what should be the
# protection and compare it with what there really is.
TemplateInThePage = understandBlock(text, TTP, TSP, TSMP, TTMP, TU)
# Only to see if the text is the same or not...
oldtext = text
# keep track of the changes for each step (edit then move)
changes = -1
if not editRestr:
# page is not edit-protected
# Deleting the template because the page doesn't need it.
if not (TTP or TSP):
raise pywikibot.Error(
'This script is not localized to use it on \n{0}. '
'Missing "templateSemiProtection" or'
'"templateTotalProtection"'.format(site.sitename))
if TU:
replaceToPerform = '|'.join(TTP + TSP + TU)
else:
replaceToPerform = '|'.join(TTP + TSP)
text, changes = re.subn(
'<noinclude>(%s)</noinclude>' % replaceToPerform, '', text)
if changes == 0:
text, changes = re.subn('(%s)' % replaceToPerform, '', text)
msg = 'The page is editable for all'
if not moveBlockCheck:
msg += ', deleting the template..'
pywikibot.output(msg + '.')
elif editRestr[0] == 'sysop':
# total edit protection
if (TemplateInThePage[0] == 'sysop-total' and TTP) or \
(TemplateInThePage[0] == 'unique' and TU):
msg = 'The page is protected to the sysop'
if not moveBlockCheck:
msg += ', skipping...'
pywikibot.output(msg)
else:
if not TNR or TU and not TNR[4] or not (TU or TNR[1]):
raise pywikibot.Error(
'This script is not localized to use it on \n{0}. '
'Missing "templateNoRegex"'.format(site.sitename))
pywikibot.output('The page is protected to the sysop, but the '
'template seems not correct. Fixing...')
if TU:
text, changes = re.subn(TemplateInThePage[1], TNR[4], text)
else:
text, changes = re.subn(TemplateInThePage[1], TNR[1], text)
elif TSP or TU:
# implicitly editRestr[0] = 'autoconfirmed', edit-Semi-protection
if TemplateInThePage[0] == 'autoconfirmed-total' or \
TemplateInThePage[0] == 'unique':
msg = 'The page is editable only for the autoconfirmed users'
if not moveBlockCheck:
msg += ', skipping...'
pywikibot.output(msg)
else:
if not TNR or TU and not TNR[4] or not (TU or TNR[1]):
raise pywikibot.Error(
'This script is not localized to use it on \n{0}. '
'Missing "templateNoRegex"'.format(site.sitename))
pywikibot.output('The page is editable only for the '
'autoconfirmed users, but the template '
'seems not correct. Fixing...')
if TU:
text, changes = re.subn(TemplateInThePage[1], TNR[4], text)
else:
text, changes = re.subn(TemplateInThePage[1], TNR[0], text)
if changes == 0:
# We tried to fix edit-protection templates, but it did not work.
pywikibot.warning('No edit-protection template could be found')
if moveBlockCheck and changes > -1:
# checking move protection now
try:
moveRestr = restrictions['move']
except KeyError:
moveRestr = False
changes = -1
if not moveRestr:
pywikibot.output('The page is movable for all, deleting the '
'template...')
# Deleting the template because the page doesn't need it.
if TU:
replaceToPerform = '|'.join(TSMP + TTMP + TU)
else:
replaceToPerform = '|'.join(TSMP + TTMP)
text, changes = re.subn(
'<noinclude>(%s)</noinclude>' % replaceToPerform, '', text)
if changes == 0:
text, changes = re.subn('({})'.format(replaceToPerform),
'', text)
elif moveRestr[0] == 'sysop':
# move-total-protection
if (TemplateInThePage[0] == 'sysop-move' and TTMP) or \
(TemplateInThePage[0] == 'unique' and TU):
pywikibot.output('The page is protected from moving to '
'the sysop, skipping...')
if TU:
# no changes needed, better to revert the old text.
text = oldtext
else:
pywikibot.output('The page is protected from moving to '
'the sysop, but the template seems not '
'correct. Fixing...')
if TU:
text, changes = re.subn(TemplateInThePage[1], TNR[4],
text)
else:
text, changes = re.subn(TemplateInThePage[1], TNR[3],
text)
elif TSMP or TU:
# implicitly moveRestr[0] = 'autoconfirmed',
# move-semi-protection
if TemplateInThePage[0] == 'autoconfirmed-move' or \
TemplateInThePage[0] == 'unique':
pywikibot.output('The page is movable only for the '
'autoconfirmed users, skipping...')
if TU:
# no changes needed, better to revert the old text.
text = oldtext
else:
pywikibot.output('The page is movable only for the '
'autoconfirmed users, but the template '
'seems not correct. Fixing...')
if TU:
text, changes = re.subn(TemplateInThePage[1], TNR[4],
text)
else:
text, changes = re.subn(TemplateInThePage[1], TNR[2],
text)
if changes == 0:
# We tried to fix move-protection templates but it did not work
pywikibot.warning('No move-protection template could be found')
if oldtext != text:
# Ok, asking if the change has to be performed and do it if yes.
pywikibot.output(
color_format('\n\n>>> {lightpurple}{0}{default} <<<',
page.title()))
pywikibot.showDiff(oldtext, text)
if not always:
choice = pywikibot.input_choice(
'Do you want to accept these '
'changes?', [('Yes', 'y'), ('No', 'n'), ('All', 'a')], 'n')
if choice == 'a':
always = True
if always or choice == 'y':
save_page(page, text, commentUsed)
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: str
"""
local_args = pywikibot.handle_args(args)
# default values for options
extra_summary = None
for arg in local_args:
option, sep, value = arg.partition(':')
if option == '-summary':
extra_summary = value
else:
pywikibot.warning("Unrecognized option {}".format(option))
def check_option(option, value):
if not value:
pywikibot.error("Missing argument for option '{}'".format(option))
return False
return True
page_number_regex = re.compile('\|([1-9][0-9]*)\}')
filename_number_regex = re.compile('([1-9][0-9]*)')
templates_ready = ['QC image', 'File information', 'Self']
site = pywikibot.Site()
looked_at = set()
pages = request_pages()
for p in pages:
if p['title'] in looked_at:
pywikibot.output("Done.")
break
else:
looked_at.add(p['title'])
try:
page_title = 'File:' + p['title']
page = pywikibot.Page(site, page_title)
click_url = ROOT_URL + 'wiki/' + page.title(underscore=True)
pywikibot.output("Page '{0}', id={1} | {2}".format(page_title, p['id'], click_url))
ts = page.templatesWithParams()
if len(ts) > 0:
found_ready = False
for t in ts:
for r in templates_ready:
if r in t[0].title():
pywikibot.output(color_format("Page {lightgreen}{0}{default} has template: {1}",
page_title, t[0]))
found_ready = True
break
if found_ready:
pywikibot.output("\tSkipping.")
continue
old_text = page.get()
# categories = getCategoryLinks(old_text, site)
# categories_text = '\n'.join(map(lambda c:c.aslink(), categories))
(header, body, footer) = extract_sections(old_text, site)
summary = None
licensing = None
description = None
for section in body:
if 'ummary' in section[0] or 'escription' in section[0]:
summary = section[1]
if 'icens' in section[0]:
licensing = section[1]
got_summary_from_header = False
if summary is None:
got_summary_from_header = True
summary = header
new_text = None
pywikibot.output(color_format("Editing page {lightblue}{0}{default}.", page_title))
if summary is not None and len(summary.strip()) > 0:
summary = summary.strip()
pywikibot.output("Have \"Summary\":\n\t{}".format(summary))
i = summary.find('{')
if i > 0:
summary = summary[0:i]
i = summary.find(' in ')
if i > 0:
summary = summary[0:i]
summary = summary.strip()
if summary[-1] == '.':
summary = summary[0:-1]
pywikibot.output("Will have \"Summary\":\n\t{}".format(summary))
choice = pywikibot.input_choice("Is it a good summary?",
[('Yes', 'y'), ('No', 'n'), ('open in Browser', 'b')], 'n')
if choice == 'y':
description = summary
elif choice == 'n':
pass
elif choice == 'b':
pywikibot.bot.open_webbrowser(page)
if description is None:
pywikibot.output("Type '[s]kip' to skip the image completely.")
description = pywikibot.input("Please describe the file:")
if description in ['s', 'skip']:
continue
if licensing is not None:
pywikibot.output("Have \"Licensing\":\n\t{}".format(licensing.strip()))
comic_num = None
m = page_number_regex.search(old_text)
if m:
try:
comic_num = int(m.group(1))
except:
pass
if comic_num is None:
m = filename_number_regex.search(page.title())
if m:
try:
comic_num = int(m.group(1))
except:
pass
if comic_num is not None:
pywikibot.output("Have comic #:\n\t{}".format(comic_num))
choice = pywikibot.input_choice("Is it a good comic number?",
[('Yes', 'y'), ('No', 'n'), ('open in Browser', 'b')], 'n')
if choice == 'y':
pass
else:
comic_num = None
if choice == 'b':
pywikibot.bot.open_webbrowser(page)
while comic_num is None:
try:
pywikibot.output("Need comic number. Type 0 to skip")
comic_num = int(pywikibot.input("Comic number: "))
except ValueError:
pass
if comic_num == 0:
comic_num = ''
new_text = dedent("""
== Summary ==
{{{{QC image|{0}|{1}}}}}
== Licensing ==
{{{{Fairuse}}}}
""".format(description, comic_num)).strip()
header = header.strip()
if not got_summary_from_header and len(header) > 0:
new_text = header + '\n\n' + new_text
footer = footer.strip()
if len(footer) > 0:
new_text += '\n\n' + footer
# check if the edit is sensible
if old_text == new_text:
pywikibot.output("No changes. Nothing to do.")
continue
# report what will happen
pywikibot.showDiff(old_text, new_text, context=3)
summary = "add [[Template:QC image]]; mark as fair use " + \
"([[User:AndrybakBot#Image maintenance|Image maintenance bot task]])"
if extra_summary:
summary = summary + " ({})".format(extra_summary)
pywikibot.output(color_format("Summary will be" +
"\n\t{lightblue}{0}{default}", summary))
choice = pywikibot.input_choice(
"Do you want to accept these changes?",
[('Yes', 'y'), ('No', 'n'), ('open in Browser', 'b')], 'n')
# if choice == 'y':
# pywikibot.output("Test run, doing nothing.")
# continue
if choice == 'n':
pywikibot.output("Okay, doing nothing.")
continue
elif choice == 'b':
pywikibot.bot.open_webbrowser(page)
elif choice == 'y':
error_count = 0
while True:
result = put_text(page, new_text, summary, error_count)
if result is not None:
pywikibot.output("Got result of saving: {}".format(result))
break
error_count += 1
continue
elif choice == 'q':
break
except pywikibot.NoPage:
pywikibot.error("{} doesn't exist, skipping.".format(page.title()))
continue
except pywikibot.IsRedirectPage:
pywikibot.error("{} is a redirect, skipping".format(page.title()))
continue
except pywikibot.Error as e:
pywikibot.bot.suggest_help(exception=e)
continue
except QuitKeyboardInterrupt:
sys.exit("User quit bot run.")
else:
pass
def __iter__(self):
"""Iterator."""
n = 0
event = None
while self._total is None or n < self._total:
if not hasattr(self, 'source'):
self.source = EventSource(**self.sse_kwargs)
# sseclient >= 0.0.18 is required for eventstreams (T184713)
# we don't have a version string inside but the instance
# variable 'chunk_size' was newly introduced with 0.0.18
if not hasattr(self.source, 'chunk_size'):
warning(
'You may not have the right sseclient version;\n'
'sseclient >= 0.0.18 is required for eventstreams.\n'
"Install it with 'pip install \"sseclient>=0.0.18\"'")
try:
event = next(self.source)
except (ProtocolError, socket.error, httplib.IncompleteRead) as e:
warning('Connection error: {0}.\n'
'Try to re-establish connection.'.format(e))
del self.source
if event is not None:
self.sse_kwargs['last_id'] = event.id
continue
if event.event == 'message':
if event.data:
try:
element = json.loads(event.data)
except ValueError as e:
warning(
'Could not load json data from\n{0}\n{1}'.format(
event, e))
else:
if self.streamfilter(element):
n += 1
yield element
else:
warning('Empty message found.')
elif event.event == 'error':
warning('Encountered error: {0}'.format(event.data))
else:
warning('Unknown event {0} occured.'.format(event.event))
else:
debug(
'{0}: Stopped iterating due to '
'exceeding item limit.'.format(self.__class__.__name__),
_logger)
del self.source
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
add_cat = None
gen = None
# summary message
edit_summary = u""
# Array which will collect commandline parameters.
# First element is original text, second element is replacement text.
commandline_replacements = []
# A list of 2-tuples of original text and replacement text.
replacements = []
# Don't edit pages which contain certain texts.
exceptions = {
'title': [],
'text-contains': [],
'inside': [],
'inside-tags': [],
'require-title': [], # using a seperate requirements dict needs some
} # major refactoring of code.
# Should the elements of 'replacements' and 'exceptions' be interpreted
# as regular expressions?
regex = False
# Predefined fixes from dictionary 'fixes' (see above).
fixes_set = []
# the dump's path, either absolute or relative, which will be used
# if -xml flag is present
xmlFilename = None
useSql = False
# will become True when the user presses a ('yes to all') or uses the
# -always flag.
acceptall = False
# Will become True if the user inputs the commandline parameter -nocase
caseInsensitive = False
# Will become True if the user inputs the commandline parameter -dotall
dotall = False
# Will become True if the user inputs the commandline parameter -multiline
multiline = False
# Do all hits when they overlap
allowoverlap = False
# Do not recurse replacement
recursive = False
# Between a regex and another (using -fix) sleep some time (not to waste
# too much CPU
sleep = None
# Request manual replacements even if replacements are already defined
manual_input = False
# Replacements loaded from a file
replacement_file = None
replacement_file_arg_misplaced = False
# Read commandline parameters.
local_args = pywikibot.handle_args(args)
genFactory = pagegenerators.GeneratorFactory()
for arg in local_args:
if genFactory.handleArg(arg):
continue
if arg == '-regex':
regex = True
elif arg.startswith('-xmlstart'):
if len(arg) == 9:
xmlStart = pywikibot.input(
u'Please enter the dumped article to start with:')
else:
xmlStart = arg[10:]
elif arg.startswith('-xml'):
if len(arg) == 4:
xmlFilename = i18n.input('pywikibot-enter-xml-filename')
else:
xmlFilename = arg[5:]
elif arg == '-sql':
useSql = True
elif arg.startswith('-excepttitle:'):
exceptions['title'].append(arg[13:])
elif arg.startswith('-requiretitle:'):
exceptions['require-title'].append(arg[14:])
elif arg.startswith('-excepttext:'):
exceptions['text-contains'].append(arg[12:])
elif arg.startswith('-exceptinside:'):
exceptions['inside'].append(arg[14:])
elif arg.startswith('-exceptinsidetag:'):
exceptions['inside-tags'].append(arg[17:])
elif arg.startswith('-fix:'):
fixes_set += [arg[5:]]
elif arg.startswith('-sleep:'):
sleep = float(arg[7:])
elif arg == '-always':
acceptall = True
elif arg == '-recursive':
recursive = True
elif arg == '-nocase':
caseInsensitive = True
elif arg == '-dotall':
dotall = True
elif arg == '-multiline':
multiline = True
elif arg.startswith('-addcat:'):
add_cat = arg[8:]
elif arg.startswith('-summary:'):
edit_summary = arg[9:]
elif arg.startswith('-automaticsummary'):
edit_summary = True
elif arg.startswith('-allowoverlap'):
allowoverlap = True
elif arg.startswith('-manualinput'):
manual_input = True
elif arg.startswith('-replacementfile'):
issue_deprecation_warning('-replacementfile', '-pairsfile', 2,
ArgumentDeprecationWarning)
elif arg.startswith('-pairsfile'):
if len(commandline_replacements) % 2:
replacement_file_arg_misplaced = True
if arg == '-pairsfile':
replacement_file = pywikibot.input(
u'Please enter the filename to read replacements from:')
else:
replacement_file = arg[len('-pairsfile:'):]
else:
commandline_replacements.append(arg)
site = pywikibot.Site()
if len(commandline_replacements) % 2:
pywikibot.error('Incomplete command line pattern replacement pair.')
return False
if replacement_file_arg_misplaced:
pywikibot.error('-pairsfile used between a pattern replacement pair.')
return False
if replacement_file:
try:
with codecs.open(replacement_file, 'r', 'utf-8') as f:
# strip newlines, but not other characters
file_replacements = f.read().splitlines()
except (IOError, OSError) as e:
pywikibot.error(u'Error loading {0}: {1}'.format(
replacement_file, e))
return False
if len(file_replacements) % 2:
pywikibot.error(
'{0} contains an incomplete pattern replacement pair.'.format(
replacement_file))
return False
# Strip BOM from first line
file_replacements[0].lstrip(u'\uFEFF')
commandline_replacements.extend(file_replacements)
if not (commandline_replacements or fixes_set) or manual_input:
old = pywikibot.input('Please enter the text that should be replaced:')
while old:
new = pywikibot.input(u'Please enter the new text:')
commandline_replacements += [old, new]
old = pywikibot.input(
'Please enter another text that should be replaced,'
'\nor press Enter to start:')
# The summary stored here won't be actually used but is only an example
single_summary = None
for i in range(0, len(commandline_replacements), 2):
replacement = Replacement(commandline_replacements[i],
commandline_replacements[i + 1])
if not single_summary:
single_summary = i18n.twtranslate(site, 'replace-replacing', {
'description':
' (-%s +%s)' % (replacement.old, replacement.new)
})
replacements.append(replacement)
# Perform one of the predefined actions.
missing_fixes_summaries = [] # which a fixes/replacements miss a summary
for fix_name in fixes_set:
try:
fix = fixes.fixes[fix_name]
except KeyError:
pywikibot.output(u'Available predefined fixes are: %s' %
', '.join(fixes.fixes.keys()))
if not fixes.user_fixes_loaded:
pywikibot.output('The user fixes file could not be found: '
'{0}'.format(fixes.filename))
return
if not fix['replacements']:
pywikibot.warning('No replacements defined for fix '
'"{0}"'.format(fix_name))
continue
if "msg" in fix:
if isinstance(fix['msg'], basestring):
set_summary = i18n.twtranslate(site, str(fix['msg']))
else:
set_summary = i18n.translate(site, fix['msg'], fallback=True)
else:
set_summary = None
replacement_set = ReplacementList(fix.get('regex'),
fix.get('exceptions'),
fix.get('nocase'),
set_summary,
name=fix_name)
# Whether some replacements have a summary, if so only show which
# have none, otherwise just mention the complete fix
missing_fix_summaries = []
for index, replacement in enumerate(fix['replacements'], start=1):
summary = None if len(replacement) < 3 else replacement[2]
if not set_summary and not summary:
missing_fix_summaries.append('"{0}" (replacement #{1})'.format(
fix_name, index))
if chars.contains_invisible(replacement[0]):
pywikibot.warning('The old string "{0}" contains formatting '
'characters like U+200E'.format(
chars.replace_invisible(replacement[0])))
if (not callable(replacement[1])
and chars.contains_invisible(replacement[1])):
pywikibot.warning('The new string "{0}" contains formatting '
'characters like U+200E'.format(
chars.replace_invisible(replacement[1])))
replacement_set.append(
ReplacementListEntry(
old=replacement[0],
new=replacement[1],
fix_set=replacement_set,
edit_summary=summary,
))
# Exceptions specified via 'fix' shall be merged to those via CLI.
if replacement_set:
replacements.extend(replacement_set)
if replacement_set._exceptions is not None:
for k, v in replacement_set._exceptions.items():
if k in exceptions:
exceptions[k] = list(set(exceptions[k]) | set(v))
else:
exceptions[k] = v
if len(fix['replacements']) == len(missing_fix_summaries):
missing_fixes_summaries.append(
'"{0}" (all replacements)'.format(fix_name))
else:
missing_fixes_summaries += missing_fix_summaries
if ((not edit_summary or edit_summary is True)
and (missing_fixes_summaries or single_summary)):
if single_summary:
pywikibot.output(u'The summary message for the command line '
'replacements will be something like: %s' %
single_summary)
if missing_fixes_summaries:
pywikibot.output('The summary will not be used when the fix has '
'one defined but the following fix(es) do(es) '
'not have a summary defined: '
'{0}'.format(', '.join(missing_fixes_summaries)))
if edit_summary is not True:
edit_summary = pywikibot.input(
'Press Enter to use this automatic message, or enter a '
'description of the\nchanges your bot will make:')
else:
edit_summary = ''
# Set the regular expression flags
flags = re.UNICODE
if caseInsensitive:
flags = flags | re.IGNORECASE
if dotall:
flags = flags | re.DOTALL
if multiline:
flags = flags | re.MULTILINE
# Pre-compile all regular expressions here to save time later
for replacement in replacements:
replacement.compile(regex, flags)
precompile_exceptions(exceptions, regex, flags)
if xmlFilename:
try:
xmlStart
except NameError:
xmlStart = None
gen = XmlDumpReplacePageGenerator(xmlFilename, xmlStart, replacements,
exceptions, site)
elif useSql:
whereClause = 'WHERE (%s)' % ' OR '.join([
"old_text RLIKE '%s'" % prepareRegexForMySQL(old_regexp.pattern)
for (old_regexp, new_text) in replacements
])
if exceptions:
exceptClause = 'AND NOT (%s)' % ' OR '.join([
"old_text RLIKE '%s'" % prepareRegexForMySQL(exc.pattern)
for exc in exceptions
])
else:
exceptClause = ''
query = u"""
SELECT page_namespace, page_title
FROM page
JOIN text ON (page_id = old_id)
%s
%s
LIMIT 200""" % (whereClause, exceptClause)
gen = pagegenerators.MySQLPageGenerator(query)
gen = genFactory.getCombinedGenerator(gen, preload=True)
if not gen:
pywikibot.bot.suggest_help(missing_generator=True)
return False
bot = ReplaceRobot(gen,
replacements,
exceptions,
allowoverlap,
recursive,
add_cat,
sleep,
edit_summary,
always=acceptall,
site=site)
site.login()
bot.run()
# Explicitly call pywikibot.stopme(). It will make sure the callback is
# triggered before replace.py is unloaded.
pywikibot.stopme()
pywikibot.output(u'\n%s pages changed.' % bot.changed_pages)
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: unicode
"""
local_args = pywikibot.handle_args(args)
group_id = ''
photoset_id = ''
user_id = ''
start_id = ''
end_id = ''
tags = ''
addCategory = ''
removeCategories = False
autonomous = False
totalPhotos = 0
uploadedPhotos = 0
# Do we mark the images as reviewed right away?
if config.flickr['review']:
flickrreview = config.flickr['review']
else:
flickrreview = False
# Set the Flickr reviewer
if config.flickr['reviewer']:
reviewer = config.flickr['reviewer']
elif 'commons' in config.sysopnames['commons']:
pywikibot.output(config.sysopnames['commons'])
reviewer = config.sysopnames['commons']['commons']
elif 'commons' in config.usernames['commons']:
reviewer = config.usernames['commons']['commons']
else:
reviewer = ''
# Should be renamed to overrideLicense or something like that
override = ''
for arg in local_args:
if arg.startswith('-group_id'):
if len(arg) == 9:
group_id = pywikibot.input('What is the group_id of the pool?')
else:
group_id = arg[10:]
elif arg.startswith('-photoset_id'):
if len(arg) == 12:
photoset_id = pywikibot.input('What is the photoset_id?')
else:
photoset_id = arg[13:]
elif arg.startswith('-user_id'):
if len(arg) == 8:
user_id = pywikibot.input(
'What is the user_id of the flickr user?')
else:
user_id = arg[9:]
elif arg.startswith('-start_id'):
if len(arg) == 9:
start_id = pywikibot.input(
'What is the id of the photo you want to start at?')
else:
start_id = arg[10:]
elif arg.startswith('-end_id'):
if len(arg) == 7:
end_id = pywikibot.input(
'What is the id of the photo you want to end at?')
else:
end_id = arg[8:]
elif arg.startswith('-tags'):
if len(arg) == 5:
tags = pywikibot.input(
'What is the tag you want to filter out (currently only '
'one supported)?')
else:
tags = arg[6:]
elif arg == '-flickrreview':
flickrreview = True
elif arg.startswith('-reviewer'):
if len(arg) == 9:
reviewer = pywikibot.input('Who is the reviewer?')
else:
reviewer = arg[10:]
elif arg.startswith('-override'):
if len(arg) == 9:
override = pywikibot.input('What is the override text?')
else:
override = arg[10:]
elif arg.startswith('-addcategory'):
if len(arg) == 12:
addCategory = pywikibot.input(
'What category do you want to add?')
else:
addCategory = arg[13:]
elif arg == '-removecategories':
removeCategories = True
elif arg == '-autonomous':
autonomous = True
if isinstance(flickrapi, Exception):
pywikibot.error('This script requires the python flickrapi module. \n'
'See: http://stuvel.eu/projects/flickrapi')
elif not config.flickr['api_key']:
pywikibot.warning('Flickr api key not found! Get yourself an api key\n'
'Any flickr user can get a key at\n'
'https://www.flickr.com/services/api/keys/apply/')
elif user_id or group_id or photoset_id:
if 'api_secret' in config.flickr and config.flickr['api_secret']:
flickr = flickrapi.FlickrAPI(config.flickr['api_key'],
config.flickr['api_secret'])
else:
pywikibot.output('Accessing public content only')
flickr = flickrapi.FlickrAPI(config.flickr['api_key'])
for photo_id in getPhotos(flickr, user_id, group_id, photoset_id,
start_id, end_id, tags):
uploadedPhotos += processPhoto(flickr, photo_id, flickrreview,
reviewer, override, addCategory,
removeCategories, autonomous)
totalPhotos += 1
pywikibot.output('Finished running')
pywikibot.output('Total photos: ' + str(totalPhotos))
pywikibot.output('Uploaded photos: ' + str(uploadedPhotos))
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
oldName = None
options = {}
fromToPairs = []
# Process global args and prepare generator args parser
local_args = pywikibot.handle_args(args)
genFactory = pagegenerators.GeneratorFactory()
for arg in local_args:
if arg == '-pairs' or arg.startswith('-pairs:'):
issue_deprecation_warning('-pairs', '-pairsfile', 2,
ArgumentDeprecationWarning)
elif arg.startswith('-pairsfile'):
if len(arg) == len('-pairsfile'):
filename = pywikibot.input(
u'Enter the name of the file containing pairs:')
else:
filename = arg[len('-pairsfile:'):]
oldName1 = None
for page in pagegenerators.TextfilePageGenerator(filename):
if oldName1:
fromToPairs.append([oldName1, page.title()])
oldName1 = None
else:
oldName1 = page.title()
if oldName1:
pywikibot.warning(u'file %s contains odd number of links' %
filename)
elif arg == '-noredirect':
options['noredirect'] = True
elif arg == '-notalkpage':
options['movetalkpage'] = False
elif arg == '-always':
options['always'] = True
elif arg == '-skipredirects':
options['skipredirects'] = True
elif arg.startswith('-from:'):
if oldName:
pywikibot.warning(u'-from:%s without -to:' % oldName)
oldName = arg[len('-from:'):]
elif arg.startswith('-to:'):
if oldName:
fromToPairs.append([oldName, arg[len('-to:'):]])
oldName = None
else:
pywikibot.warning(u'%s without -from' % arg)
elif arg.startswith('-prefix'):
if len(arg) == len('-prefix'):
options['prefix'] = pywikibot.input(u'Enter the prefix:')
else:
options['prefix'] = arg[8:]
elif arg.startswith('-summary'):
if len(arg) == len('-summary'):
options['summary'] = pywikibot.input(u'Enter the summary:')
else:
options['summary'] = arg[9:]
else:
genFactory.handleArg(arg)
if oldName:
pywikibot.warning(u'-from:%s without -to:' % oldName)
site = pywikibot.Site()
for pair in fromToPairs:
page = pywikibot.Page(site, pair[0])
bot = MovePagesBot(None, **options)
bot.moveOne(page, pair[1])
gen = genFactory.getCombinedGenerator(preload=True)
if gen:
bot = MovePagesBot(gen, **options)
bot.run()
return True
if not fromToPairs:
pywikibot.bot.suggest_help(missing_generator=True)
return False
else:
return True
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: str
"""
local_args = pywikibot.handle_args(args)
fromsite = pywikibot.Site()
tolang = fromsite.code
tofamily = fromsite.family.name
prefix = ''
overwrite = False
target = False
gen_args = []
for arg in local_args:
if arg.startswith('-tofamily'):
tofamily = arg[len('-tofamily:'):]
elif arg.startswith('-tolang'):
tolang = arg[len('-tolang:'):]
elif arg.startswith('-prefix'):
prefix = arg[len('-prefix:'):]
elif arg == '-overwrite':
overwrite = True
elif arg == '-target':
target = True
else:
gen_args.append(arg)
tosite = pywikibot.Site(tolang, tofamily)
additional_text = ('Target site not different from source site.'
if fromsite == tosite else '')
gen_factory = pagegenerators.GeneratorFactory(
site=tosite if target else fromsite)
unknown_args = [arg for arg in gen_args if not gen_factory.handleArg(arg)]
gen = gen_factory.getCombinedGenerator()
if suggest_help(missing_generator=not gen,
additional_text=additional_text,
unknown_parameters=unknown_args):
return
gen_args = ' '.join(gen_args)
pywikibot.output("""
Page transfer configuration
---------------------------
Source: {fromsite}
Target: {tosite}
Generator of pages to transfer: {gen_args}
{target}
Prefix for transferred pages: {prefix}
""".format(fromsite=fromsite,
tosite=tosite,
gen_args=gen_args,
prefix=prefix if prefix else '(none)',
target='from target site\n' if target else ''))
for page in gen:
title = page.namespace().canonical_prefix() + page.title(with_ns=False)
if target:
# page is at target site, fetch it from source
target_title = prefix + page.title()
page = pywikibot.Page(fromsite, title)
else:
target_title = (prefix + title)
targetpage = pywikibot.Page(tosite, target_title)
edithistpage = pywikibot.Page(tosite, target_title + '/edithistory')
if targetpage.exists():
if not overwrite:
pywikibot.warning(
'Skipped {0} (target page {1} exists)'.format(
page.title(as_link=True, force_interwiki=True),
targetpage.title(as_link=True)))
continue
if not targetpage.botMayEdit():
pywikibot.warning(
'Target page {0} is not editable by bots'.format(
targetpage.title(as_link=True)))
continue
if not page.exists():
pywikibot.warning("Page {0} doesn't exist".format(
page.title(as_link=True)))
continue
pywikibot.output('Moving {0} to {1}...'.format(
page.title(as_link=True, force_interwiki=True),
targetpage.title(as_link=True)))
pywikibot.log('Getting page text.')
text = page.get(get_redirect=True)
source_link = page.title(as_link=True, insite=targetpage.site)
text += (
'<noinclude>\n\n<small>This page was moved from {0}. Its '
'edit history can be viewed at {1}</small></noinclude>'.format(
source_link,
edithistpage.title(as_link=True, insite=targetpage.site)))
pywikibot.log('Getting edit history.')
historytable = page.getVersionHistoryTable()
pywikibot.log('Putting edit history.')
summary = 'Moved page from {source}'.format(source=source_link)
edithistpage.put(historytable, summary=summary)
pywikibot.log('Putting page text.')
edithist_link = ' ([[{target}/edithistory|history]])'.format(
target=targetpage.title(
) if not targetpage.namespace().subpages else '')
summary += edithist_link
targetpage.put(text, summary=summary)
def processPhoto(flickr,
photo_id='',
flickrreview=False,
reviewer='',
override='',
addCategory='',
removeCategories=False,
autonomous=False):
"""Process a single Flickr photo.
For each image:
* Check the license
* Check if it isn't already on Commons
* Build suggested filename
* Check for name collision and maybe alter it
* Pull description from Flinfo
* Show image and description to user
* Add a nice hotcat lookalike for the adding of categories
* Filter the categories
* Upload the image
"""
if photo_id:
pywikibot.output(str(photo_id))
(photoInfo, photoSizes) = getPhoto(flickr, photo_id)
if isAllowedLicense(photoInfo) or override:
# Get the url of the largest photo
photoUrl = getPhotoUrl(photoSizes)
# Should download the photo only once
photo = downloadPhoto(photoUrl)
# Don't upload duplicate images, should add override option
duplicates = findDuplicateImages(photo)
if duplicates:
pywikibot.output('Found duplicate image at {}'.format(
duplicates.pop()))
else:
filename = getFilename(photoInfo)
flinfoDescription = getFlinfoDescription(photo_id)
photoDescription = buildDescription(flinfoDescription,
flickrreview, reviewer,
override, addCategory,
removeCategories)
# pywikibot.output(photoDescription)
if not isinstance(Tkdialog, ImportError) and not autonomous:
try:
(newPhotoDescription, newFilename,
skip) = Tkdialog(photoDescription, photo,
filename).show_dialog()
except ImportError as e:
pywikibot.warning(e)
pywikibot.warning('Switching to autonomous mode.')
autonomous = True
elif not autonomous:
pywikibot.warning('Switching to autonomous mode because GUI '
'interface cannot be used')
pywikibot.warning(Tkdialog)
autonomous = True
if autonomous:
newPhotoDescription = photoDescription
newFilename = filename
skip = False
# Do the actual upload
# Would be nice to check before I upload if the file is already at
# Commons. Not that important for this program, but maybe for
# derived programs
if not skip:
bot = UploadRobot(photoUrl,
description=newPhotoDescription,
useFilename=newFilename,
keepFilename=True,
verifyDescription=False)
bot.upload_image(debug=False)
return 1
else:
pywikibot.output('Invalid license')
return 0
def badNameFilter(self, name, force=False):
"""Check for bad names."""
if not globalvar.filtBadName:
return False
# initialize blacklist
if not hasattr(self, '_blacklist') or force:
elenco = [
' ano',
' anus',
'anal ',
'babies',
'baldracca',
'balle',
'bastardo',
'bestiali',
'bestiale',
'bastarda',
'b.i.t.c.h.',
'bitch',
'boobie',
'bordello',
'breast',
'cacata',
'cacca',
'cachapera',
'cagata',
'cane',
'cazz',
'cazzo',
'cazzata',
'chiavare',
'chiavata',
'chick',
'christ ',
'cristo',
'clitoride',
'coione',
'cojdioonear',
'cojones',
'cojo',
'coglione',
'coglioni',
'cornuto',
'cula',
'culatone',
'culattone',
'culo',
'deficiente',
'deficente',
'dio',
'die ',
'died ',
'ditalino',
'ejackulate',
'enculer',
'eroticunt',
'fanculo',
'f******o',
'fica ',
'ficken',
'figa',
'sfiga',
'fottere',
'fotter',
'fottuto',
'f**k',
'f.u.c.k.',
'funkyass',
'gay',
'hentai.com',
'horne',
'horney',
'virgin',
'hotties',
'idiot',
'@alice.it',
'incest',
'jesus',
'gesu',
'gesù',
'kazzo',
'kill',
'leccaculo',
'lesbian',
'lesbica',
'lesbo',
'masturbazione',
'masturbare',
'masturbo',
'merda',
'merdata',
'merdoso',
'mignotta',
'minchia',
'minkia',
'minchione',
'mona',
'nudo',
'nuda',
'nudi',
'oral',
'sex',
'orgasmso',
'porc',
'pompa',
'pompino',
'porno',
'puttana',
'puzza',
'puzzone',
'racchia',
'sborone',
'sborrone',
'sborata',
'sborolata',
'sboro',
'scopata',
'scopare',
'scroto',
'scrotum',
'sega',
'sesso',
'shit',
'shiz',
's.h.i.t.',
'sadomaso',
'sodomist',
'stronzata',
'stronzo',
'succhiamelo',
'succhiacazzi',
'testicol',
'troia',
'universetoday.net',
'vaffanculo',
'v****a',
'vibrator',
'vacca',
'yiddiot',
'zoccola',
]
elenco_others = [
'@',
".com",
".sex",
".org",
".uk",
".en",
".it",
"admin",
"administrator",
"amministratore",
'@yahoo.com',
'@alice.com',
'amministratrice',
'burocrate',
'checkuser',
'developer',
'http://',
'jimbo',
'mediawiki',
'on wheals',
'on wheal',
'on wheel',
'planante',
'razinger',
'sysop',
'troll',
'vandal',
' v.f. ',
'v. fighter',
'vandal f.',
'vandal fighter',
'wales jimmy',
'wheels',
'wales',
'www.',
]
# blacklist from wikipage
badword_page = pywikibot.Page(self.site,
i18n.translate(self.site, bad_pag))
list_loaded = []
if badword_page.exists():
pywikibot.output(u'\nLoading the bad words list from %s...' %
self.site)
list_loaded = load_word_function(badword_page.get())
else:
showStatus(4)
pywikibot.output(u'The bad word page doesn\'t exist!')
self._blacklist = elenco + elenco_others + list_loaded
del elenco, elenco_others, list_loaded
if not hasattr(self, '_whitelist') or force:
# initialize whitelist
whitelist_default = ['emiliano']
wtlpg = i18n.translate(self.site, whitelist_pg)
list_white = []
if wtlpg:
whitelist_page = pywikibot.Page(self.site, wtlpg)
if whitelist_page.exists():
pywikibot.output(u'\nLoading the whitelist from %s...' %
self.site)
list_white = load_word_function(whitelist_page.get())
else:
showStatus(4)
pywikibot.output(u"The whitelist's page doesn't exist!")
else:
showStatus(4)
pywikibot.warning(u"The whitelist hasn't been setted!")
# Join the whitelist words.
self._whitelist = list_white + whitelist_default
del list_white, whitelist_default
try:
for wname in self._whitelist:
if wname.lower() in str(name).lower():
name = name.lower().replace(wname.lower(), '')
for bname in self._blacklist:
self.bname[name] = bname
return bname.lower() in name.lower()
except UnicodeEncodeError:
pass
try:
for bname in self._blacklist:
if bname.lower() in str(name).lower(): # bad name positive
self.bname[name] = bname
return True
except UnicodeEncodeError:
pass
return False
from __future__ import absolute_import, division, unicode_literals
import pywikibot
# Requires PyMySql as first choice or
# MySQLdb <https://sourceforge.net/projects/mysql-python/>
try:
import pymysql as mysqldb
except ImportError:
try:
import MySQLdb as mysqldb # noqa: N813
except ImportError:
raise ImportError('No supported MySQL library installed. '
'Please install PyMySQL.')
else:
pywikibot.warning("PyMySQL not found. It'll fallback "
'on the deprecated library MySQLdb.')
else:
mysqldb.install_as_MySQLdb()
from pywikibot import config2 as config
from pywikibot.tools import deprecated_args, UnicodeType
@deprecated_args(encoding=None)
def mysql_query(query, params=None, dbname=None, verbose=None):
"""Yield rows from a MySQL query.
An example query that yields all ns0 pages might look like::
SELECT
page_namespace,
def delete_1_broken_redirect(self, redir_name):
"""Treat one broken redirect."""
if isinstance(redir_name, basestring):
redir_page = pywikibot.Page(self.site, redir_name)
else:
redir_page = redir_name
# Show the title of the page we're working on.
# Highlight the title in purple.
done = not self.getOption('delete')
pywikibot.output(
color_format('\n\n>>> {lightpurple}{0}{default} <<<',
redir_page.title()))
try:
targetPage = redir_page.getRedirectTarget()
except pywikibot.IsNotRedirectPage:
pywikibot.output(u'%s is not a redirect.' % redir_page.title())
except pywikibot.CircularRedirect:
pywikibot.output('%s is a circular redirect.' % redir_page.title())
except pywikibot.NoPage:
pywikibot.output(u'%s doesn\'t exist.' % redir_page.title())
except pywikibot.InvalidTitle:
pywikibot.exception()
except pywikibot.InterwikiRedirectPage:
pywikibot.output('%s is on another site.' % redir_page.title())
else:
try:
targetPage.get()
except pywikibot.BadTitle as e:
pywikibot.warning(
u'Redirect target %s is not a valid page title.' %
str(e)[10:])
except pywikibot.InvalidTitle:
pywikibot.exception()
except pywikibot.NoPage:
movedTarget = None
try:
movedTarget = targetPage.moved_target()
except pywikibot.NoMoveTarget:
pass
if movedTarget:
if not movedTarget.exists():
# FIXME: Test to another move
pywikibot.output(u'Target page %s does not exist' %
(movedTarget))
elif redir_page == movedTarget:
pywikibot.output(
'Redirect to target page forms a redirect loop')
else:
pywikibot.output(u'%s has been moved to %s' %
(redir_page, movedTarget))
reason = i18n.twtranslate(
self.site, 'redirect-fix-broken-moved',
{'to': movedTarget.title(asLink=True)})
content = redir_page.get(get_redirect=True)
redir_page.set_redirect_target(movedTarget,
keep_section=True,
save=False)
pywikibot.showDiff(content, redir_page.text)
pywikibot.output(u'Summary - %s' % reason)
if self.user_confirm(
u'Redirect target %s has been moved to %s.\n'
u'Do you want to fix %s?' %
(targetPage, movedTarget, redir_page)):
try:
redir_page.save(reason)
except pywikibot.NoUsername:
pywikibot.output(
u"Page [[%s]] not saved; "
u"sysop privileges required." %
redir_page.title())
except pywikibot.LockedPage:
pywikibot.output(u'%s is locked.' %
redir_page.title())
except pywikibot.PageSaveRelatedError:
pywikibot.exception()
else:
done = True
if not done and self.user_confirm(
u'Redirect target %s does not exist.\n'
u'Do you want to delete %s?' %
(targetPage.title(asLink=True),
redir_page.title(asLink=True))):
self.delete_redirect(redir_page, 'redirect-remove-broken')
elif not (self.getOption('delete') or movedTarget):
pywikibot.output(
'Cannot fix or delete the broken redirect')
except pywikibot.IsRedirectPage:
pywikibot.output(
"Redirect target {0} is also a redirect! {1}".format(
targetPage.title(asLink=True), "Won't delete anything."
if self.getOption('delete') else "Skipping."))
else:
# we successfully get the target page, meaning that
# it exists and is not a redirect: no reason to touch it.
pywikibot.output("Redirect target {0} does exist! {1}".format(
targetPage.title(asLink=True), "Won't delete anything."
if self.getOption('delete') else "Skipping."))
def fix_1_double_redirect(self, redir_name):
"""Treat one double redirect."""
if isinstance(redir_name, basestring):
redir = pywikibot.Page(self.site, redir_name)
else:
redir = redir_name
# Show the title of the page we're working on.
# Highlight the title in purple.
pywikibot.output(
color_format('\n\n>>> {lightpurple}{0}{default} <<<',
redir.title()))
newRedir = redir
redirList = [] # bookkeeping to detect loops
while True:
redirList.append(
u'%s:%s' %
(newRedir.site.lang, newRedir.title(withSection=False)))
try:
targetPage = newRedir.getRedirectTarget()
except pywikibot.IsNotRedirectPage:
if len(redirList) == 1:
pywikibot.output(u'Skipping: Page %s is not a redirect.' %
redir.title(asLink=True))
break # do nothing
elif len(redirList) == 2:
pywikibot.output(
u'Skipping: Redirect target %s is not a redirect.' %
newRedir.title(asLink=True))
break # do nothing
# else target found
except pywikibot.SectionError:
pywikibot.warning(
u"Redirect target section %s doesn't exist." %
newRedir.title(asLink=True))
except (pywikibot.CircularRedirect,
pywikibot.InterwikiRedirectPage,
pywikibot.UnsupportedPage) as e:
pywikibot.exception(e)
pywikibot.output(u"Skipping %s." % newRedir)
break
except pywikibot.BadTitle as e:
# str(e) is in the format 'BadTitle: [[Foo]]'
pywikibot.warning(
u'Redirect target %s is not a valid page title.' %
str(e)[10:])
break
except pywikibot.NoPage:
if len(redirList) == 1:
pywikibot.output(u'Skipping: Page %s does not exist.' %
redir.title(asLink=True))
break
else:
if self.getOption('always'):
pywikibot.output(
u"Skipping: Redirect target %s doesn't exist." %
newRedir.title(asLink=True))
break # skip if automatic
else:
pywikibot.warning(
u"Redirect target %s doesn't exist." %
newRedir.title(asLink=True))
except pywikibot.ServerError:
pywikibot.output(u'Skipping due to server error: '
u'No textarea found')
break
else:
pywikibot.output(u' Links to: %s.' %
targetPage.title(asLink=True))
try:
mw_msg = targetPage.site.mediawiki_message(
'wikieditor-toolbar-tool-redirect-example')
except KeyError:
pass
else:
if targetPage.title() == mw_msg:
pywikibot.output(
u"Skipping toolbar example: Redirect source is "
u"potentially vandalized.")
break
# watch out for redirect loops
if redirList.count(u'%s:%s' %
(targetPage.site.lang,
targetPage.title(withSection=False))):
pywikibot.warning(
u'Redirect target %s forms a redirect loop.' %
targetPage.title(asLink=True))
break # FIXME: doesn't work. edits twice!
if self.getOption('delete'):
# Delete the two redirects
# TODO: Check whether pages aren't vandalized
# and (maybe) do not have a version history
self.delete_redirect(targetPage,
'redirect-remove-loop')
self.delete_redirect(redir, 'redirect-remove-loop')
break
else: # redirect target found
if targetPage.isStaticRedirect():
pywikibot.output(
u" Redirect target is STATICREDIRECT.")
else:
newRedir = targetPage
continue
try:
oldText = redir.get(get_redirect=True)
except pywikibot.BadTitle:
pywikibot.output(u"Bad Title Error")
break
if self.is_repo and redir.namespace() == self.repo.item_namespace:
redir = pywikibot.ItemPage(self.repo, redir.title())
targetPage = pywikibot.ItemPage(self.repo, targetPage.title())
pywikibot.output('Fixing double item redirect')
redir.set_redirect_target(targetPage)
break
redir.set_redirect_target(targetPage,
keep_section=True,
save=False)
summary = i18n.twtranslate(self.site, 'redirect-fix-double',
{'to': targetPage.title(asLink=True)})
pywikibot.showDiff(oldText, redir.text)
if self.user_confirm(u'Do you want to accept the changes?'):
try:
redir.save(summary)
except pywikibot.LockedPage:
pywikibot.output(u'%s is locked.' % redir.title())
except pywikibot.SpamfilterError as error:
pywikibot.output(
u"Saving page [[%s]] prevented by spam filter: %s" %
(redir.title(), error.url))
except pywikibot.PageNotSaved as error:
pywikibot.output(u"Saving page [[%s]] failed: %s" %
(redir.title(), error))
except pywikibot.NoUsername:
pywikibot.output(
u"Page [[%s]] not saved; sysop privileges required." %
redir.title())
except pywikibot.Error as error:
pywikibot.output(
u"Unexpected error occurred trying to save [[%s]]: %s"
% (redir.title(), error))
break
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
for arg in pywikibot.handle_args(args):
arg, sep, val = arg.partition(':')
if arg == '-edit':
globalvar.attachEditCount = int(val or pywikibot.input(
'After how many edits would you like to welcome new users? '
'(0 is allowed)'))
elif arg == '-timeoffset':
globalvar.timeoffset = int(val or pywikibot.input(
'Which time offset (in minutes) for new users would you like '
'to use?'))
elif arg == '-time':
globalvar.timeRecur = int(val or pywikibot.input(
'For how many seconds would you like to bot to sleep before '
'checking again?'))
elif arg == '-offset':
if not val:
val = pywikibot.input(
'Which time offset for new users would you like to use? '
'(yyyymmddhhmmss)')
try:
globalvar.offset = pywikibot.Timestamp.fromtimestampformat(val)
except ValueError:
# upon request, we could check for software version here
raise ValueError(
"Mediawiki has changed, -offset:# is not supported "
"anymore, but -offset:TIMESTAMP is, assuming TIMESTAMP "
"is yyyymmddhhmmss. -timeoffset is now also supported. "
"Please read this script source header for documentation.")
elif arg == '-file':
globalvar.randomSign = True
globalvar.signFileName = val or pywikibot.input(
'Where have you saved your signatures?')
elif arg == '-sign':
globalvar.defaultSign = val or pywikibot.input(
'Which signature to use?')
globalvar.defaultSign += timeselected
elif arg == '-break':
globalvar.recursive = False
elif arg == '-nlog':
globalvar.makeWelcomeLog = False
elif arg == '-ask':
globalvar.confirm = True
elif arg == '-filter':
globalvar.filtBadName = True
elif arg == '-savedata':
globalvar.saveSignIndex = True
elif arg == '-random':
globalvar.randomSign = True
elif arg == '-sul':
globalvar.welcomeAuto = True
elif arg == '-limit':
globalvar.queryLimit = int(val or pywikibot.input(
u'How many of the latest new users would you like to load?'))
elif arg == '-numberlog':
globalvar.dumpToLog = int(val or pywikibot.input(
'After how many welcomed users would you like to update the '
'welcome log?'))
elif arg == '-quiet':
globalvar.quiet = True
elif arg == '-quick':
issue_deprecation_warning('The usage of "-quick" option',
None,
2,
since='20160211')
# Filename and Pywikibot path
# file where is stored the random signature index
filename = pywikibot.config.datafilepath(
'welcome-%s-%s.data' %
(pywikibot.Site().family.name, pywikibot.Site().code))
if globalvar.offset and globalvar.timeoffset:
pywikibot.warning(
'both -offset and -timeoffset were provided, ignoring -offset')
globalvar.offset = 0
try:
bot = WelcomeBot()
except KeyError as error:
# site not managed by welcome.py
pywikibot.bot.suggest_help(exception=error)
return False
try:
bot.run()
except KeyboardInterrupt:
if bot.welcomed_users:
showStatus()
pywikibot.output("Put welcomed users before quit...")
bot.makelogpage(bot.welcomed_users)
pywikibot.output("\nQuitting...")
finally:
# If there is the savedata, the script must save the number_user.
if globalvar.randomSign and globalvar.saveSignIndex and \
bot.welcomed_users:
if sys.version_info[0] > 2:
import pickle as cPickle
else:
import cPickle
with open(filename, 'wb') as f:
cPickle.dump(bot.welcomed_users,
f,
protocol=config.pickle_protocol)
def delete_1_broken_redirect(self):
"""Treat one broken redirect."""
redir_page = self.current_page
done = not self.getOption('delete')
try:
targetPage = redir_page.getRedirectTarget()
except (pywikibot.CircularRedirect,
pywikibot.InvalidTitle,
RuntimeError):
pywikibot.exception()
except pywikibot.InterwikiRedirectPage:
pywikibot.output('{0} is on another site.'
.format(redir_page.title()))
else:
try:
targetPage.get()
except pywikibot.BadTitle as e:
pywikibot.warning(
'Redirect target {0} is not a valid page title.'
.format(str(e)[10:]))
except pywikibot.InvalidTitle:
pywikibot.exception()
except pywikibot.NoPage:
movedTarget = None
try:
movedTarget = targetPage.moved_target()
except pywikibot.NoMoveTarget:
pass
if movedTarget:
if not movedTarget.exists():
# FIXME: Test to another move
pywikibot.output('Target page {0} does not exist'
.format(movedTarget))
elif redir_page == movedTarget:
pywikibot.output(
'Redirect to target page forms a redirect loop')
else:
pywikibot.output('{0} has been moved to {1}'
.format(redir_page, movedTarget))
reason = i18n.twtranslate(self.site,
'redirect-fix-broken-moved',
{'to': movedTarget.title(
as_link=True)})
content = redir_page.get(get_redirect=True)
redir_page.set_redirect_target(
movedTarget, keep_section=True, save=False)
pywikibot.output('Summary - ' + reason)
done = self.userPut(redir_page, content,
redir_page.text, summary=reason,
ignore_save_related_errors=True,
ignore_server_errors=True)
if not done and self.user_confirm(
'Redirect target {0} does not exist.\n'
'Do you want to delete {1}?'
.format(targetPage.title(as_link=True),
redir_page.title(as_link=True))):
self.delete_redirect(redir_page, 'redirect-remove-broken')
elif not (self.getOption('delete') or movedTarget):
pywikibot.output(
'Cannot fix or delete the broken redirect')
except pywikibot.IsRedirectPage:
pywikibot.output(
'Redirect target {0} is also a redirect! {1}'.format(
targetPage.title(as_link=True),
"Won't delete anything."
if self.getOption('delete') else 'Skipping.'))
else:
# we successfully get the target page, meaning that
# it exists and is not a redirect: no reason to touch it.
pywikibot.output(
'Redirect target {0} does exist! {1}'.format(
targetPage.title(as_link=True),
"Won't delete anything."
if self.getOption('delete') else 'Skipping.'))
