def __init__(self,
page: 'pywikibot.page.Page',
message: Optional[str] = None) -> None:
"""
Initializer.
:param page: Page that caused the exception
"""
if message:
self.message = message
if self.message is None:
raise Error("PageRelatedError is abstract. Can't instantiate it!")
self.page = page
self.title = page.title(as_link=True)
self.site = page.site
if re.search(r'\{\w+\}', self.message):
msg = self.message.format_map(self.__dict__)
elif re.search(r'%\(\w+\)s', self.message):
issue_deprecation_warning(
"'%' style messages are deprecated, "
'please use str.format() style instead',
since='20210504')
msg = self.message % self.__dict__
elif '%s' in self.message:
msg = self.message % page
else:
msg = self.message.format(page)
super().__init__(msg)
def _get_baserevid(self, claim, baserevid):
"""Check that claim.on_item is set and matches baserevid if used."""
if not claim.on_item:
issue_deprecation_warning('claim without on_item set',
depth=3,
warning_class=FutureWarning,
since='20160309')
if not baserevid:
warn('Neither claim.on_item nor baserevid provided',
UserWarning, 3)
return baserevid
if not baserevid:
return claim.on_item.latest_revision_id
issue_deprecation_warning('Site method with baserevid',
'claim with on_item set',
depth=3,
since='20150910')
if baserevid != claim.on_item.latest_revision_id:
warn(
'Using baserevid {} instead of claim baserevid {}'.format(
baserevid, claim.on_item.latest_revision_id), UserWarning,
3)
return baserevid
def _handle_dry_param(self, **kwargs):
"""
Read the dry parameter and set the simulate variable instead.
This is a private method. It prints a deprecation warning for old
-dry paramter and sets the global simulate variable and informs
the user about this setting.
The constuctor of the super class ignores it because it is not
part of self.availableOptions.
@note: You should ommit this method in your own application.
@keyword dry: deprecated option to prevent changes on live wiki.
Use -simulate instead.
@type dry: bool
"""
if 'dry' in kwargs:
issue_deprecation_warning('dry argument',
'pywikibot.config.simulate',
1,
warning_class=FutureWarning,
since='20160124')
# use simulate variable instead
pywikibot.config.simulate = True
pywikibot.output('config.simulate was set to True')
def replace_plural(match):
selector = match.group(1)
variants = match.group(2)
num = parameters[selector]
if not isinstance(num, int):
issue_deprecation_warning(
'type {0} for value {1} ({2})'.format(type(num), selector,
num), 'an int', 1)
num = int(num)
plural_entries = []
specific_entries = {}
for number, plural in re.findall(r'\|?(?: *(\d+) *= *)?([^|]+)',
variants):
if number:
specific_entries[int(number)] = plural
else:
assert not specific_entries, \
'generic entries defined after specific in "{0}"'.format(variants)
plural_entries += [plural]
if num in specific_entries:
return specific_entries[num]
index = plural_value(num)
if rule['nplurals'] == 1:
assert index == 0
if index >= len(plural_entries):
raise IndexError(
'requested plural {0} for {1} but only {2} ("{3}") '
'provided'.format(index, selector, len(plural_entries),
'", "'.join(plural_entries)))
return plural_entries[index]
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
options = {}
local_args = pywikibot.handle_args(args)
issue_deprecation_warning('featured.py script',
'Wikibase Client extension', 0, UserWarning)
for arg in local_args:
if arg.startswith('-fromlang:'):
options[arg[1:9]] = arg[10:].split(",")
elif arg.startswith('-after:'):
options['afterpage'] = arg[7:]
elif arg.startswith('-nocache:'):
options[arg[1:8]] = arg[9:].split(",")
else:
options[arg[1:].lower()] = True
bot = FeaturedBot(**options)
bot.run()
def _handle_dry_param(self, **kwargs):
if 'dry' in kwargs:
issue_deprecation_warning('dry argument',
'pywikibot.config.simulate', 1)
# use simulate variable instead
pywikibot.config.simulate = True
pywikibot.output('config.simulate was set to True')
def _clean_args(self, args: tuple, kwargs: dict):
"""Cleanup positional arguments.
Replace positional arguments with keyword arguments for
backwards compatibility.
@param: args: tuple of positional arguments
@param: kwargs: mutable dict of keyword arguments to be updated
"""
keys = ( # postional argument keys by old order
'revid', 'timestamp', 'user', 'anon', 'comment', 'text', 'minor',
'rollbacktoken', 'parentid', 'contentmodel', 'sha1', 'slots'
)
# replace positional arguments with keyword arguments
for i, (arg, key) in enumerate(zip(args, keys)): # pragma: no cover
issue_deprecation_warning('Positional argument {} ({})'
.format(i + 1, arg),
'keyword argument "{}={}"'
.format(key, arg),
warning_class=FutureWarning,
since='20200802')
if key in kwargs:
warning('"{}" is given as keyword argument "{}" already; '
'ignoring "{}"'.format(key, arg, kwargs[key]))
else:
kwargs[key] = arg
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: str
"""
options = {}
for arg in pywikibot.handle_args(args):
if arg == '-replacealways':
options['replace'] = True
options['replacealways'] = True
elif arg == '-hash': # T132303
raise NotImplementedError(
"The '-hash' argument is not implemented anymore.")
elif arg == '-autonomous':
issue_deprecation_warning('-autonomous',
'-always',
2,
ArgumentDeprecationWarning,
since='20140724')
options['always'] = True
elif arg.startswith('-'):
if arg[1:] in ('always', 'replace', 'replaceloose', 'replaceonly'):
options[arg[1:]] = True
bot = NowCommonsDeleteBot(**options)
bot.run()
def output(text, decoder=None, newline=True, toStdout=False, **kwargs):
r"""Output a message to the user via the userinterface.
Works like print, but uses the encoding used by the user's console
(console_encoding in the configuration file) instead of ASCII.
If decoder is None, text should be a unicode string. Otherwise it
should be encoded in the given encoding.
If newline is True, a line feed will be added after printing the text.
If toStdout is True, the text will be sent to standard output,
so that it can be piped to another process. All other text will
be sent to stderr. See: https://en.wikipedia.org/wiki/Pipeline_%28Unix%29
text can contain special sequences to create colored output. These
consist of the escape character \03 and the color name in curly braces,
e. g. \03{lightpurple}. \03{default} resets the color. By using the
color_format method from pywikibot.tools.formatter, the escape character
may be omitted.
Other keyword arguments are passed unchanged to the logger; so far, the
only argument that is useful is "exc_info=True", which causes the
log message to include an exception traceback.
"""
if toStdout: # maintained for backwards-compatibity only
from pywikibot.tools import issue_deprecation_warning
issue_deprecation_warning('"toStdout" parameter', 'pywikibot.stdout()',
2)
logoutput(text, decoder, newline, STDOUT, **kwargs)
else:
logoutput(text, decoder, newline, INFO, **kwargs)
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
options = {}
local_args = pywikibot.handle_args(args)
issue_deprecation_warning(
'featured.py script', 'Wikibase Client extension',
0, UserWarning)
for arg in local_args:
if arg.startswith('-fromlang:'):
options[arg[1:9]] = arg[10:].split(",")
elif arg.startswith('-after:'):
options['afterpage'] = arg[7:]
elif arg.startswith('-nocache:'):
options[arg[1:8]] = arg[9:].split(",")
else:
options[arg[1:].lower()] = True
bot = FeaturedBot(**options)
bot.run()
def replace_plural(match):
selector = match.group(1)
variants = match.group(2)
num = parameters[selector]
if not isinstance(num, int):
issue_deprecation_warning("type {0} for value {1} ({2})".format(type(num), selector, num), "an int", 1)
num = int(num)
plural_entries = []
specific_entries = {}
for number, plural in re.findall(r"\|?(?: *(\d+) *= *)?([^|]+)", variants):
if number:
specific_entries[int(number)] = plural
else:
assert not specific_entries, 'generic entries defined after specific in "{0}"'.format(variants)
plural_entries += [plural]
if num in specific_entries:
return specific_entries[num]
index = plural_value(num)
if rule["nplurals"] == 1:
assert index == 0
if index >= len(plural_entries):
raise IndexError(
'requested plural {0} for {1} but only {2} ("{3}") '
"provided".format(index, selector, len(plural_entries), '", "'.join(plural_entries))
)
return plural_entries[index]
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
# Process global arguments to determine desired site
local_args = pywikibot.handle_args(args)
# This factory is responsible for processing command line arguments
# that are also used by other scripts and that determine on which pages
# to work on.
genFactory = pagegenerators.GeneratorFactory()
# Parse command line arguments
for arg in local_args:
if arg == '-dry':
issue_deprecation_warning('-dry option', '-simulate', 1)
pywikibot.config.simulate = True
else:
genFactory.handleArg(arg)
gen = genFactory.getCombinedGenerator()
if gen:
# The preloading generator is responsible for downloading multiple
# pages from the wiki simultaneously.
gen = pagegenerators.PreloadingGenerator(gen)
bot = BasicBot(gen)
bot.run()
else:
pywikibot.showHelp()
def output(text, decoder=None, newline=True, toStdout=False, **kwargs):
r"""Output a message to the user via the userinterface.
Works like print, but uses the encoding used by the user's console
(console_encoding in the configuration file) instead of ASCII.
If decoder is None, text should be a unicode string. Otherwise it
should be encoded in the given encoding.
If newline is True, a line feed will be added after printing the text.
If toStdout is True, the text will be sent to standard output,
so that it can be piped to another process. All other text will
be sent to stderr. See: https://en.wikipedia.org/wiki/Pipeline_%28Unix%29
text can contain special sequences to create colored output. These
consist of the escape character \03 and the color name in curly braces,
e. g. \03{lightpurple}. \03{default} resets the color. By using the
color_format method from pywikibot.tools.formatter, the escape character
may be omitted.
Other keyword arguments are passed unchanged to the logger; so far, the
only argument that is useful is "exc_info=True", which causes the
log message to include an exception traceback.
"""
if toStdout: # maintained for backwards-compatibity only
from pywikibot.tools import issue_deprecation_warning # noqa
issue_deprecation_warning('"toStdout" parameter',
'pywikibot.stdout()', 2)
logoutput(text, decoder, newline, STDOUT, **kwargs)
else:
logoutput(text, decoder, newline, INFO, **kwargs)
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: str
"""
options = {}
gen = None
local_args = pywikibot.handle_args(args)
# This factory is responsible for processing command line arguments
# that are also used by other scripts and that determine on which pages
# to work on.
gen_factory = pagegenerators.GeneratorFactory(positional_arg_name='page')
for arg in local_args:
option, sep, value = arg.partition(':')
if option == '-xml':
filename = value or pywikibot.input(
"Please enter the XML dump's filename:")
gen = TableXmlDumpPageGenerator(filename)
elif option == '-auto':
issue_deprecation_warning(
'The usage of "-auto"', '-always',
1, ArgumentDeprecationWarning, since='20170205')
options['always'] = True
elif option in ['-always', '-quiet', '-skipwarning']:
options[option[1:]] = True
else:
if option in ('-sql', '-mysqlquery'):
if option == '-sql':
issue_deprecation_warning(
'The usage of "-sql"', '-mysqlquery',
1, ArgumentDeprecationWarning, since='20170205')
query = value or """
SELECT page_namespace, page_title
FROM page JOIN text ON (page_id = old_id)
WHERE old_text LIKE '%<table%'
"""
arg = '-mysqlquery:' + query
gen_factory.handleArg(arg)
if gen:
gen = pagegenerators.NamespaceFilterPageGenerator(
gen, gen_factory.namespaces)
else:
gen = gen_factory.getCombinedGenerator()
if gen:
if not gen_factory.nopreload:
gen = pagegenerators.PreloadingGenerator(gen)
bot = Table2WikiRobot(generator=gen, **options)
bot.run()
else:
suggest_help(missing_generator=True)
def checkstr(string: str) -> Tuple[str, str]:
"""
Return the key and duration extracted from the string.
@param string: a string defining a time period
Examples::
300s - 300 seconds
36h - 36 hours
7d - 7 days
2w - 2 weeks (14 days)
1y - 1 year
@return: key and duration extracted form the string
"""
if string.isdigit():
key = 's'
duration = string
issue_deprecation_warning('Time period without qualifier',
string + key,
1,
FutureWarning,
since='20161009')
else:
key = string[-1]
duration = string[:-1]
return key, duration
def checkstr(string):
"""
Return the key and duration extracted from the string.
@param string: a string defining a time period:
300s - 300 seconds
36h - 36 hours
7d - 7 days
2w - 2 weeks (14 days)
1y - 1 year
@type string: str
@return: key and duration extracted form the string
@rtype: (str, str)
"""
key = string[-1]
if string.isdigit():
key = 's'
duration = string
issue_deprecation_warning('Time period without qualifier',
string + key,
1,
UserWarning,
since='20161009')
else:
duration = string[:-1]
return key, duration
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
generator = None
local_args = pywikibot.handle_args(args)
site = pywikibot.Site()
if site.code != 'commons' or site.family.name != 'commons':
pywikibot.warning(
'This script is primarily written for Wikimedia '
'Commons, but has been invoked with site {0}. It '
'might work for other sites but there is no '
'guarantee that it does the right thing.'.format(site))
choice = pywikibot.input_choice(
'How do you want to continue?',
(('Continue using {0}'.format(site), 'c'),
('Switch to Wikimedia Commons', 's'), ('Quit', 'q')),
automatic_quit=False)
if choice == 's':
site = pywikibot.Site('commons', 'commons')
elif choice == 'q':
return False
genFactory = pagegenerators.GeneratorFactory(site)
for arg in local_args:
if arg.startswith('-yesterday'):
generator = uploadedYesterday(site)
issue_deprecation_warning(
'The usage of "-yesterday"',
'-logevents:"upload,,YYYYMMDD,YYYYMMDD"', 2,
ArgumentDeprecationWarning)
elif arg.startswith('-recentchanges'):
generator = recentChanges(site=site, delay=120)
else:
genFactory.handleArg(arg)
generator = genFactory.getCombinedGenerator(gen=generator)
if not generator:
pywikibot.bot.suggest_help(missing_generator=True)
return False
else:
pregenerator = pagegenerators.PreloadingGenerator(generator)
site.login()
for page in pregenerator:
pywikibot.output(page.title())
if page.exists() and (page.namespace() == 6) \
and (not page.isRedirectPage()):
if isUncat(page):
addUncat(page)
return True
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
generator = None
local_args = pywikibot.handle_args(args)
site = pywikibot.Site()
if site.code != 'commons' or site.family.name != 'commons':
pywikibot.warning('This script is primarily written for Wikimedia '
'Commons, but has been invoked with site {0}. It '
'might work for other sites but there is no '
'guarantee that it does the right thing.'.format(site))
choice = pywikibot.input_choice(
'How do you want to continue?',
(('Continue using {0}'.format(site), 'c'),
('Switch to Wikimedia Commons', 's'),
('Quit', 'q')),
automatic_quit=False)
if choice == 's':
site = pywikibot.Site('commons', 'commons')
elif choice == 'q':
return False
genFactory = pagegenerators.GeneratorFactory(site)
for arg in local_args:
if arg.startswith('-yesterday'):
generator = uploadedYesterday(site)
issue_deprecation_warning(
'The usage of "-yesterday"',
'-logevents:"upload,,YYYYMMDD,YYYYMMDD"',
2, ArgumentDeprecationWarning)
elif arg.startswith('-recentchanges'):
generator = recentChanges(site=site, delay=120)
else:
genFactory.handleArg(arg)
generator = genFactory.getCombinedGenerator(gen=generator)
if not generator:
pywikibot.bot.suggest_help(missing_generator=True)
return False
else:
pregenerator = pagegenerators.PreloadingGenerator(generator)
site.login()
for page in pregenerator:
pywikibot.output(page.title())
if page.exists() and (page.namespace() == 6) \
and (not page.isRedirectPage()):
if isUncat(page):
addUncat(page)
return True
def request(site=None,
uri=None,
method='GET',
params=None,
body=None,
headers=None,
data=None,
**kwargs):
"""
Request to Site with default error handling and response decoding.
See L{requests.Session.request} for additional parameters.
If the site argument is provided, the uri is a relative uri from
and including the document root '/'.
If the site argument is None, the uri must be absolute.
@param site: The Site to connect to
@type site: L{pywikibot.site.BaseSite}
@param uri: the URI to retrieve
@type uri: str
@keyword charset: Either a valid charset (usable for str.decode()) or None
to automatically chose the charset from the returned header (defaults
to latin-1)
@type charset: CodecInfo, str, None
@return: The received data
@rtype: a unicode string
"""
# body and data parameters both map to the data parameter of
# requests.Session.request.
if data:
body = data
assert (site or uri)
if not site:
# +1 because of @deprecate_arg
issue_deprecation_warning(
'Invoking http.request without argument site',
'http.fetch()',
3,
since='20150814')
r = fetch(uri, method, params, body, headers, **kwargs)
return r.text
kwargs.setdefault('disable_ssl_certificate_validation',
site.ignore_certificate_error())
if not headers:
headers = {}
format_string = None
else:
format_string = headers.get('user-agent')
headers['user-agent'] = user_agent(site, format_string)
baseuri = site.base_url(uri)
r = fetch(baseuri, method, params, body, headers, **kwargs)
site.throttle.retry_after = int(r.response_headers.get('retry-after', 0))
return r.text
def out(self) -> str:
"""Return the contents."""
if not hasattr(self._outputter, 'out'):
issue_deprecation_warning('{} without "out" property'
.format(self.__class__.__name__),
since='6.2.0')
self._outputter.output()
return ''
return self._outputter.out
def __call__(self, key='general', force=False, dump=False):
"""DEPRECATED: Return the entry for key or dump the complete cache."""
issue_deprecation_warning('Calling siteinfo',
'itself as a dictionary',
since='20161221',
warning_class=FutureWarning)
result = self.get(key, expiry=force)
if not dump:
return result
return self._cache
def __init__(self, title: str, _now=None, timestripper=None) -> None:
"""Initializer."""
if _now is not None:
issue_deprecation_warning(
'Argument "now" in DiscussionThread.__init__()',
warning_class=FutureWarning,
since='20200727')
assert timestripper is not None
self.title = title
self.ts = timestripper
self.code = self.ts.site.code
self.content = ''
self.timestamp = None
def request(site=None, uri=None, method='GET', params=None, body=None,
headers=None, data=None, **kwargs):
"""
Request to Site with default error handling and response decoding.
See L{requests.Session.request} for additional parameters.
If the site argument is provided, the uri is a relative uri from
and including the document root '/'.
If the site argument is None, the uri must be absolute.
@param site: The Site to connect to
@type site: L{pywikibot.site.BaseSite}
@param uri: the URI to retrieve
@type uri: str
@param charset: Either a valid charset (usable for str.decode()) or None
to automatically chose the charset from the returned header (defaults
to latin-1)
@type charset: CodecInfo, str, None
@return: The received data
@rtype: a unicode string
"""
# body and data parameters both map to the data parameter of
# requests.Session.request.
if data:
body = data
assert(site or uri)
if not site:
# +1 because of @deprecate_arg
issue_deprecation_warning(
'Invoking http.request without argument site', 'http.fetch()', 3)
r = fetch(uri, method, params, body, headers, **kwargs)
return r.content
baseuri = site.base_url(uri)
kwargs.setdefault("disable_ssl_certificate_validation",
site.ignore_certificate_error())
if not headers:
headers = {}
format_string = None
else:
format_string = headers.get('user-agent', None)
headers['user-agent'] = user_agent(site, format_string)
r = fetch(baseuri, method, params, body, headers, **kwargs)
return r.content
def __init__(self, generator, dry=False, **kwargs):
"""
Constructor.
@param generator: the page generator that determines on which pages
to work
@type generator: generator
@param dry: if True, doesn't do any real changes, but only shows
what would have been changed
@type dry: bool
"""
if dry:
issue_deprecation_warning('dry argument', 'pywikibot.config.simulate', 1)
pywikibot.config.simulate = True
super(BasicBot, self).__init__(site=True, **kwargs)
self.generator = generator
# Set the edit summary message
self.summary = i18n.twtranslate(self.site, 'basic-changing')
def __init__(self, generator, dry=False, **kwargs):
"""
Constructor.
@param generator: the page generator that determines on which pages
to work
@type generator: generator
@param dry: if True, doesn't do any real changes, but only shows
what would have been changed
@type dry: bool
"""
if dry:
issue_deprecation_warning('dry argument',
'pywikibot.config.simulate', 1)
pywikibot.config.simulate = True
super(BasicBot, self).__init__(site=True, **kwargs)
self.generator = generator
# Set the edit summary message
self.summary = i18n.twtranslate(self.site, 'basic-changing')
def request(site,
uri: Optional[str] = None,
headers: Optional[dict] = None,
**kwargs) -> requests.Response:
"""
Request to Site with default error handling and response decoding.
See L{requests.Session.request} for additional parameters.
The optional uri is a relative uri from site base uri including the
document root '/'.
@param site: The Site to connect to
@type: site: pywikibot.site.BaseSite
@param uri: the URI to retrieve
@keyword charset: Either a valid charset (usable for str.decode()) or None
to automatically chose the charset from the returned header (defaults
to latin-1)
@type charset: CodecInfo, str, None
@return: The received data Response
"""
kwargs.setdefault('verify', site.verify_SSL_certificate())
old_validation = kwargs.pop('disable_ssl_certificate_validation', None)
if old_validation is not None:
issue_deprecation_warning('disable_ssl_certificate_validation',
instead='verify',
warning_class=FutureWarning,
since='20201220')
kwargs.update(verify=not old_validation)
if not headers:
headers = {}
format_string = None
else:
format_string = headers.get('user-agent')
headers['user-agent'] = user_agent(site, format_string)
baseuri = site.base_url(uri)
r = fetch(baseuri, headers=headers, **kwargs)
site.throttle.retry_after = int(r.headers.get('retry-after', 0))
return r
def replace_plural(match):
selector = match.group(1)
variants = match.group(2)
num = parameters[selector]
if not isinstance(num, int):
issue_deprecation_warning('type {0} for value {1} ({2})'.format(
type(num), selector, num),
'an int',
1,
since='20151009')
num = int(num)
plural_entries = []
specific_entries = {}
# A plural entry can not start at the end of the variants list,
# and must end with | or the end of the variants list.
for number, plural in re.findall(
r'(?!$)(?: *(\d+) *= *)?(.*?)(?:\||$)', variants):
if number:
specific_entries[int(number)] = plural
else:
assert not specific_entries, (
'generic entries defined after specific in "{0}"'.format(
variants))
plural_entries += [plural]
if num in specific_entries:
return specific_entries[num]
index = plural_value(num)
needed = rule['nplurals']
if needed == 1:
assert index == 0
if index >= len(plural_entries):
raise IndexError(
'language "{}" requires {} plural variants for "{}" but '
'only {} ("{}") provided'.format(code, needed, selector,
len(plural_entries),
'", "'.join(plural_entries)))
return plural_entries[index]
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: unicode
"""
gen = None
options = {}
# Process global args and prepare generator args parser
local_args = pywikibot.handle_args(args)
gen_factory = pagegenerators.GeneratorFactory()
bot_class = TouchBot
for arg in local_args:
if gen_factory.handleArg(arg):
continue
if arg == '-purge':
bot_class = PurgeBot
elif arg == '-redir':
issue_deprecation_warning('\n-redir',
None,
1,
ArgumentDeprecationWarning,
since='20150514')
elif arg.startswith('-'):
options[arg[1:].lower()] = True
gen = gen_factory.getCombinedGenerator(preload=True)
if gen:
bot = bot_class(generator=gen, **options)
pywikibot.Site().login()
bot.run()
return True
else:
pywikibot.bot.suggest_help(missing_generator=True)
return False
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: str
"""
options = {}
total = None
local_args = pywikibot.handle_args(args)
for arg in local_args:
arg, sep, value = arg.partition(':')
if arg == '-limit':
total = value
elif arg == '-total':
total = value
issue_deprecation_warning('The usage of "{0}"'.format(arg),
'-limit',
2,
ArgumentDeprecationWarning,
since='20190120')
else:
options[arg[1:]] = True
site = pywikibot.Site()
gen = pagegenerators.UnusedFilesGenerator(total=total, site=site)
gen = pagegenerators.PreloadingGenerator(gen)
bot = UnusedFilesBot(site=site, generator=gen, **options)
try:
bot.run()
except pywikibot.Error as e:
pywikibot.bot.suggest_help(exception=e)
return False
else:
return True
def replace_plural(match):
selector = match.group(1)
variants = match.group(2)
num = parameters[selector]
if not isinstance(num, int):
issue_deprecation_warning(
'type {} for value {} ({})'
.format(type(num), selector, num),
'an int', 1,
warning_class=FutureWarning, since='20151009')
num = int(num)
plural_entries = []
specific_entries = {}
# A plural entry cannot start at the end of the variants list,
# and must end with | or the end of the variants list.
for number, plural in re.findall(
r'(?!$)(?: *(\d+) *= *)?(.*?)(?:\||$)', variants
):
if number:
specific_entries[int(number)] = plural
else:
assert not specific_entries, (
'generic entries defined after specific in "{}"'
.format(variants))
plural_entries += [plural]
if num in specific_entries:
return specific_entries[num]
index = plural_value(num)
needed = rule['nplurals']
if needed == 1:
assert index == 0
if index >= len(plural_entries):
# take the last entry in that case, see
# https://translatewiki.net/wiki/Plural#Plural_syntax_in_MediaWiki
index = -1
return plural_entries[index]
def replace_plural(match):
selector = match.group(1)
variants = match.group(2)
num = parameters[selector]
if not isinstance(num, int):
issue_deprecation_warning(
'type {0} for value {1} ({2})'
.format(type(num), selector, num),
'an int', 1, since='20151009')
num = int(num)
plural_entries = []
specific_entries = {}
# A plural entry can not start at the end of the variants list,
# and must end with | or the end of the variants list.
for number, plural in re.findall(
r'(?!$)(?: *(\d+) *= *)?(.*?)(?:\||$)', variants
):
if number:
specific_entries[int(number)] = plural
else:
assert not specific_entries, (
'generic entries defined after specific in "{0}"'
.format(variants))
plural_entries += [plural]
if num in specific_entries:
return specific_entries[num]
index = plural_value(num)
needed = rule['nplurals']
if needed == 1:
assert index == 0
if index >= len(plural_entries):
# take the last entry in that case, see
# https://translatewiki.net/wiki/Plural#Plural_syntax_in_MediaWiki
index = -1
return plural_entries[index]
def _handle_dry_param(self, **kwargs):
"""
Read the dry parameter and set the simulate variable instead.
This is a private method. It prints a deprecation warning for old
-dry paramter and sets the global simulate variable and informs
the user about this setting.
The constuctor of the super class ignores it because it is not
part of self.availableOptions.
@note: You should ommit this method in your own application.
@keyword dry: deprecated option to prevent changes on live wiki.
Use -simulate instead.
@type dry: bool
"""
if "dry" in kwargs:
issue_deprecation_warning("dry argument", "pywikibot.config.simulate", 1)
# use simulate variable instead
pywikibot.config.simulate = True
pywikibot.output("config.simulate was set to True")
def checkstr(string):
"""
Return the key and duration extracted from the string.
@param string: a string defining a time period:
300s - 300 seconds
36h - 36 hours
7d - 7 days
2w - 2 weeks (14 days)
1y - 1 year
@type string: str
@return: key and duration extracted form the string
@rtype: (str, str)
"""
key = string[-1]
if string.isdigit():
key = 's'
duration = string
issue_deprecation_warning('Time period without qualifier',
string + key, 1, UserWarning)
else:
duration = string[:-1]
return key, duration
def replace_plural(match):
selector = match.group(1)
variants = match.group(2)
num = parameters[selector]
if not isinstance(num, int):
issue_deprecation_warning(
'type {0} for value {1} ({2})'.format(type(num), selector, num),
'an int', 1)
num = int(num)
plural_entries = []
specific_entries = {}
# A plural entry can not start at the end of the variants list,
# and must end with | or the end of the variants list.
for number, plural in re.findall(r'(?!$)(?: *(\d+) *= *)?(.*?)(?:\||$)',
variants):
if number:
specific_entries[int(number)] = plural
else:
assert not specific_entries, \
'generic entries defined after specific in "{0}"'.format(variants)
plural_entries += [plural]
if num in specific_entries:
return specific_entries[num]
index = plural_value(num)
if rule['nplurals'] == 1:
assert index == 0
if index >= len(plural_entries):
raise IndexError(
'requested plural {0} for {1} but only {2} ("{3}") '
'provided'.format(
index, selector, len(plural_entries),
'", "'.join(plural_entries)))
return plural_entries[index]
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
# Process global arguments to determine desired site
local_args = pywikibot.handle_args(args)
# This factory is responsible for processing command line arguments
# that are also used by other scripts and that determine on which pages
# to work on.
genFactory = pagegenerators.GeneratorFactory()
# Parse command line arguments
for arg in local_args:
if arg == '-dry':
issue_deprecation_warning('-dry option', '-simulate', 1)
pywikibot.config.simulate = True
else:
genFactory.handleArg(arg)
gen = genFactory.getCombinedGenerator()
if gen:
# The preloading generator is responsible for downloading multiple
# pages from the wiki simultaneously.
gen = pagegenerators.PreloadingGenerator(gen)
bot = BasicBot(gen)
bot.run()
return True
else:
pywikibot.bot.suggest_help(missing_generator=True)
return False
def fetch(uri: str,
method: str = 'GET',
headers: Optional[dict] = None,
default_error_handling: bool = True,
use_fake_user_agent: Union[bool, str] = False,
**kwargs):
"""
HTTP request.
See :py:obj:`requests.Session.request` for parameters.
:param uri: URL to send
:param method: HTTP method of the request (default: GET)
:param headers: dictionary of headers of the request
:param default_error_handling: Use default error handling
:param use_fake_user_agent: Set to True to use fake UA, False to use
pywikibot's UA, str to specify own UA. This behaviour might be
overridden by domain in config.
:keyword charset: Either a valid charset (usable for str.decode()) or None
to automatically chose the charset from the returned header (defaults
to latin-1)
:type charset: CodecInfo, str, None
:keyword verify: verify the SSL certificate (default is True)
:type verify: bool or path to certificates
:keyword callbacks: Methods to call once data is fetched
:type callbacks: list of callable
:rtype: :py:obj:`requests.Response`
"""
# Change user agent depending on fake UA settings.
# Set header to new UA if needed.
headers = headers or {}
headers.update(config.extra_headers.copy() or {})
def assign_fake_user_agent(use_fake_user_agent, uri):
uri_domain = urlparse(uri).netloc
use_fake_user_agent = config.fake_user_agent_exceptions.get(
uri_domain, use_fake_user_agent)
if use_fake_user_agent is False:
return user_agent()
if use_fake_user_agent is True:
return fake_user_agent()
if use_fake_user_agent and isinstance(use_fake_user_agent, str):
return use_fake_user_agent # Custom UA.
raise ValueError('Invalid parameter: '
'use_fake_user_agent={}'.format(use_fake_user_agent))
def assign_user_agent(user_agent_format_string):
if not user_agent_format_string or '{' in user_agent_format_string:
return user_agent(None, user_agent_format_string)
# do nothing, it is already a UA
return user_agent_format_string
# If not already specified.
if 'user-agent' not in headers:
# Get fake UA exceptions from `fake_user_agent_exceptions` config.
headers['user-agent'] = assign_fake_user_agent(use_fake_user_agent,
uri)
# Already specified.
else:
headers['user-agent'] = assign_user_agent(headers.get('user-agent'))
callbacks = kwargs.pop('callbacks', [])
# error_handling_callback will be executed first.
if default_error_handling:
callbacks.insert(0, error_handling_callback)
charset = kwargs.pop('charset', None)
auth = get_authentication(uri)
if auth is not None and len(auth) == 4:
if isinstance(requests_oauthlib, ImportError):
warn(str(requests_oauthlib), ImportWarning)
error('OAuth authentication not supported: {}'.format(
requests_oauthlib))
auth = None
else:
auth = requests_oauthlib.OAuth1(*auth)
timeout = config.socket_timeout
old_validation = kwargs.pop('disable_ssl_certificate_validation', None)
if old_validation is not None:
issue_deprecation_warning('disable_ssl_certificate_validation',
instead='verify',
since='20201220')
kwargs.update(verify=not old_validation)
try:
# Note that the connections are pooled which mean that a future
# HTTPS request can succeed even if the certificate is invalid and
# verify=True, when a request with verify=False happened before
response = session.request(method,
uri,
headers=headers,
auth=auth,
timeout=timeout,
**kwargs)
except Exception as e:
response = e
else:
response.encoding = _decide_encoding(response, charset)
for callback in callbacks:
callback(response)
return response
def main(*args: Tuple[str, ...]) -> None:
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
"""
# If none, the var is set only for check purpose.
summary = None
addText = None
regexSkipUrl = None
always = False
textfile = None
talkPage = False
reorderEnabled = True
# Put the text above or below the text?
up = False
# Process global args and prepare generator args parser
local_args = pywikibot.handle_args(args)
genFactory = pagegenerators.GeneratorFactory()
# Loading the arguments
for arg in local_args:
option, sep, value = arg.partition(':')
if option == '-textfile':
textfile = value or pywikibot.input(
'Which textfile do you want to add?')
elif option == '-text':
addText = value or pywikibot.input('What text do you want to add?')
elif option == '-summary':
summary = value or pywikibot.input(
'What summary do you want to use?')
elif option == '-excepturl':
regexSkipUrl = value or pywikibot.input('What text should I skip?')
elif option == '-except':
new_arg = ''.join(['-grepnot', sep, value])
issue_deprecation_warning(arg,
new_arg,
2,
ArgumentDeprecationWarning,
since='20201224')
genFactory.handle_arg(new_arg)
elif option == '-up':
up = True
elif option == '-noreorder':
reorderEnabled = False
elif option == '-always':
always = True
elif option in ('-talk', '-talkpage'):
talkPage = True
else:
genFactory.handle_arg(arg)
if textfile and not addText:
with codecs.open(textfile, 'r', config.textfile_encoding) as f:
addText = f.read()
generator = genFactory.getCombinedGenerator()
additional_text = '' if addText else "The text to add wasn't given."
if pywikibot.bot.suggest_help(missing_generator=not generator,
additional_text=additional_text):
return
if talkPage:
generator = pagegenerators.PageWithTalkPageGenerator(generator, True)
for page in generator:
(_, newtext, _) = add_text(page,
addText,
summary,
regexSkipUrl=regexSkipUrl,
always=always,
up=up,
reorderEnabled=reorderEnabled,
create=talkPage)
else:
import _winreg as winreg
# Normalize old PYWIKIBOT2 environment variables and issue a deprecation warn.
for env_name in (
'PYWIKIBOT2_DIR', 'PYWIKIBOT2_DIR_PWB', 'PYWIKIBOT2_NO_USER_CONFIG',
):
if env_name not in environ:
continue
env_value = environ[env_name]
new_env_name = env_name.replace('PYWIKIBOT2_', 'PYWIKIBOT_')
del environ[env_name]
if new_env_name not in environ:
environ[new_env_name] = env_value
issue_deprecation_warning(
env_name + ' environment variable', new_env_name, 0, since='20180803')
# This frozen set should contain all imported modules/variables, so it must
# occur directly after the imports. At that point globals() only contains the
# names and some magic variables (like __name__)
_imports = frozenset(name for name in globals() if not name.startswith('_'))
__no_user_config = getenv('PYWIKIBOT_NO_USER_CONFIG')
if __no_user_config == '0':
__no_user_config = None
class _ConfigurationDeprecationWarning(UserWarning):
"""Feature that is no longer supported."""
if OSWIN32:
import winreg
# Normalize old PYWIKIBOT2 environment variables and issue a deprecation warn.
for env_name in (
'PYWIKIBOT2_DIR', 'PYWIKIBOT2_DIR_PWB', 'PYWIKIBOT2_NO_USER_CONFIG',
):
if env_name not in environ:
continue
env_value = environ[env_name]
new_env_name = env_name.replace('PYWIKIBOT2_', 'PYWIKIBOT_')
del environ[env_name]
if new_env_name not in environ:
environ[new_env_name] = env_value
issue_deprecation_warning(
env_name + ' environment variable', new_env_name, 0, since='20180803')
# This frozen set should contain all imported modules/variables, so it must
# occur directly after the imports. At that point globals() only contains the
# names and some magic variables (like __name__)
_imports = frozenset(name for name in globals() if not name.startswith('_'))
__no_user_config = getenv('PYWIKIBOT_NO_USER_CONFIG')
if __no_user_config == '0':
__no_user_config = None
class _ConfigurationDeprecationWarning(UserWarning):
"""Feature that is no longer supported."""
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: unicode
"""
oldName = None
options = {}
fromToPairs = []
# Process global args and prepare generator args parser
local_args = pywikibot.handle_args(args)
genFactory = pagegenerators.GeneratorFactory()
for arg in local_args:
if arg == '-pairs' or arg.startswith('-pairs:'):
issue_deprecation_warning(
'-pairs',
'-pairsfile',
2, ArgumentDeprecationWarning, since='20160304')
elif arg.startswith('-pairsfile'):
if len(arg) == len('-pairsfile'):
filename = pywikibot.input(
'Enter the name of the file containing pairs:')
else:
filename = arg[len('-pairsfile:'):]
oldName1 = None
for page in pagegenerators.TextfilePageGenerator(filename):
if oldName1:
fromToPairs.append([oldName1, page.title()])
oldName1 = None
else:
oldName1 = page.title()
if oldName1:
pywikibot.warning(
'file {0} contains odd number of links'.format(filename))
elif arg == '-noredirect':
options['noredirect'] = True
elif arg == '-notalkpage':
options['movetalkpage'] = False
elif arg == '-always':
options['always'] = True
elif arg == '-skipredirects':
options['skipredirects'] = True
elif arg.startswith('-from:'):
if oldName:
pywikibot.warning('-from:{0} without -to:'.format(oldName))
oldName = arg[len('-from:'):]
elif arg.startswith('-to:'):
if oldName:
fromToPairs.append([oldName, arg[len('-to:'):]])
oldName = None
else:
pywikibot.warning('{0} without -from'.format(arg))
elif arg.startswith('-prefix'):
if len(arg) == len('-prefix'):
options['prefix'] = pywikibot.input('Enter the prefix:')
else:
options['prefix'] = arg[8:]
elif arg.startswith('-summary'):
if len(arg) == len('-summary'):
options['summary'] = pywikibot.input('Enter the summary:')
else:
options['summary'] = arg[9:]
else:
genFactory.handleArg(arg)
if oldName:
pywikibot.warning('-from:{0} without -to:'.format(oldName))
site = pywikibot.Site()
for pair in fromToPairs:
page = pywikibot.Page(site, pair[0])
bot = MovePagesBot(None, **options)
bot.moveOne(page, pair[1])
gen = genFactory.getCombinedGenerator(preload=True)
if gen:
bot = MovePagesBot(gen, **options)
bot.run()
return True
if not fromToPairs:
pywikibot.bot.suggest_help(missing_generator=True)
return False
else:
return True
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
options = {}
gen_options = {}
# what the bot should do (either resolve double redirs, or process broken
# redirs)
action = None
namespaces = []
source = set()
for arg in pywikibot.handle_args(args):
arg, sep, value = arg.partition(':')
option = arg.partition('-')[2]
# bot options
if arg == 'do':
action = 'double'
elif arg == 'br':
action = 'broken'
elif arg in ('both', 'broken', 'double'):
action = arg
elif option in ('always', 'delete'):
options[option] = True
elif option == 'total':
options[option] = gen_options[option] = int(value)
elif option == 'sdtemplate':
options['sdtemplate'] = value or pywikibot.input(
'Which speedy deletion template to use?')
# generator options
elif option in ('fullscan', 'moves'):
gen_options[option] = True
source.add(arg)
elif option == 'xml':
gen_options[option] = value or i18n.input(
'pywikibot-enter-xml-filename')
source.add(arg)
elif option == 'namespace':
# "-namespace:" does NOT yield -namespace:0 further down the road!
ns = value or i18n.input('pywikibot-enter-namespace-number')
# TODO: at least for some generators enter a namespace by its name
# or number
if ns == '':
ns = '0'
try:
ns = int(ns)
except ValueError:
# -namespace:all Process all namespaces.
# Only works with the API read interface.
pass
if ns not in namespaces:
namespaces.append(ns)
elif option == 'offset':
gen_options[option] = int(value)
elif option in ('page', 'start', 'until'):
gen_options[option] = value
# deprecated or unknown options
elif option == 'step':
issue_deprecation_warning('The usage of "{0}"'.format(arg),
2, ArgumentDeprecationWarning)
else:
pywikibot.output(u'Unknown argument: %s' % arg)
gen_options['namespaces'] = namespaces
if len(source) > 1:
problem = 'You can only use one of {0} options.'.format(
' or '.join(source))
pywikibot.bot.suggest_help(additional_text=problem,
missing_action=not action)
return
if not action:
pywikibot.bot.suggest_help(missing_action=True)
else:
pywikibot.Site().login()
options['generator'] = RedirectGenerator(action, **gen_options)
bot = RedirectRobot(action, **options)
bot.run()
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
options = {}
# what the bot should do (either resolve double redirs, or delete broken
# redirs)
action = None
# where the bot should get his infos from (either None to load the
# maintenance special page from the live wiki, or the filename of a
# local XML dump file)
xmlFilename = None
# Which namespace should be processed when using a XML dump
# default to -1 which means all namespaces will be processed
namespaces = []
# at which redirect shall we start searching double redirects again
# (only with dump); default to -1 which means all redirects are checked
offset = -1
moved_pages = False
fullscan = False
start = ''
until = ''
number = None
pagename = None
for arg in pywikibot.handle_args(args):
arg, sep, value = arg.partition(':')
option = arg[1:]
# bot options
if arg == 'do':
action = 'double'
elif arg == 'br':
action = 'broken'
elif arg in ('both', 'broken', 'double'):
action = arg
elif option in ('always', 'delete'):
options[option] = True
elif option == 'total':
options['number'] = number = int(value)
# generator options
elif option == 'fullscan':
fullscan = True
elif option == 'xml':
xmlFilename = value or i18n.input('pywikibot-enter-xml-filename')
elif option == 'moves':
moved_pages = True
elif option == 'namespace':
# "-namespace:" does NOT yield -namespace:0 further down the road!
ns = value or i18n.input('pywikibot-enter-namespace-number')
# TODO: at least for some generators enter a namespace by its name
# or number
if ns == '':
ns = '0'
try:
ns = int(ns)
except ValueError:
# -namespace:all Process all namespaces.
# Only works with the API read interface.
pass
if ns not in namespaces:
namespaces.append(ns)
elif option == 'offset':
offset = int(value)
elif option == 'start':
start = value
elif option == 'until':
until = value
elif option == 'page':
pagename = value
# deprecated or unknown options
elif option == 'step':
issue_deprecation_warning('The usage of "{0}"'.format(arg),
2, ArgumentDeprecationWarning)
else:
pywikibot.output(u'Unknown argument: {0!s}'.format(arg))
if not action or xmlFilename and (moved_pages or fullscan):
problems = []
if xmlFilename and moved_pages:
problems += ['Either use a XML file or the moved pages from the API']
if xmlFilename and fullscan:
problems += ['Either use a XML file or do a full scan using the API']
pywikibot.bot.suggest_help(additional_text='\n'.join(problems),
missing_action=not action)
else:
pywikibot.Site().login()
gen = RedirectGenerator(xmlFilename, namespaces, offset, moved_pages,
fullscan, start, until, number, pagename)
bot = RedirectRobot(action, gen, **options)
bot.run()
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
options = {}
# what the bot should do (either resolve double redirs, or process broken
# redirs)
action = None
# where the bot should get his infos from (either None to load the
# maintenance special page from the live wiki, or the filename of a
# local XML dump file)
xmlFilename = None
# Which namespace should be processed when using a XML dump
# default to -1 which means all namespaces will be processed
namespaces = []
# at which redirect shall we start searching double redirects again
# (only with dump); default to -1 which means all redirects are checked
offset = -1
moved_pages = False
fullscan = False
start = ""
until = ""
number = None
pagename = None
for arg in pywikibot.handle_args(args):
arg, sep, value = arg.partition(":")
option = arg[1:]
# bot options
if arg == "do":
action = "double"
elif arg == "br":
action = "broken"
elif arg in ("both", "broken", "double"):
action = arg
elif option in ("always", "delete"):
options[option] = True
elif option == "total":
options["number"] = number = int(value)
# generator options
elif option == "fullscan":
fullscan = True
elif option == "xml":
xmlFilename = value or i18n.input("pywikibot-enter-xml-filename")
elif option == "moves":
moved_pages = True
elif option == "namespace":
# "-namespace:" does NOT yield -namespace:0 further down the road!
ns = value or i18n.input("pywikibot-enter-namespace-number")
# TODO: at least for some generators enter a namespace by its name
# or number
if ns == "":
ns = "0"
try:
ns = int(ns)
except ValueError:
# -namespace:all Process all namespaces.
# Only works with the API read interface.
pass
if ns not in namespaces:
namespaces.append(ns)
elif option == "offset":
offset = int(value)
elif option == "start":
start = value
elif option == "until":
until = value
elif option == "page":
pagename = value
# deprecated or unknown options
elif option == "step":
issue_deprecation_warning('The usage of "{0}"'.format(arg), 2, ArgumentDeprecationWarning)
else:
pywikibot.output("Unknown argument: %s" % arg)
if not action or xmlFilename and (moved_pages or fullscan):
problems = []
if xmlFilename and moved_pages:
problems += ["Either use a XML file or the moved pages from the API"]
if xmlFilename and fullscan:
problems += ["Either use a XML file or do a full scan using the API"]
pywikibot.bot.suggest_help(additional_text="\n".join(problems), missing_action=not action)
else:
pywikibot.Site().login()
gen = RedirectGenerator(xmlFilename, namespaces, offset, moved_pages, fullscan, start, until, number, pagename)
bot = RedirectRobot(action, gen, **options)
bot.run()
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
gen = None
oldName = None
options = {}
fromToPairs = []
# Process global args and prepare generator args parser
local_args = pywikibot.handle_args(args)
genFactory = pagegenerators.GeneratorFactory()
for arg in local_args:
if arg.startswith('-pairs'):
issue_deprecation_warning(
'-pairs',
'-pairsfile',
2, ArgumentDeprecationWarning)
elif arg.startswith('-pairsfile'):
if len(arg) == len('-pairsfile'):
filename = pywikibot.input(
u'Enter the name of the file containing pairs:')
else:
filename = arg[len('-pairsfile:'):]
oldName1 = None
for page in pagegenerators.TextfilePageGenerator(filename):
if oldName1:
fromToPairs.append([oldName1, page.title()])
oldName1 = None
else:
oldName1 = page.title()
if oldName1:
pywikibot.warning(
u'file %s contains odd number of links' % filename)
elif arg == '-noredirect':
options['noredirect'] = True
elif arg == '-notalkpage':
options['movetalkpage'] = False
elif arg == '-always':
options['always'] = True
elif arg == '-skipredirects':
options['skipredirects'] = True
elif arg.startswith('-from:'):
if oldName:
pywikibot.warning(u'-from:%s without -to:' % oldName)
oldName = arg[len('-from:'):]
elif arg.startswith('-to:'):
if oldName:
fromToPairs.append([oldName, arg[len('-to:'):]])
oldName = None
else:
pywikibot.warning(u'%s without -from' % arg)
elif arg.startswith('-prefix'):
if len(arg) == len('-prefix'):
options['prefix'] = pywikibot.input(u'Enter the prefix:')
else:
options['prefix'] = arg[8:]
elif arg.startswith('-summary'):
if len(arg) == len('-summary'):
options['summary'] = pywikibot.input(u'Enter the summary:')
else:
options['summary'] = arg[9:]
else:
genFactory.handleArg(arg)
if oldName:
pywikibot.warning(u'-from:%s without -to:' % oldName)
site = pywikibot.Site()
for pair in fromToPairs:
page = pywikibot.Page(site, pair[0])
bot = MovePagesBot(None, **options)
bot.moveOne(page, pair[1])
if not gen:
gen = genFactory.getCombinedGenerator()
if gen:
preloadingGen = pagegenerators.PreloadingGenerator(gen)
bot = MovePagesBot(preloadingGen, **options)
bot.run()
return True
if not fromToPairs:
pywikibot.bot.suggest_help(missing_generator=True)
return False
else:
return True
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
options = {}
gen = None
local_args = pywikibot.handle_args(args)
# This factory is responsible for processing command line arguments
# that are also used by other scripts and that determine on which pages
# to work on.
genFactory = pagegenerators.GeneratorFactory(positional_arg_name='page')
for arg in local_args:
option, sep, value = arg.partition(':')
if option == '-xml':
filename = value or pywikibot.input(
"Please enter the XML dump's filename:")
gen = TableXmlDumpPageGenerator(filename)
elif option == '-auto':
issue_deprecation_warning(
'The usage of "-auto"', '-always',
1, ArgumentDeprecationWarning)
options['always'] = True
elif option in ['-always', '-quiet', '-skipwarning']:
options[option[1:]] = True
else:
if option in ['-sql', '-mysqlquery']:
if not (has_module('oursql') or has_module('MySQLdb')):
raise NotImplementedError(
'Neither "oursql" nor "MySQLdb" library is installed.')
if option == '-sql':
issue_deprecation_warning(
'The usage of "-sql"', '-mysqlquery',
1, ArgumentDeprecationWarning)
query = value or """
SELECT page_namespace, page_title
FROM page JOIN text ON (page_id = old_id)
WHERE old_text LIKE '%<table%'
"""
arg = '-mysqlquery:' + query
genFactory.handleArg(arg)
if gen:
gen = pagegenerators.NamespaceFilterPageGenerator(
gen, genFactory.namespaces)
else:
gen = genFactory.getCombinedGenerator()
if gen:
if not genFactory.nopreload:
gen = pagegenerators.PreloadingGenerator(gen)
bot = Table2WikiRobot(generator=gen, **options)
bot.run()
return True
else:
suggest_help(missing_generator=True)
return False
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
add_cat = None
gen = None
# summary message
edit_summary = ""
# Array which will collect commandline parameters.
# First element is original text, second element is replacement text.
commandline_replacements = []
# A list of 2-tuples of original text and replacement text.
replacements = []
# Don't edit pages which contain certain texts.
exceptions = {
"title": [],
"text-contains": [],
"inside": [],
"inside-tags": [],
"require-title": [], # using a seperate requirements dict needs some
} # major refactoring of code.
# Should the elements of 'replacements' and 'exceptions' be interpreted
# as regular expressions?
regex = False
# Predefined fixes from dictionary 'fixes' (see above).
fixes_set = []
# the dump's path, either absolute or relative, which will be used
# if -xml flag is present
xmlFilename = None
useSql = False
# will become True when the user presses a ('yes to all') or uses the
# -always flag.
acceptall = False
# Will become True if the user inputs the commandline parameter -nocase
caseInsensitive = False
# Will become True if the user inputs the commandline parameter -dotall
dotall = False
# Will become True if the user inputs the commandline parameter -multiline
multiline = False
# Do all hits when they overlap
allowoverlap = False
# Do not recurse replacement
recursive = False
# Between a regex and another (using -fix) sleep some time (not to waste
# too much CPU
sleep = None
# Request manual replacements even if replacements are already defined
manual_input = False
# Replacements loaded from a file
replacement_file = None
replacement_file_arg_misplaced = False
# Read commandline parameters.
local_args = pywikibot.handle_args(args)
genFactory = pagegenerators.GeneratorFactory()
for arg in local_args:
if genFactory.handleArg(arg):
continue
if arg == "-regex":
regex = True
elif arg.startswith("-xmlstart"):
if len(arg) == 9:
xmlStart = pywikibot.input("Please enter the dumped article to start with:")
else:
xmlStart = arg[10:]
elif arg.startswith("-xml"):
if len(arg) == 4:
xmlFilename = i18n.input("pywikibot-enter-xml-filename")
else:
xmlFilename = arg[5:]
elif arg == "-sql":
useSql = True
elif arg.startswith("-excepttitle:"):
exceptions["title"].append(arg[13:])
elif arg.startswith("-requiretitle:"):
exceptions["require-title"].append(arg[14:])
elif arg.startswith("-excepttext:"):
exceptions["text-contains"].append(arg[12:])
elif arg.startswith("-exceptinside:"):
exceptions["inside"].append(arg[14:])
elif arg.startswith("-exceptinsidetag:"):
exceptions["inside-tags"].append(arg[17:])
elif arg.startswith("-fix:"):
fixes_set += [arg[5:]]
elif arg.startswith("-sleep:"):
sleep = float(arg[7:])
elif arg == "-always":
acceptall = True
elif arg == "-recursive":
recursive = True
elif arg == "-nocase":
caseInsensitive = True
elif arg == "-dotall":
dotall = True
elif arg == "-multiline":
multiline = True
elif arg.startswith("-addcat:"):
add_cat = arg[8:]
elif arg.startswith("-summary:"):
edit_summary = arg[9:]
elif arg.startswith("-automaticsummary"):
edit_summary = True
elif arg.startswith("-allowoverlap"):
allowoverlap = True
elif arg.startswith("-manualinput"):
manual_input = True
elif arg.startswith("-replacementfile"):
issue_deprecation_warning("-replacementfile", "-pairsfile", 2, ArgumentDeprecationWarning)
elif arg.startswith("-pairsfile"):
if len(commandline_replacements) % 2:
replacement_file_arg_misplaced = True
if arg == "-pairsfile":
replacement_file = pywikibot.input("Please enter the filename to read replacements from:")
else:
replacement_file = arg[len("-pairsfile:") :]
else:
commandline_replacements.append(arg)
site = pywikibot.Site()
if len(commandline_replacements) % 2:
pywikibot.error("Incomplete command line pattern replacement pair.")
return False
if replacement_file_arg_misplaced:
pywikibot.error("-pairsfile used between a pattern replacement pair.")
return False
if replacement_file:
try:
with codecs.open(replacement_file, "r", "utf-8") as f:
# strip newlines, but not other characters
file_replacements = f.read().splitlines()
except (IOError, OSError) as e:
pywikibot.error("Error loading {0}: {1}".format(replacement_file, e))
return False
if len(file_replacements) % 2:
pywikibot.error("{0} contains an incomplete pattern replacement pair.".format(replacement_file))
return False
# Strip BOM from first line
file_replacements[0].lstrip("\uFEFF")
commandline_replacements.extend(file_replacements)
if not (commandline_replacements or fixes_set) or manual_input:
old = pywikibot.input("Please enter the text that should be replaced:")
while old:
new = pywikibot.input("Please enter the new text:")
commandline_replacements += [old, new]
old = pywikibot.input("Please enter another text that should be replaced," "\nor press Enter to start:")
# The summary stored here won't be actually used but is only an example
single_summary = None
for i in range(0, len(commandline_replacements), 2):
replacement = Replacement(commandline_replacements[i], commandline_replacements[i + 1])
if not single_summary:
single_summary = i18n.twtranslate(
site, "replace-replacing", {"description": " (-%s +%s)" % (replacement.old, replacement.new)}
)
replacements.append(replacement)
# Perform one of the predefined actions.
missing_fixes_summaries = [] # which a fixes/replacements miss a summary
for fix_name in fixes_set:
try:
fix = fixes.fixes[fix_name]
except KeyError:
pywikibot.output("Available predefined fixes are: %s" % ", ".join(fixes.fixes.keys()))
if not fixes.user_fixes_loaded:
pywikibot.output("The user fixes file could not be found: " "{0}".format(fixes.filename))
return
if not fix["replacements"]:
pywikibot.warning("No replacements defined for fix " '"{0}"'.format(fix_name))
continue
if "msg" in fix:
if isinstance(fix["msg"], basestring):
set_summary = i18n.twtranslate(site, str(fix["msg"]))
else:
set_summary = i18n.translate(site, fix["msg"], fallback=True)
else:
set_summary = None
replacement_set = ReplacementList(
fix.get("regex"), fix.get("exceptions"), fix.get("nocase"), set_summary, name=fix_name
)
# Whether some replacements have a summary, if so only show which
# have none, otherwise just mention the complete fix
missing_fix_summaries = []
for index, replacement in enumerate(fix["replacements"], start=1):
summary = None if len(replacement) < 3 else replacement[2]
if not set_summary and not summary:
missing_fix_summaries.append('"{0}" (replacement #{1})'.format(fix_name, index))
if chars.contains_invisible(replacement[0]):
pywikibot.warning(
'The old string "{0}" contains formatting '
"characters like U+200E".format(chars.replace_invisible(replacement[0]))
)
if not callable(replacement[1]) and chars.contains_invisible(replacement[1]):
pywikibot.warning(
'The new string "{0}" contains formatting '
"characters like U+200E".format(chars.replace_invisible(replacement[1]))
)
replacement_set.append(
ReplacementListEntry(
old=replacement[0], new=replacement[1], fix_set=replacement_set, edit_summary=summary
)
)
if replacement_set:
replacements.extend(replacement_set)
if len(fix["replacements"]) == len(missing_fix_summaries):
missing_fixes_summaries.append('"{0}" (all replacements)'.format(fix_name))
else:
missing_fixes_summaries += missing_fix_summaries
if (not edit_summary or edit_summary is True) and (missing_fixes_summaries or single_summary):
if single_summary:
pywikibot.output(
"The summary message for the command line " "replacements will be something like: %s" % single_summary
)
if missing_fixes_summaries:
pywikibot.output(
"The summary will not be used when the fix has "
"one defined but the following fix(es) do(es) not "
"have a summary defined: "
"{0}".format(", ".join(missing_fixes_summaries))
)
if edit_summary is not True:
edit_summary = pywikibot.input(
"Press Enter to use this automatic message, or enter a "
"description of the\nchanges your bot will make:"
)
else:
edit_summary = ""
# Set the regular expression flags
flags = re.UNICODE
if caseInsensitive:
flags = flags | re.IGNORECASE
if dotall:
flags = flags | re.DOTALL
if multiline:
flags = flags | re.MULTILINE
# Pre-compile all regular expressions here to save time later
for replacement in replacements:
replacement.compile(regex, flags)
precompile_exceptions(exceptions, regex, flags)
if xmlFilename:
try:
xmlStart
except NameError:
xmlStart = None
gen = XmlDumpReplacePageGenerator(xmlFilename, xmlStart, replacements, exceptions, site)
elif useSql:
whereClause = "WHERE (%s)" % " OR ".join(
[
"old_text RLIKE '%s'" % prepareRegexForMySQL(old_regexp.pattern)
for (old_regexp, new_text) in replacements
]
)
if exceptions:
exceptClause = "AND NOT (%s)" % " OR ".join(
["old_text RLIKE '%s'" % prepareRegexForMySQL(exc.pattern) for exc in exceptions]
)
else:
exceptClause = ""
query = """
SELECT page_namespace, page_title
FROM page
JOIN text ON (page_id = old_id)
%s
%s
LIMIT 200""" % (
whereClause,
exceptClause,
)
gen = pagegenerators.MySQLPageGenerator(query)
gen = genFactory.getCombinedGenerator(gen)
if not gen:
pywikibot.bot.suggest_help(missing_generator=True)
return False
preloadingGen = pagegenerators.PreloadingGenerator(gen)
bot = ReplaceRobot(
preloadingGen,
replacements,
exceptions,
allowoverlap,
recursive,
add_cat,
sleep,
edit_summary,
always=acceptall,
site=site,
)
site.login()
bot.run()
# Explicitly call pywikibot.stopme().
# It will make sure the callback is triggered before replace.py is unloaded.
pywikibot.stopme()
pywikibot.output("\n%s pages changed." % bot.changed_pages)
def translate(code, xdict, parameters=None, fallback=False):
"""Return the most appropriate translation from a translation dict.
Given a language code and a dictionary, returns the dictionary's value for
key 'code' if this key exists; otherwise tries to return a value for an
alternative language that is most applicable to use on the wiki in
language 'code' except fallback is False.
The language itself is always checked first, then languages that
have been defined to be alternatives, and finally English. If none of
the options gives result, we just take the one language from xdict which may
not be always the same. When fallback is iterable it'll return None if no
code applies (instead of returning one).
For PLURAL support have a look at the twtranslate method.
@param code: The language code
@type code: string or Site object
@param xdict: dictionary with language codes as keys or extended dictionary
with family names as keys containing language dictionaries or
a single (unicode) string. May contain PLURAL tags as
described in twtranslate
@type xdict: dict, string, unicode
@param parameters: For passing (plural) parameters
@type parameters: dict, string, unicode, int
@param fallback: Try an alternate language code. If it's iterable it'll
also try those entries and choose the first match.
@type fallback: boolean or iterable
@raise IndexError: If the language supports and requires more plurals than
defined for the given translation template.
"""
family = pywikibot.config.family
# If a site is given instead of a code, use its language
if hasattr(code, 'code'):
family = code.family.name
code = code.code
# Check whether xdict has multiple projects
if isinstance(xdict, dict):
if family in xdict:
xdict = xdict[family]
elif 'wikipedia' in xdict:
xdict = xdict['wikipedia']
# Get the translated string
if not isinstance(xdict, dict):
trans = xdict
elif not xdict:
trans = None
else:
codes = [code]
if fallback is True:
codes += _altlang(code) + ['_default', 'en']
elif fallback is not False:
codes += list(fallback)
for code in codes:
if code in xdict:
trans = xdict[code]
break
else:
if fallback is not True:
# this shouldn't simply return "any one" code but when fallback
# was True before 65518573d2b0, it did just that. When False it
# did just return None. It's now also returning None in the new
# iterable mode.
return
code = list(xdict.keys())[0]
trans = xdict[code]
if trans is None:
return # return None if we have no translation found
if parameters is None:
return trans
if not isinstance(parameters, Mapping):
issue_deprecation_warning('parameters not being a mapping', None, 2)
plural_parameters = _PluralMappingAlias(parameters)
else:
plural_parameters = parameters
# else we check for PLURAL variants
trans = _extract_plural(code, trans, plural_parameters)
if parameters:
try:
return trans % parameters
except (KeyError, TypeError):
# parameter is for PLURAL variants only, don't change the string
pass
return trans
def twtranslate(source, twtitle, parameters=None, fallback=True,
only_plural=False):
"""
Translate a message using JSON files in messages_package_name.
fallback parameter must be True for i18n and False for L10N or testing
purposes.
Support for plural is implemented like in MediaWiki extension. If the
TranslateWiki message contains a plural tag inside which looks like::
{{PLURAL:<number>|<variant1>|<variant2>[|<variantn>]}}
it takes that variant calculated by the plural_rules depending on the number
value. Multiple plurals are allowed.
As an examples, if we had several json dictionaries in test folder like:
en.json::
{
"test-plural": "Bot: Changing %(num)s {{PLURAL:%(num)d|page|pages}}.",
}
fr.json::
{
"test-plural": "Robot: Changer %(descr)s {{PLURAL:num|une page|quelques pages}}.",
}
and so on.
>>> from pywikibot import i18n
>>> i18n.set_messages_package('tests.i18n')
>>> # use a dictionary
>>> str(i18n.twtranslate('en', 'test-plural', {'num':2}))
'Bot: Changing 2 pages.'
>>> # use additional format strings
>>> str(i18n.twtranslate('fr', 'test-plural', {'num': 1, 'descr': 'seulement'}))
'Robot: Changer seulement une page.'
>>> # use format strings also outside
>>> str(i18n.twtranslate('fr', 'test-plural', {'num': 10}, only_plural=True)
... % {'descr': 'seulement'})
'Robot: Changer seulement quelques pages.'
@param source: When it's a site it's using the lang attribute and otherwise
it is using the value directly.
@type source: BaseSite or str
@param twtitle: The TranslateWiki string title, in <package>-<key> format
@param parameters: For passing parameters. It should be a mapping but for
backwards compatibility can also be a list, tuple or a single value.
They are also used for plural entries in which case they must be a
Mapping and will cause a TypeError otherwise.
@param fallback: Try an alternate language code
@type fallback: boolean
@param only_plural: Define whether the parameters should be only applied to
plural instances. If this is False it will apply the parameters also
to the resulting string. If this is True the placeholders must be
manually applied afterwards.
@type only_plural: bool
@raise IndexError: If the language supports and requires more plurals than
defined for the given translation template.
"""
if not messages_available():
raise TranslationError(
'Unable to load messages package %s for bundle %s'
'\nIt can happen due to lack of i18n submodule or files. '
'Read %s/i18n'
% (_messages_package_name, twtitle, __url__))
source_needed = False
# If a site is given instead of a lang, use its language
if hasattr(source, 'lang'):
lang = source.lang
# check whether we need the language code back
elif isinstance(source, list):
# For backwards compatibility still support lists, when twntranslate
# was not deprecated and needed a way to get the used language code back
warn('The source argument should not be a list but either a BaseSite '
'or a str/unicode.', DeprecationWarning, 2)
lang = source.pop()
source_needed = True
else:
lang = source
# There are two possible failure modes: the translation dict might not have
# the language altogether, or a specific key could be untranslated. Both
# modes are caught with the KeyError.
langs = [lang]
if fallback:
langs += _altlang(lang) + ['en']
for alt in langs:
trans = _get_translation(alt, twtitle)
if trans:
break
else:
raise TranslationError(
'No %s translation has been defined for TranslateWiki key'
' %r\nIt can happen due to lack of i18n submodule or files. '
'Read https://mediawiki.org/wiki/PWB/i18n'
% ('English' if 'en' in langs else "'%s'" % lang,
twtitle))
# send the language code back via the given mutable list parameter
if source_needed:
source.append(alt)
if '{{PLURAL:' in trans:
# _extract_plural supports in theory non-mappings, but they are
# deprecated
if not isinstance(parameters, Mapping):
raise TypeError('parameters must be a mapping.')
trans = _extract_plural(alt, trans, parameters)
# this is only the case when called in twntranslate, and that didn't apply
# parameters when it wasn't a dict
if isinstance(parameters, _PluralMappingAlias):
# This is called due to the old twntranslate function which ignored
# KeyError. Instead only_plural should be used.
if isinstance(parameters.source, dict):
try:
trans %= parameters.source
except KeyError:
pass
parameters = None
if parameters is not None and not isinstance(parameters, Mapping):
issue_deprecation_warning('parameters not being a Mapping', None, 2)
if not only_plural and parameters:
return trans % parameters
else:
return trans
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
for arg in pywikibot.handle_args(args):
arg, sep, val = arg.partition(':')
if arg == '-edit':
globalvar.attachEditCount = int(val or pywikibot.input(
'After how many edits would you like to welcome new users? '
'(0 is allowed)'))
elif arg == '-timeoffset':
globalvar.timeoffset = int(val or pywikibot.input(
'Which time offset (in minutes) for new users would you like '
'to use?'))
elif arg == '-time':
globalvar.timeRecur = int(val or pywikibot.input(
'For how many seconds would you like to bot to sleep before '
'checking again?'))
elif arg == '-offset':
if not val:
val = pywikibot.input(
'Which time offset for new users would you like to use? '
'(yyyymmddhhmmss)')
try:
globalvar.offset = pywikibot.Timestamp.fromtimestampformat(val)
except ValueError:
# upon request, we might want to check for software version here
raise ValueError(
"Mediawiki has changed, -offset:# is not supported "
"anymore, but -offset:TIMESTAMP is, assuming TIMESTAMP "
"is yyyymmddhhmmss. -timeoffset is now also supported. "
"Please read this script source header for documentation.")
elif arg == '-file':
globalvar.randomSign = True
globalvar.signFileName = val or pywikibot.input(
'Where have you saved your signatures?')
elif arg == '-sign':
globalvar.defaultSign = val or pywikibot.input(
'Which signature to use?')
globalvar.defaultSign += timeselected
elif arg == '-break':
globalvar.recursive = False
elif arg == '-nlog':
globalvar.makeWelcomeLog = False
elif arg == '-ask':
globalvar.confirm = True
elif arg == '-filter':
globalvar.filtBadName = True
elif arg == '-savedata':
globalvar.saveSignIndex = True
elif arg == '-random':
globalvar.randomSign = True
elif arg == '-sul':
globalvar.welcomeAuto = True
elif arg == '-limit':
globalvar.queryLimit = int(val or pywikibot.input(
u'How many of the latest new users would you like to load?'))
elif arg == '-numberlog':
globalvar.dumpToLog = int(val or pywikibot.input(
'After how many welcomed users would you like to update the '
'welcome log?'))
elif arg == '-quiet':
globalvar.quiet = True
elif arg == '-quick':
issue_deprecation_warning('The usage of "-quick" option', None, 2)
# Filename and Pywikibot path
# file where is stored the random signature index
filename = pywikibot.config.datafilepath('welcome-%s-%s.data'
% (pywikibot.Site().family.name,
pywikibot.Site().code))
if globalvar.offset and globalvar.timeoffset:
pywikibot.warning(
'both -offset and -timeoffset were provided, ignoring -offset')
globalvar.offset = 0
try:
bot = WelcomeBot()
except KeyError as error:
# site not managed by welcome.py
pywikibot.bot.suggest_help(exception=error)
return False
try:
bot.run()
except KeyboardInterrupt:
if bot.welcomed_users:
showStatus()
pywikibot.output("Put welcomed users before quit...")
bot.makelogpage(bot.welcomed_users)
pywikibot.output("\nQuitting...")
finally:
# If there is the savedata, the script must save the number_user.
if globalvar.randomSign and globalvar.saveSignIndex and \
bot.welcomed_users:
if sys.version_info[0] > 2:
import pickle as cPickle
else:
import cPickle
with open(filename, 'wb') as f:
cPickle.dump(bot.welcomed_users, f, protocol=config.pickle_protocol)
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
index = None
pages = '1-'
options = {}
# Parse command line arguments.
local_args = pywikibot.handle_args(args)
for arg in local_args:
arg, sep, value = arg.partition(':')
if arg == '-index':
index = value
elif arg == '-pages':
pages = value
elif arg == '-showdiff':
issue_deprecation_warning('The usage of -showdiff option', None, 0)
elif arg == '-summary':
options['summary'] = value
elif arg == '-force':
issue_deprecation_warning('The usage of -force option', None, 0)
elif arg == '-always':
options['always'] = True
else:
pywikibot.output('Unknown argument %s' % arg)
# index is mandatory.
if not index:
pywikibot.bot.suggest_help(missing_parameters=['-index'])
return False
site = pywikibot.Site()
if not site.has_extension('ProofreadPage'):
pywikibot.error('Site %s must have ProofreadPage extension.' % site)
return False
index = IndexPage(site, index)
if not index.exists():
pywikibot.error("Page %s doesn't exist." % index)
return False
# Parse pages param.
# Create a list of (start, end) tuples.
pages = pages.split(',')
for interval in range(len(pages)):
start, sep, end = pages[interval].partition('-')
start = 1 if not start else int(start)
if not sep:
end = start
else:
end = int(end) if end else index.num_pages
pages[interval] = (start, end)
gen_list = []
for start, end in sorted(pages):
gen = index.page_gen(start=start, end=end,
filter_ql=[1], content=False)
gen_list.append(gen)
gen = itertools.chain(*gen_list)
pywikibot.output('\nUploading text to %s\n' % index.title(asLink=True))
bot = UploadTextBot(gen, site=index.site, **options)
bot.run()
