def useHashGenerator(self):
"""Use hash generator."""
# https://toolserver.org/~multichill/nowcommons.php?language=it&page=2&filter=
lang = self.site.lang
num_page = 0
word_to_skip_translated = i18n.translate(self.site, word_to_skip)
images_processed = list()
while 1:
url = ('https://toolserver.org/~multichill/nowcommons.php?'
'language=%s&page=%s&filter=') % (lang, num_page)
HTML_text = self.site.getUrl(url, no_hostname=True)
reg = r'<[Aa] href="(?P<urllocal>.*?)">(?P<imagelocal>.*?)</[Aa]> +?</td><td>\n\s*?'
reg += r'<[Aa] href="(?P<urlcommons>http[s]?://commons.wikimedia.org/.*?)" \
>Image:(?P<imagecommons>.*?)</[Aa]> +?</td><td>'
regex = re.compile(reg, re.UNICODE)
found_something = False
change_page = True
for x in regex.finditer(HTML_text):
found_something = True
image_local = x.group('imagelocal')
image_commons = x.group('imagecommons')
if image_local in images_processed:
continue
change_page = False
images_processed.append(image_local)
# Skip images that have something in the title (useful for it.wiki)
image_to_skip = False
for word in word_to_skip_translated:
if word.lower() in image_local.lower():
image_to_skip = True
if image_to_skip:
continue
url_local = x.group('urllocal')
url_commons = x.group('urlcommons')
pywikibot.output(color_format(
'\n\n>>> {lightpurple}{0}{default} <<<',
image_local))
pywikibot.output(u'Local: %s\nCommons: %s\n'
% (url_local, url_commons))
webbrowser.open(url_local, 0, 1)
webbrowser.open(url_commons, 0, 1)
if image_local.split('Image:')[1] == image_commons:
choice = pywikibot.input_yn(
u'The local and the commons images have the same name, '
'continue?', default=False, automatic_quit=False)
else:
choice = pywikibot.input_yn(
u'Are the two images equal?',
default=False, automatic_quit=False)
if choice:
yield [image_local, image_commons]
else:
continue
# The page is dinamically updated, so we may don't need to change it
if change_page:
num_page += 1
# If no image found means that there aren't anymore, break.
if not found_something:
break
def create_user_config():
"""Create a user-config.py in base_dir."""
_fnc = os.path.join(base_dir, "user-config.py")
if not file_exists(_fnc):
main_family, main_lang, main_username = get_site_and_lang()
usernames = [(main_family, main_lang, main_username)]
while pywikibot.input_yn("Do you want to add any other projects?",
default=False, automatic_quit=False):
usernames += [get_site_and_lang(main_family, main_lang,
main_username)]
usernames = '\n'.join(
u"usernames['{0}']['{1}'] = u'{2}'".format(*username)
for username in usernames)
extended = pywikibot.input_yn("Would you like the extended version of "
"user-config.py, with explanations "
"included?", automatic_quit=False)
if extended:
# config2.py will be in the pywikibot/ directory relative to this
# script (generate_user_files)
install = os.path.dirname(os.path.abspath(__file__))
with codecs.open(os.path.join(install, "pywikibot", "config2.py"),
"r", "utf-8") as config_f:
config = config_f.read()
res = re.findall("^(############## (?:"
"LOGFILE|"
"INTERWIKI|"
"SOLVE_DISAMBIGUATION|"
"IMAGE RELATED|"
"TABLE CONVERSION BOT|"
"WEBLINK CHECKER|"
"DATABASE|"
"SEARCH ENGINE|"
"COPYRIGHT|"
"FURTHER"
") SETTINGS .*?)^(?=#####|# =====)",
config, re.MULTILINE | re.DOTALL)
config_text = '\n'.join(res)
config_content = EXTENDED_CONFIG
else:
config_content = SMALL_CONFIG
with codecs.open(_fnc, "w", "utf-8") as f:
f.write(config_content.format(**locals()))
pywikibot.output(u"'%s' written." % _fnc)
def save(self, text, page, comment=None, minorEdit=True,
botflag=True):
""" Update the given page with new text. """
# only save if something was changed
if text != page.get():
# Show the title of the page we're working on.
# Highlight the title in purple.
pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
% page.title())
# show what was changed
pywikibot.showDiff(page.get(), text)
pywikibot.output(u'Comment: %s' % comment)
if not self.dry:
if pywikibot.input_yn(
u'Do you want to accept these changes?',
default=False, automatic_quit=False):
try:
page.text = text
# Save the page
page.save(comment=comment or self.comment,
minor=minorEdit, botflag=botflag)
except pywikibot.LockedPage:
pywikibot.output(u"Page %s is locked; skipping."
% page.title(asLink=True))
except pywikibot.EditConflict:
pywikibot.output(
u'Skipping %s because of edit conflict'
% (page.title()))
except pywikibot.SpamfilterError as error:
pywikibot.output(
u'Cannot change %s because of spam blacklist entry %s'
% (page.title(), error.url))
else:
return True
return False
def main(give_url, image_url, desc):
"""Run the bot."""
url = give_url
image_url = ''
if url == '':
if image_url:
url = pywikibot.input(u"What URL range should I check "
u"(use $ for the part that is changeable)")
else:
url = pywikibot.input(u"From what URL should I get the images?")
if image_url:
minimum = 1
maximum = 99
answer = pywikibot.input(
u"What is the first number to check (default: 1)")
if answer:
minimum = int(answer)
answer = pywikibot.input(
u"What is the last number to check (default: 99)")
if answer:
maximum = int(answer)
if not desc:
basicdesc = pywikibot.input(
u"What text should be added at the end of "
u"the description of each image from this url?")
else:
basicdesc = desc
if image_url:
ilinks = []
i = minimum
while i <= maximum:
ilinks += [url.replace("$", str(i))]
i += 1
else:
ilinks = get_imagelinks(url)
for image in ilinks:
if pywikibot.input_yn('Include image %s?' % image, default=False,
automatic_quit=False):
desc = pywikibot.input(u"Give the description of this image:")
categories = []
while True:
cat = pywikibot.input(u"Specify a category (or press enter to "
u"end adding categories)")
if not cat.strip():
break
if ":" in cat:
categories.append(u"[[%s]]" % cat)
else:
categories.append(u"[[%s:%s]]"
% (mysite.namespace(14), cat))
desc += "\r\n\r\n" + basicdesc + "\r\n\r\n" + \
"\r\n".join(categories)
uploadBot = UploadRobot(image, description=desc)
uploadBot.run()
elif answer == 's':
break
def upload_file(self, file_url, debug=False):
"""Upload the image at file_url to the target wiki.
Return the filename that was used to upload the image.
If the upload fails, ask the user whether to try again or not.
If the user chooses not to retry, return null.
"""
filename = self.process_filename(file_url)
if not filename:
return None
site = self.targetSite
imagepage = pywikibot.FilePage(site, filename) # normalizes filename
imagepage.text = self.description
pywikibot.output("Uploading file to %s via API..." % site)
try:
apiIgnoreWarnings = False
if self.ignoreWarning is True:
apiIgnoreWarnings = True
if self.uploadByUrl:
site.upload(imagepage, source_url=file_url, ignore_warnings=apiIgnoreWarnings)
else:
if "://" in file_url:
temp = self.read_file_content(file_url)
else:
temp = file_url
site.upload(
imagepage, source_filename=temp, ignore_warnings=apiIgnoreWarnings, chunk_size=self.chunk_size
)
except pywikibot.data.api.UploadWarning as warn:
pywikibot.output("We got a warning message: {0} - {1}".format(warn.code, warn.message))
if self.abort_on_warn(warn.code):
answer = False
elif self.ignore_on_warn(warn.code):
answer = True
else:
answer = pywikibot.input_yn("Do you want to ignore?", default=False, automatic_quit=False)
if answer:
self.ignoreWarning = True
self.keepFilename = True
return self.upload_file(file_url, debug)
else:
pywikibot.output("Upload aborted.")
return
except pywikibot.data.api.APIError as error:
if error.code == "uploaddisabled":
pywikibot.error("Upload error: Local file uploads are disabled on %s." % site)
else:
pywikibot.error("Upload error: ", exc_info=True)
except Exception:
pywikibot.error("Upload error: ", exc_info=True)
else:
# No warning, upload complete.
pywikibot.output("Upload of %s successful." % filename)
return filename # data['filename']
def get_dest(self):
self.dest = u'%s-core.%s' % tuple(self.source.rsplit(u'.', 1))
if not self.warnonly and not pywikibot.input_yn(
u'Destination file is %s.' % self.dest,
default=True, automatic_quit=False):
pywikibot.output('Quitting...')
exit()
def save(self, text, page, comment, minorEdit=True, botflag=True):
"""Save the text."""
if text != page.text:
# Show the title of the page we're working on.
# Highlight the title in purple.
pywikibot.output(color_format(
'\n\n>>> {lightpurple}{0}{default} <<<', page.title()))
# show what was changed
pywikibot.showDiff(page.get(), text)
pywikibot.output(u'Comment: %s' % comment)
if not self.dry:
if pywikibot.input_yn(
u'Do you want to accept these changes?',
default=False, automatic_quit=False):
page.text = text
try:
# Save the page
page.save(summary=comment, minorEdit=minorEdit,
botflag=botflag)
except pywikibot.LockedPage:
pywikibot.output(u"Page %s is locked; skipping."
% page.title(asLink=True))
except pywikibot.EditConflict:
pywikibot.output(
u'Skipping %s because of edit conflict'
% (page.title()))
except pywikibot.SpamfilterError as error:
pywikibot.output(
u'Cannot change %s because of spam blacklist entry '
u'%s' % (page.title(), error.url))
else:
return True
def treat(self, page):
"""It loads the given page, does some changes, and saves it."""
choice = False
try:
# page: title, date, username, comment, loginfo, rcid, token
username = page["user"]
# when the feed isnt from the API, it used to contain
# '(not yet written)' or '(page does not exist)' when it was
# a redlink
rcid = page["rcid"]
title = page["title"]
if not rcid:
raise Exception("rcid not present")
# check whether we have wrapped around to higher rcids
# which indicates a new RC feed is being processed
if rcid > self.last_rcid:
# refresh the whitelist
self.load_whitelist()
self.repeat_start_ts = time.time()
if pywikibot.config.verbose_output or self.getOption("ask"):
pywikibot.output("User %s has created or modified page %s" % (username, title))
if self.getOption("autopatroluserns") and (page["ns"] == 2 or page["ns"] == 3):
# simple rule to whitelist any user editing their own userspace
if title.partition(":")[2].split("/")[0].startswith(username):
if pywikibot.config.verbose_output:
pywikibot.output("%s is whitelisted to modify %s" % (username, title))
choice = True
if not choice and username in self.whitelist:
if self.in_list(self.whitelist[username], title):
if pywikibot.config.verbose_output:
pywikibot.output("%s is whitelisted to modify %s" % (username, title))
choice = True
if self.getOption("ask"):
choice = pywikibot.input_yn("Do you want to mark page as patrolled?", automatic_quit=False)
# Patrol the page
if choice:
# list() iterates over patrol() which returns a generator
list(self.site.patrol(rcid))
self.patrol_counter = self.patrol_counter + 1
pywikibot.output("Patrolled %s (rcid %d) by user %s" % (title, rcid, username))
else:
if pywikibot.config.verbose_output:
pywikibot.output("Skipped")
if rcid > self.highest_rcid:
self.highest_rcid = rcid
self.last_rcid = rcid
self.rc_item_counter = self.rc_item_counter + 1
except pywikibot.NoPage:
pywikibot.output("Page %s does not exist; skipping." % title(asLink=True))
except pywikibot.IsRedirectPage:
pywikibot.output("Page %s is a redirect; skipping." % title(asLink=True))
def main(*args):
"""
Process command line arguments and generate user-config.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
global base_dir
# set the config family and mylang values to an invalid state so that
# the script can detect that the command line arguments -family & -lang
# were used and and handle_args has updated these config values,
# and 'force' mode can be activated below.
(config.family, config.mylang) = ('wikipedia', None)
pywikibot.output('You can abort at any time by pressing ctrl-c')
local_args = pywikibot.handle_args(args)
if local_args:
pywikibot.output('Unknown arguments: %s' % ' '.join(local_args))
return False
if config.mylang is not None:
force = True
pywikibot.output(u'Automatically generating user-config.py')
else:
force = False
# Force default site of en.wikipedia
(config.family, config.mylang) = ('wikipedia', 'en')
username = config.usernames[config.family].get(config.mylang)
args = (config.family, config.mylang, username)
# Only give option for directory change if user-config.py already exists
# in the directory. This will repeat if user-config.py also exists in
# the requested directory.
if not force or config.verbose_output:
pywikibot.output(u'\nYour default user directory is "%s"' % base_dir)
while os.path.isfile(os.path.join(base_dir, "user-config.py")):
pywikibot.output('user-config.py already exists'
' in the target directory.')
if pywikibot.input_yn('Would you like to change the directory?',
default=True,
automatic_quit=False, force=force):
new_base = change_base_dir()
if new_base:
base_dir = new_base
else:
break
# user-fixes.py also used to be created here, but has
# been replaced by an example file.
if not os.path.isfile(os.path.join(base_dir, "user-config.py")):
create_user_config(args, force=force)
else:
pywikibot.output('user-config.py already exists in the target '
'directory "{0}".'.format(base_dir))
def get_dest(self):
"""Ask for destination script name."""
self.dest = u'{0!s}-core.{1!s}'.format(*tuple(self.source.rsplit(u'.', 1)))
if not self.warnonly and not pywikibot.input_yn(
u'Destination file is {0!s}.'.format(self.dest),
default=True, automatic_quit=False):
pywikibot.output('Quitting...')
exit()
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
options = {}
# Process global args and prepare generator args parser
local_args = pywikibot.handle_args(args)
genFactory = pagegenerators.GeneratorFactory()
for arg in local_args:
if arg.startswith('-summary:'):
options['summary'] = arg[len('-summary:'):]
elif arg == '-always':
options['always'] = True
elif arg == '-async':
options['async'] = True
elif arg.startswith('-ignore:'):
ignore_mode = arg[len('-ignore:'):].lower()
if ignore_mode == 'method':
options['ignore'] = cosmetic_changes.CANCEL_METHOD
elif ignore_mode == 'page':
options['ignore'] = cosmetic_changes.CANCEL_PAGE
elif ignore_mode == 'match':
options['ignore'] = cosmetic_changes.CANCEL_MATCH
else:
raise ValueError('Unknown ignore mode "{0}"!'.format(ignore_mode))
else:
genFactory.handleArg(arg)
site = pywikibot.Site()
if not options.get('summary'):
# Load default summary message.
options['summary'] = i18n.twtranslate(site,
'cosmetic_changes-standalone')
gen = genFactory.getCombinedGenerator()
if gen:
if options.get('always') or pywikibot.input_yn(
warning + '\nDo you really want to continue?',
default=False, automatic_quit=False):
site.login()
preloadingGen = pagegenerators.PreloadingGenerator(gen)
bot = CosmeticChangesBot(preloadingGen, **options)
bot.run()
return True
else:
pywikibot.bot.suggest_help(missing_generator=True)
return False
def treat_page(self):
"""Suggest redirects by reordering names in titles."""
if self.current_page.isRedirectPage():
return
page_t = self.current_page.title()
split_title = page_t.split(' (')
name = split_title[0]
site = self.current_page.site
possible_names = []
if self.getOption('surnames_last'):
name_parts = name.split(', ')
if len(name_parts) == 2 and len(name.split(' ')) <= 3:
possible_names.append(name_parts[1] + ' ' + name_parts[0])
else:
words = name.split()
if len(words) == 2 and name == name.title():
possible_names.append(words[1] + ', ' + words[0])
elif len(words) == 3:
# title may have at most one non-titlecased word
if len(SequenceMatcher(None, name,
name.title()).get_matching_blocks()) <= 3:
possible_names.append(words[1] + ' ' +
words[2] + ', ' +
words[0])
possible_names.append(words[2] + ', ' +
words[0] + ' ' +
words[1])
for possible_name in possible_names:
# append disambiguation inside parenthesis if there is one
if len(split_title) == 2:
possible_name += ' (' + split_title[1]
new_page = pywikibot.Page(site, possible_name)
if new_page.exists():
pywikibot.output('%s already exists, skipping...'
% new_page.title(asLink=True))
else:
pywikibot.output('%s doesn\'t exist'
% new_page.title(asLink=True))
choice = pywikibot.input_yn(
'Do you want to create a redirect?')
if choice:
comment = i18n.twtranslate(
site,
'capitalize_redirects-create-redirect',
{'to': page_t})
new_page.set_redirect_target(self.current_page,
create=True, summary=comment)
def run(self):
"""Start the robot's action."""
# regular expression to find the original template.
# {{vfd}} does the same thing as {{Vfd}}, so both will be found.
# The old syntax, {{msg:vfd}}, will also be found.
# The group 'parameters' will either match the parameters, or an
# empty string if there are none.
replacements = []
exceptions = {}
site = pywikibot.Site()
for old, new in self.templates.items():
namespaces = list(site.namespace(10, all=True))
if not site.nocapitalize:
pattern = '[' + \
re.escape(old[0].upper()) + \
re.escape(old[0].lower()) + \
']' + re.escape(old[1:])
else:
pattern = re.escape(old)
pattern = re.sub(r'_|\\ ', r'[_ ]', pattern)
templateRegex = re.compile(r'\{\{ *(' + ':|'.join(namespaces) +
r':|[mM][sS][gG]:)?' + pattern +
r'(?P<parameters>\s*\|.+?|) *}}',
re.DOTALL)
if self.getOption('subst') and self.getOption('remove'):
replacements.append((templateRegex,
'{{subst:%s\g<parameters>}}' % new))
exceptions['inside-tags'] = ['ref', 'gallery']
elif self.getOption('subst'):
replacements.append((templateRegex,
'{{subst:%s\g<parameters>}}' % old))
exceptions['inside-tags'] = ['ref', 'gallery']
elif self.getOption('remove'):
replacements.append((templateRegex, ''))
else:
template = pywikibot.Page(site, new, ns=10)
if not template.exists():
pywikibot.warning(u'Template "%s" does not exist.' % new)
if not pywikibot.input_yn('Do you want to proceed anyway?',
default=False, automatic_quit=False):
continue
replacements.append((templateRegex,
'{{%s\g<parameters>}}' % new))
replaceBot = replace.ReplaceRobot(self.generator, replacements,
exceptions, acceptall=self.getOption('always'),
addedCat=self.getOption('addedCat'),
summary=self.getOption('summary'))
replaceBot.run()
def run(self):
"""Start the robot's action."""
# regular expression to find the original template.
# {{vfd}} does the same thing as {{Vfd}}, so both will be found.
# The old syntax, {{msg:vfd}}, will also be found.
# The group 'parameters' will either match the parameters, or an
# empty string if there are none.
replacements = []
exceptions = {}
namespace = self.site.namespaces[10]
for old, new in self.templates.items():
if namespace.case == "first-letter":
pattern = "[" + re.escape(old[0].upper()) + re.escape(old[0].lower()) + "]" + re.escape(old[1:])
else:
pattern = re.escape(old)
pattern = re.sub(r"_|\\ ", r"[_ ]", pattern)
templateRegex = re.compile(
r"\{\{ *(" + ":|".join(namespace) + r":|[mM][sS][gG]:)?" + pattern + r"(?P<parameters>\s*\|.+?|) *}}",
re.DOTALL,
)
if self.getOption("subst") and self.getOption("remove"):
replacements.append((templateRegex, r"{{subst:%s\g<parameters>}}" % new))
exceptions["inside-tags"] = ["ref", "gallery"]
elif self.getOption("subst"):
replacements.append((templateRegex, r"{{subst:%s\g<parameters>}}" % old))
exceptions["inside-tags"] = ["ref", "gallery"]
elif self.getOption("remove"):
replacements.append((templateRegex, ""))
else:
template = pywikibot.Page(self.site, new, ns=10)
if not template.exists():
pywikibot.warning('Template "%s" does not exist.' % new)
if not pywikibot.input_yn("Do you want to proceed anyway?", default=False, automatic_quit=False):
continue
replacements.append((templateRegex, r"{{%s\g<parameters>}}" % new))
replaceBot = replace.ReplaceRobot(
self.generator,
replacements,
exceptions,
acceptall=self.getOption("always"),
addedCat=self.getOption("addedCat"),
summary=self.getOption("summary"),
)
replaceBot.run()
def treat(self, page):
"""Re-directing process.
Check if pages are in the given form Something, State, and
if so, create a redirect from Something, ST..
"""
for sn in self.abbrev:
R = re.compile(r', %s$' % sn)
if R.search(page.title()):
pl = pywikibot.Page(self.site, page.title().replace(sn,
self.abbrev[sn]))
# A bit hacking here - the real work is done in the
# 'except pywikibot.NoPage' part rather than the 'try'.
try:
pl.get(get_redirect=True)
goal = pl.getRedirectTarget().title()
if pywikibot.Page(self.site, goal).exists():
pywikibot.output(
u"Not creating %s - redirect already exists."
% goal)
else:
pywikibot.warning(
u"%s already exists but redirects elsewhere!"
% goal)
except pywikibot.IsNotRedirectPage:
pywikibot.warning(
u"Page %s already exists and is not a redirect "
u"Please check page!"
% pl.title())
except pywikibot.NoPage:
if page.isRedirectPage():
p2 = page.getRedirectTarget()
pywikibot.output(
u'Note: goal page is redirect.\nCreating redirect '
u'to "%s" to avoid double redirect.' % p2.title())
else:
p2 = page
if self.force or pywikibot.input_yn('Create redirect {0}?'
.format(pl.title())):
pl.set_redirect_target(
p2, create=True,
summary=i18n.twtranslate(
self.site, 'states_redirect-comment'))
def _handle_warnings(self, warnings):
messages = '\n'.join('{0.code}: {0.info}'.format(warning)
for warning in sorted(warnings,
key=lambda w: w.code))
if len(warnings) > 1:
messages = '\n' + messages
pywikibot.output('We got the following warning(s): ' + messages)
answer = True
for warning in warnings:
this_answer = self._handle_warning(warning.code)
if this_answer is False:
answer = False
break
elif this_answer is None:
answer = None
if answer is None:
answer = pywikibot.input_yn(u"Do you want to ignore?",
default=False, automatic_quit=False)
return answer
def _handle_warnings(self, warnings):
messages = '\n'.join(
'{0.code}: {0.info}'.format(warning)
for warning in sorted(warnings, key=lambda w: w.code))
if len(warnings) > 1:
messages = '\n' + messages
pywikibot.output('We got the following warning(s): ' + messages)
answer = True
for warning in warnings:
this_answer = self._handle_warning(warning.code)
if this_answer is False:
answer = False
break
if this_answer is None:
answer = None
if answer is None:
answer = pywikibot.input_yn('Do you want to ignore?',
default=False,
automatic_quit=False)
return answer
def change_base_dir():
"""Create a new user directory."""
while True:
new_base = pywikibot.input("New user directory? ")
new_base = os.path.abspath(new_base)
if os.path.exists(new_base):
if os.path.isfile(new_base):
pywikibot.error("there is an existing file with that name.")
continue
# make sure user can read and write this directory
if not os.access(new_base, os.R_OK | os.W_OK):
pywikibot.error("directory access restricted")
continue
pywikibot.output("Using existing directory")
break
else:
try:
os.mkdir(new_base, pywikibot.config2.private_files_permission)
except Exception:
pywikibot.error("ERROR: directory creation failed")
continue
pywikibot.output("Created new directory.")
break
if new_base == pywikibot.config2.get_base_dir(new_base):
# config would find that file
return new_base
msg = wrap(u"""WARNING: Your user files will be created in the directory
'%(new_base)s' you have chosen. To access these files, you will either have
to use the argument "-dir:%(new_base)s" every time you run the bot, or set
the environment variable "PYWIKIBOT2_DIR" equal to this directory name in
your operating system. See your operating system documentation for how to
set environment variables.""" % {'new_base': new_base},
width=76)
for line in msg:
pywikibot.output(line)
if pywikibot.input_yn('Is this OK?', default=False, automatic_quit=False):
return new_base
pywikibot.output("Aborting changes.")
return False
def change_base_dir():
"""Create a new user directory."""
while True:
new_base = pywikibot.input("New user directory? ")
new_base = os.path.abspath(new_base)
if os.path.exists(new_base):
if os.path.isfile(new_base):
pywikibot.error("there is an existing file with that name.")
continue
# make sure user can read and write this directory
if not os.access(new_base, os.R_OK | os.W_OK):
pywikibot.error("directory access restricted")
continue
pywikibot.output("Using existing directory")
break
else:
try:
os.mkdir(new_base, pywikibot.config2.private_files_permission)
except Exception:
pywikibot.error("ERROR: directory creation failed")
continue
pywikibot.output("Created new directory.")
break
if new_base == pywikibot.config2.get_base_dir(new_base):
# config would find that file
return new_base
msg = wrap(u"""WARNING: Your user files will be created in the directory
'%(new_base)s' you have chosen. To access these files, you will either have
to use the argument "-dir:%(new_base)s" every time you run the bot, or set
the environment variable "PYWIKIBOT2_DIR" equal to this directory name in
your operating system. See your operating system documentation for how to
set environment variables.""" % {'new_base': new_base}, width=76)
for line in msg:
pywikibot.output(line)
if pywikibot.input_yn('Is this OK?', default=False, automatic_quit=False):
return new_base
pywikibot.output("Aborting changes.")
return False
def run_bot(give_url, image_url, desc, shown):
"""Run the bot."""
if not give_url and image_url:
url = pywikibot.input('What URL range should I check '
'(use $ for the part that is changeable)')
minimum = int(
pywikibot.input('What is the first number to check (default: 1)')
or 1)
maximum = int(
pywikibot.input('What is the last number to check (default: 99)')
or 99)
ilinks = (url.replace('$', str(i))
for i in range(minimum, maximum + 1))
else:
url = (give_url
or pywikibot.input('From what URL should I get the images?'))
ilinks = get_imagelinks(url, shown)
basicdesc = desc or pywikibot.input(
'What text should be added at the end of '
'the description of each image from this url?')
mysite = pywikibot.Site()
for image in ilinks:
try:
include = pywikibot.input_yn('Include image {}?'.format(image),
default=False)
except QuitKeyboardInterrupt:
break
if not include:
continue
categories = get_categories(mysite)
desc = pywikibot.input('Give the description of this image:')
desc += '\n\n' + basicdesc + '\n\n' + '\n'.join(categories)
UploadRobot(image, description=desc).run()
def main(*args: str) -> None:
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
:param args: command line arguments
"""
options = {}
# Process global args and prepare generator args parser
gen_factory = pagegenerators.GeneratorFactory()
local_args = pywikibot.handle_args(args)
local_args = gen_factory.handle_args(local_args)
for arg in local_args:
opt, _, value = arg.partition(':')
if opt == '-summary':
options['summary'] = value
elif opt in ('-always', '-async'):
options[opt[1:]] = True
elif opt == '-ignore':
value = value.upper()
try:
options['ignore'] = getattr(CANCEL, value)
except AttributeError:
raise ValueError('Unknown ignore mode {!r}!'.format(value))
gen = gen_factory.getCombinedGenerator(preload=True)
if not pywikibot.bot.suggest_help(missing_generator=not gen) \
and (options.get('always')
or config.simulate
or pywikibot.input_yn(
warning + '\nDo you really want to continue?',
default=False, automatic_quit=False)):
bot = CosmeticChangesBot(generator=gen, **options)
bot.run()
def create_user_config(args=None, force=False):
"""Create a user-config.py in base_dir."""
_fnc = os.path.join(base_dir, "user-config.py")
if file_exists(_fnc):
return
if args and force and not config.verbose_output:
# main_username may be None, which is used in the next block
main_family, main_lang, main_username = args
usernames = [args]
else:
main_family, main_lang, main_username = get_site_and_lang(*args,
force=force)
usernames = [(main_family, main_lang, main_username)]
while pywikibot.input_yn("Do you want to add any other projects?",
force=force,
default=False, automatic_quit=False):
usernames += [get_site_and_lang(main_family, main_lang,
main_username)]
if not main_username:
usernames = "# usernames['{0}']['{1}'] = u'MyUsername'".format(
main_family, main_lang)
else:
usernames = '\n'.join(
u"usernames['{0}']['{1}'] = u'{2}'".format(*username)
for username in usernames)
config_text = ''
config_content = SMALL_CONFIG
if ((force and not config.verbose_output) or
pywikibot.input_yn('Would you like the extended version of '
'user-config.py, with explanations '
'included?', automatic_quit=False,
default=True, force=force)):
try:
# config2.py will be in the pywikibot/ directory relative to this
# script (generate_user_files)
install = os.path.dirname(os.path.abspath(__file__))
with codecs.open(os.path.join(install, "pywikibot", "config2.py"),
"r", "utf-8") as config_f:
config_file = config_f.read()
res = re.findall("^(# ############# (?:"
"LOGFILE|"
"INTERWIKI|"
"SOLVE_DISAMBIGUATION|"
"IMAGE RELATED|"
"TABLE CONVERSION BOT|"
"WEBLINK CHECKER|"
"DATABASE|"
"SEARCH ENGINE|"
"COPYRIGHT|"
"FURTHER"
") SETTINGS .*)^(?=#####|# =====)",
config_file, re.MULTILINE | re.DOTALL)
if not res:
warn('Extended config extraction failed', UserWarning)
config_text = '\n'.join(res)
if len(config_text.splitlines()) < 350:
warn('Extended config extraction too short: %d'
% len(config_text.splitlines()),
UserWarning)
config_content = EXTENDED_CONFIG
except Exception as e:
# If the warning was explicitly enabled, raise
if isinstance(e, UserWarning):
raise
pywikibot.output('Exception while creating extended user-config; '
'falling back to simple user-config.')
pywikibot.exception()
try:
with codecs.open(_fnc, "w", "utf-8") as f:
f.write(config_content.format(main_family=main_family,
main_lang=main_lang,
usernames=usernames,
config_text=config_text))
pywikibot.output(u"'%s' written." % _fnc)
except:
try:
os.remove(_fnc)
except:
pass
raise
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
gen = None
notitle = False
fmt = '1'
outputlang = None
page_get = False
base_dir = None
encoding = config.textfile_encoding
page_target = None
overwrite = False
summary = 'listpages-save-list'
# Process global args and prepare generator args parser
local_args = pywikibot.handle_args(args)
genFactory = GeneratorFactory()
for arg in local_args:
if arg == '-notitle':
notitle = True
elif arg.startswith('-format:'):
fmt = arg[len('-format:'):]
fmt = fmt.replace(u'\\03{{', u'\03{{')
elif arg.startswith('-outputlang:'):
outputlang = arg[len('-outputlang:'):]
elif arg == '-get':
page_get = True
elif arg.startswith('-save'):
base_dir = arg.partition(':')[2] or '.'
elif arg.startswith('-encode:'):
encoding = arg.partition(':')[2]
elif arg.startswith('-put:'):
page_target = arg.partition(':')[2]
elif arg.startswith('-overwrite'):
overwrite = True
elif arg.startswith('-summary:'):
summary = arg.partition(':')[2]
else:
genFactory.handleArg(arg)
if base_dir:
base_dir = os.path.expanduser(base_dir)
if not os.path.isabs(base_dir):
base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir))
if not os.path.exists(base_dir):
pywikibot.output(u'Directory "{0!s}" does not exist.'.format(base_dir))
choice = pywikibot.input_yn(
u'Do you want to create it ("No" to continue without saving)?')
if choice:
os.makedirs(base_dir, mode=0o744)
else:
base_dir = None
elif not os.path.isdir(base_dir):
# base_dir is a file.
pywikibot.warning(u'Not a directory: "%s"\n'
u'Skipping saving ...'
% base_dir)
base_dir = None
if page_target:
site = pywikibot.Site()
page_target = pywikibot.Page(site, page_target)
if not overwrite and page_target.exists():
pywikibot.bot.suggest_help(
additional_text='Page "{0}" already exists.'.format(
page_target.title()))
return False
if re.match('^[a-z_-]+$', summary):
summary = i18n.twtranslate(site, summary)
gen = genFactory.getCombinedGenerator()
if gen:
i = 0
output_list = []
for i, page in enumerate(gen, start=1):
if not notitle:
page_fmt = Formatter(page, outputlang)
output_list += [page_fmt.output(num=i, fmt=fmt)]
pywikibot.stdout(output_list[-1])
if page_get:
try:
pywikibot.stdout(page.text)
except pywikibot.Error as err:
pywikibot.output(err)
if base_dir:
filename = os.path.join(base_dir, page.title(as_filename=True))
pywikibot.output(u'Saving {0!s} to {1!s}'.format(page.title(), filename))
with open(filename, mode='wb') as f:
f.write(page.text.encode(encoding))
pywikibot.output(u"{0:d} page(s) found".format(i))
if page_target:
page_target.text = '\n'.join(output_list)
page_target.save(summary=summary)
return True
else:
pywikibot.bot.suggest_help(missing_generator=True)
return False
def create_user_config(args=None, force=False):
"""Create a user-config.py in base_dir."""
_fnc = os.path.join(base_dir, "user-config.py")
if file_exists(_fnc):
return
if args and force and not config.verbose_output:
# main_username may be None, which is used in the next block
main_family, main_lang, main_username = args
usernames = [args]
else:
main_family, main_lang, main_username = get_site_and_lang(*args,
force=force)
usernames = [(main_family, main_lang, main_username)]
while pywikibot.input_yn("Do you want to add any other projects?",
force=force,
default=False,
automatic_quit=False):
usernames += [
get_site_and_lang(main_family, main_lang, main_username)
]
if not main_username:
usernames = "# usernames['{0}']['{1}'] = u'MyUsername'".format(
main_family, main_lang)
else:
usernames = '\n'.join(
u"usernames['{0}']['{1}'] = u'{2}'".format(*username)
for username in usernames)
config_text = ''
config_content = SMALL_CONFIG
if ((force and not config.verbose_output) or pywikibot.input_yn(
'Would you like the extended version of '
'user-config.py, with explanations '
'included?',
automatic_quit=False,
default=True,
force=force)):
try:
# config2.py will be in the pywikibot/ directory relative to this
# script (generate_user_files)
install = os.path.dirname(os.path.abspath(__file__))
with codecs.open(os.path.join(install, "pywikibot", "config2.py"),
"r", "utf-8") as config_f:
config_file = config_f.read()
res = re.findall(
"^(# ############# (?:"
"LOGFILE|"
"INTERWIKI|"
"SOLVE_DISAMBIGUATION|"
"IMAGE RELATED|"
"TABLE CONVERSION BOT|"
"WEBLINK CHECKER|"
"DATABASE|"
"SEARCH ENGINE|"
"COPYRIGHT|"
"FURTHER"
") SETTINGS .*)^(?=#####|# =====)", config_file,
re.MULTILINE | re.DOTALL)
if not res:
warn('Extended config extraction failed', UserWarning)
config_text = '\n'.join(res)
if len(config_text.splitlines()) < 350:
warn(
'Extended config extraction too short: %d' %
len(config_text.splitlines()), UserWarning)
config_content = EXTENDED_CONFIG
except Exception as e:
# If the warning was explicitly enabled, raise
if isinstance(e, UserWarning):
raise
pywikibot.output('Exception while creating extended user-config; '
'falling back to simple user-config.')
pywikibot.exception()
try:
with codecs.open(_fnc, "w", "utf-8") as f:
f.write(
config_content.format(main_family=main_family,
main_lang=main_lang,
usernames=usernames,
config_text=config_text))
pywikibot.output(u"'%s' written." % _fnc)
except:
try:
os.remove(_fnc)
except:
pass
raise
def process_filename(self):
"""Return base filename portion of self.url."""
# Isolate the pure name
filename = self.url
# Filename may be either a local file path or a URL
if "://" in filename:
# extract the path portion of the URL
filename = urlparse(filename).path
filename = os.path.basename(filename)
if self.useFilename:
filename = self.useFilename
if not self.keepFilename:
pywikibot.output(
u"The filename on the target wiki will default to: %s"
% filename)
# FIXME: these 2 belong somewhere else, presumably in family
forbidden = '/' # to be extended
allowed_formats = (u'gif', u'jpg', u'jpeg', u'mid', u'midi',
u'ogg', u'png', u'svg', u'xcf', u'djvu',
u'ogv', u'oga', u'tif', u'tiff')
# ask until it's valid
while True:
newfn = pywikibot.input(
u'Enter a better name, or press enter to accept:')
if newfn == "":
newfn = filename
break
ext = os.path.splitext(newfn)[1].lower().strip('.')
# are any chars in forbidden also in newfn?
invalid = set(forbidden) & set(newfn)
if invalid:
c = "".join(invalid)
pywikibot.output(
'Invalid character(s): %s. Please try again' % c)
continue
if ext not in allowed_formats:
if not pywikibot.input_yn(
u"File format is not one of [%s], but %s. Continue?"
% (u' '.join(allowed_formats), ext),
default=False, automatic_quit=False):
continue
break
if newfn != '':
filename = newfn
# A proper description for the submission.
# Empty descriptions are not accepted.
pywikibot.output(u'The suggested description is:\n%s'
% self.description)
# Description must be set and verified
if not self.description:
self.verifyDescription = True
while not self.description or self.verifyDescription:
if not self.description:
pywikibot.output(
u'\03{lightred}It is not possible to upload a file '
'without a summary/description.\03{default}')
# if no description, default is 'yes'
if pywikibot.input_yn(
u'Do you want to change this description?',
default=not self.description):
from pywikibot import editor as editarticle
editor = editarticle.TextEditor()
try:
newDescription = editor.edit(self.description)
except Exception as e:
pywikibot.error(e)
continue
# if user saved / didn't press Cancel
if newDescription:
self.description = newDescription
self.verifyDescription = False
return filename
def treat_page(self):
"""Treat a single page."""
page = self.current_page
pagetitle = page.title(with_ns=False)
namesp = page.site.namespace(page.namespace())
if self.appendAll:
newPageTitle = '{}{}{}'.format(self.pagestart, pagetitle,
self.pageend)
if not self.noNamespace and namesp:
newPageTitle = '{}:{}'.format(namesp, newPageTitle)
elif self.regexAll:
newPageTitle = self.regex.sub(self.replacePattern, pagetitle)
if not self.noNamespace and namesp:
newPageTitle = '{}:{}'.format(namesp, newPageTitle)
if self.opt.prefix:
newPageTitle = '{}{}'.format(self.opt.prefix, pagetitle)
if self.opt.prefix or self.appendAll or self.regexAll:
if self.user_confirm(
'Change the page title to {!r}?'.format(newPageTitle)):
self.moveOne(page, newPageTitle)
return
# else:
choice = pywikibot.input_choice('What do you want to do?',
[('change page name', 'c'),
('append to page name', 'a'),
('use a regular expression', 'r'),
('next page', 'n')])
if choice == 'c':
newPageTitle = pywikibot.input('New page name:')
self.moveOne(page, newPageTitle)
elif choice == 'a':
self.pagestart = pywikibot.input('Append this to the start:')
self.pageend = pywikibot.input('Append this to the end:')
newPageTitle = ('{}{}{}'.format(self.pagestart, pagetitle,
self.pageend))
if namesp:
if pywikibot.input_yn('Do you want to remove the '
'namespace prefix "{}:"?'.format(namesp),
automatic_quit=False):
self.noNamespace = True
else:
newPageTitle = ('{}:{}'.format(namesp, newPageTitle))
choice2 = pywikibot.input_choice(
'Change the page title to {!r}?'.format(newPageTitle),
[('yes', 'y'), ('no', 'n'), ('all', 'a')])
if choice2 == 'y':
self.moveOne(page, newPageTitle)
elif choice2 == 'a':
self.appendAll = True
self.moveOne(page, newPageTitle)
elif choice == 'r':
searchPattern = pywikibot.input('Enter the search pattern:')
self.replacePattern = pywikibot.input('Enter the replace pattern:')
self.regex = re.compile(searchPattern)
if page.title() == page.title(with_ns=False):
newPageTitle = self.regex.sub(self.replacePattern,
page.title())
else:
if pywikibot.input_yn('Do you want to remove the '
'namespace prefix "{}:"?'.format(namesp),
automatic_quit=False):
newPageTitle = self.regex.sub(self.replacePattern,
page.title(with_ns=False))
self.noNamespace = True
else:
newPageTitle = self.regex.sub(self.replacePattern,
page.title())
choice2 = pywikibot.input_choice(
'Change the page title to {!r}?'.format(newPageTitle),
[('yes', 'y'), ('no', 'n'), ('all', 'a')])
if choice2 == 'y':
self.moveOne(page, newPageTitle)
elif choice2 == 'a':
self.regexAll = True
self.moveOne(page, newPageTitle)
def treat(self, page):
self.current_page = page
if self.getOption('skipredirects') and page.isRedirectPage():
pywikibot.output(u'Page %s is a redirect; skipping.' % page.title())
return
pagetitle = page.title(withNamespace=False)
namesp = page.site.namespace(page.namespace())
if self.appendAll:
newPageTitle = (u'%s%s%s'
% (self.pagestart, pagetitle, self.pageend))
if not self.noNamespace and namesp:
newPageTitle = (u'%s:%s' % (namesp, newPageTitle))
elif self.regexAll:
newPageTitle = self.regex.sub(self.replacePattern, pagetitle)
if not self.noNamespace and namesp:
newPageTitle = (u'%s:%s' % (namesp, newPageTitle))
if self.getOption('prefix'):
newPageTitle = (u'%s%s' % (self.getOption('prefix'), pagetitle))
if self.getOption('prefix') or self.appendAll or self.regexAll:
if not self.getOption('always'):
choice2 = pywikibot.input_choice(
u'Change the page title to "%s"?' % newPageTitle,
[('yes', 'y'), ('no', 'n'), ('all', 'a')])
if choice2 == 'y':
self.moveOne(page, newPageTitle)
elif choice2 == 'a':
self.options['always'] = True
self.moveOne(page, newPageTitle)
elif choice2 == 'n':
pass
else:
self.treat(page)
else:
self.moveOne(page, newPageTitle)
else:
choice = pywikibot.input_choice(u'What do you want to do?',
[('change page name', 'c'),
('append to page name', 'a'),
('use a regular expression', 'r'),
('next page', 'n')])
if choice == 'c':
newPageTitle = pywikibot.input(u'New page name:')
self.moveOne(page, newPageTitle)
elif choice == 'a':
self.pagestart = pywikibot.input(u'Append this to the start:')
self.pageend = pywikibot.input(u'Append this to the end:')
newPageTitle = (u'%s%s%s'
% (self.pagestart, pagetitle, self.pageend))
if namesp:
if pywikibot.input_yn(u'Do you want to remove the '
'namespace prefix "%s:"?' % namesp,
automatic_quit=False):
self.noNamespace = True
else:
newPageTitle = (u'%s:%s' % (namesp, newPageTitle))
choice2 = pywikibot.input_choice(
u'Change the page title to "%s"?'
% newPageTitle, [('yes', 'y'), ('no', 'n'), ('all', 'a')])
if choice2 == 'y':
self.moveOne(page, newPageTitle)
elif choice2 == 'a':
self.appendAll = True
self.moveOne(page, newPageTitle)
elif choice2 == 'n':
pass
else:
self.treat(page)
elif choice == 'r':
searchPattern = pywikibot.input(u'Enter the search pattern:')
self.replacePattern = pywikibot.input(
u'Enter the replace pattern:')
self.regex = re.compile(searchPattern)
if page.title() == page.title(withNamespace=False):
newPageTitle = self.regex.sub(self.replacePattern,
page.title())
else:
if pywikibot.input_yn(u'Do you want to remove the '
'namespace prefix "%s:"?' % namesp,
automatic_quit=False):
newPageTitle = self.regex.sub(
self.replacePattern, page.title(withNamespace=False))
self.noNamespace = True
else:
newPageTitle = self.regex.sub(self.replacePattern,
page.title())
choice2 = pywikibot.input_choice(
u'Change the page title to "%s"?'
% newPageTitle, [('yes', 'y'), ('no', 'n'), ('all', 'a')])
if choice2 == 'y':
self.moveOne(page, newPageTitle)
elif choice2 == 'a':
self.regexAll = True
self.moveOne(page, newPageTitle)
elif choice2 == 'n':
pass
else:
self.treat(page)
elif choice == 'n':
pass
else:
self.treat(page)
def __init__(self, generator, templates, **kwargs):
"""
Constructor.
@param generator: the pages to work on
@type generator: iterable
@param templates: a dictionary which maps old template names to
their replacements. If remove or subst is True, it maps the
names of the templates that should be removed/resolved to None.
@type templates: dict
"""
self.availableOptions.update({
'subst': False,
'remove': False,
'summary': None,
'addedCat': None,
})
Bot.__init__(self, generator=generator, **kwargs)
self.templates = templates
# get edit summary message if it's empty
if not self.getOption('summary'):
comma = self.site.mediawiki_message('comma-separator')
params = {
'list': comma.join(self.templates.keys()),
'num': len(self.templates)
}
site = self.site
if self.getOption('remove'):
self.options['summary'] = i18n.twtranslate(
site, 'template-removing', params)
elif self.getOption('subst'):
self.options['summary'] = i18n.twtranslate(
site, 'template-substituting', params)
else:
self.options['summary'] = i18n.twtranslate(
site, 'template-changing', params)
# regular expression to find the original template.
# {{vfd}} does the same thing as {{Vfd}}, so both will be found.
# The old syntax, {{msg:vfd}}, will also be found.
# The group 'parameters' will either match the parameters, or an
# empty string if there are none.
replacements = []
exceptions = {}
builder = textlib._MultiTemplateMatchBuilder(site)
for old, new in self.templates.items():
templateRegex = builder.pattern(old)
if self.getOption('subst') and self.getOption('remove'):
replacements.append(
(templateRegex, r'{{subst:%s\g<parameters>}}' % new))
exceptions['inside-tags'] = ['ref', 'gallery']
elif self.getOption('subst'):
replacements.append(
(templateRegex, r'{{subst:%s\g<parameters>}}' % old))
exceptions['inside-tags'] = ['ref', 'gallery']
elif self.getOption('remove'):
replacements.append((templateRegex, ''))
else:
template = pywikibot.Page(self.site, new, ns=10)
if not template.exists():
pywikibot.warning(u'Template "%s" does not exist.' % new)
if not pywikibot.input_yn('Do you want to proceed anyway?',
default=False,
automatic_quit=False):
continue
replacements.append(
(templateRegex, r'{{%s\g<parameters>}}' % new))
super(TemplateRobot,
self).__init__(generator,
replacements,
exceptions,
always=self.getOption('always'),
addedCat=self.getOption('addedCat'),
summary=self.getOption('summary'))
def run(self):
commons = pywikibot.Site('commons', 'commons')
comment = i18n.translate(self.site, nowCommonsMessage, fallback=True)
for page in self.getPageGenerator():
if self.getOption('use_hash'):
# Page -> Has the namespace | commons image -> Not
images_list = page # 0 -> local image, 1 -> commons image
page = pywikibot.Page(self.site, images_list[0])
else:
# If use_hash is true, we have already print this before, no need
self.current_page = page
try:
localImagePage = pywikibot.FilePage(self.site, page.title())
if localImagePage.fileIsShared():
pywikibot.output(u'File is already on Commons.')
continue
sha1 = localImagePage.latest_file_info.sha1
if self.getOption('use_hash'):
filenameOnCommons = images_list[1]
else:
filenameOnCommons = self.findFilenameOnCommons(
localImagePage)
if not filenameOnCommons and not self.getOption('use_hash'):
pywikibot.output(u'NowCommons template not found.')
continue
commonsImagePage = pywikibot.FilePage(
commons, 'Image:%s' % filenameOnCommons)
if localImagePage.title(withNamespace=False) == \
commonsImagePage.title(withNamespace=False) and self.getOption('use_hash'):
pywikibot.output(
u'The local and the commons images have the same name')
if localImagePage.title(withNamespace=False) != \
commonsImagePage.title(withNamespace=False):
usingPages = list(localImagePage.usingPages())
if usingPages and usingPages != [localImagePage]:
pywikibot.output(
u'\"\03{lightred}%s\03{default}\" is still used in %i pages.'
% (localImagePage.title(withNamespace=False),
len(usingPages)))
if self.getOption('replace') is True:
pywikibot.output(
u'Replacing \"\03{lightred}%s\03{default}\" by \
\"\03{lightgreen}%s\03{default}\".' %
(localImagePage.title(withNamespace=False),
commonsImagePage.title(withNamespace=False)))
oImageRobot = image.ImageRobot(
pg.FileLinksGenerator(localImagePage),
localImagePage.title(withNamespace=False),
commonsImagePage.title(withNamespace=False),
'', self.getOption('replacealways'),
self.getOption('replaceloose'))
oImageRobot.run()
# If the image is used with the urlname the
# previous function won't work
if len(list(pywikibot.FilePage(self.site,
page.title()).usingPages())) > 0 and \
self.getOption('replaceloose'):
oImageRobot = image.ImageRobot(
pg.FileLinksGenerator(localImagePage),
localImagePage.title(withNamespace=False,
asUrl=True),
commonsImagePage.title(
withNamespace=False), '',
self.getOption('replacealways'),
self.getOption('replaceloose'))
oImageRobot.run()
# refresh because we want the updated list
usingPages = len(
list(
pywikibot.FilePage(
self.site, page.title()).usingPages()))
if usingPages > 0 and self.getOption('use_hash'):
# just an enter
pywikibot.input(
u'There are still %s pages with this \
image, confirm the manual removal from them please.'
% usingPages)
else:
pywikibot.output(u'Please change them manually.')
continue
else:
pywikibot.output(
u'No page is using \"\03{lightgreen}%s\03{default}\" anymore.'
% localImagePage.title(withNamespace=False))
commonsText = commonsImagePage.get()
if self.getOption('replaceonly') is False:
if sha1 == commonsImagePage.latest_file_info.sha1:
pywikibot.output(
u'The image is identical to the one on Commons.')
if len(localImagePage.getFileVersionHistory()
) > 1 and not self.getOption('use_hash'):
pywikibot.output(
u"This image has a version history. Please \
delete it manually after making sure that the \
old versions are not worth keeping."
"")
continue
if self.getOption('always') is False:
pywikibot.output(
u'\n\n>>>> Description on \03{lightpurple}%s\03{default} <<<<\n'
% page.title())
pywikibot.output(localImagePage.get())
pywikibot.output(
u'\n\n>>>> Description on \03{lightpurple}%s\03{default} <<<<\n'
% commonsImagePage.title())
pywikibot.output(commonsText)
if pywikibot.input_yn(
u'Does the description on Commons contain '
'all required source and license\n'
'information?',
default=False,
automatic_quit=False):
localImagePage.delete(
'%s [[:commons:Image:%s]]' %
(comment, filenameOnCommons),
prompt=False)
else:
localImagePage.delete(
comment +
' [[:commons:Image:%s]]' % filenameOnCommons,
prompt=False)
else:
pywikibot.output(
u'The image is not identical to the one on Commons.'
)
except (pywikibot.NoPage, pywikibot.IsRedirectPage) as e:
pywikibot.output(u'%s' % e[0])
continue
def useHashGenerator(self):
# https://toolserver.org/~multichill/nowcommons.php?language=it&page=2&filter=
lang = self.site.lang
num_page = 0
word_to_skip_translated = i18n.translate(self.site, word_to_skip)
images_processed = list()
while 1:
url = ('https://toolserver.org/~multichill/nowcommons.php?'
'language=%s&page=%s&filter=') % (lang, num_page)
HTML_text = self.site.getUrl(url, no_hostname=True)
reg = r'<[Aa] href="(?P<urllocal>.*?)">(?P<imagelocal>.*?)</[Aa]> +?</td><td>\n\s*?'
reg += r'<[Aa] href="(?P<urlcommons>http[s]?://commons.wikimedia.org/.*?)" \
>Image:(?P<imagecommons>.*?)</[Aa]> +?</td><td>'
regex = re.compile(reg, re.UNICODE)
found_something = False
change_page = True
for x in regex.finditer(HTML_text):
found_something = True
image_local = x.group('imagelocal')
image_commons = x.group('imagecommons')
if image_local in images_processed:
continue
change_page = False
images_processed.append(image_local)
# Skip images that have something in the title (useful for it.wiki)
image_to_skip = False
for word in word_to_skip_translated:
if word.lower() in image_local.lower():
image_to_skip = True
if image_to_skip:
continue
url_local = x.group('urllocal')
url_commons = x.group('urlcommons')
pywikibot.output(
u"\n\n>>> \03{lightpurple}%s\03{default} <<<" %
image_local)
pywikibot.output(u'Local: %s\nCommons: %s\n' %
(url_local, url_commons))
webbrowser.open(url_local, 0, 1)
webbrowser.open(url_commons, 0, 1)
if image_local.split('Image:')[1] == image_commons:
choice = pywikibot.input_yn(
u'The local and the commons images have the same name, '
'continue?',
default=False,
automatic_quit=False)
else:
choice = pywikibot.input_yn(u'Are the two images equal?',
default=False,
automatic_quit=False)
if choice:
yield [image_local, image_commons]
else:
continue
# The page is dinamically updated, so we may don't need to change it
if change_page:
num_page += 1
# If no image found means that there aren't anymore, break.
if not found_something:
break
def __init__(self, generator, templates, **kwargs):
"""
Initializer.
@param generator: the pages to work on
@type generator: iterable
@param templates: a dictionary which maps old template names to
their replacements. If remove or subst is True, it maps the
names of the templates that should be removed/resolved to None.
@type templates: dict
"""
self.availableOptions.update({
'subst': False,
'remove': False,
'summary': None,
'addedCat': None,
})
Bot.__init__(self, generator=generator, **kwargs)
self.templates = templates
# get edit summary message if it's empty
if not self.getOption('summary'):
comma = self.site.mediawiki_message('comma-separator')
params = {
'list': comma.join(self.templates.keys()),
'num': len(self.templates)
}
if self.getOption('remove'):
self.options['summary'] = i18n.twtranslate(
self.site, 'template-removing', params)
elif self.getOption('subst'):
self.options['summary'] = i18n.twtranslate(
self.site, 'template-substituting', params)
else:
self.options['summary'] = i18n.twtranslate(
self.site, 'template-changing', params)
replacements = []
exceptions = {}
builder = textlib._MultiTemplateMatchBuilder(self.site)
for old, new in self.templates.items():
template_regex = builder.pattern(old)
if self.getOption('subst') and self.getOption('remove'):
replacements.append(
(template_regex, r'{{subst:%s\g<parameters>}}' % new))
exceptions['inside-tags'] = [
'ref',
'gallery',
'poem',
'pagelist',
]
elif self.getOption('subst'):
replacements.append(
(template_regex, r'{{subst:%s\g<parameters>}}' % old))
exceptions['inside-tags'] = [
'ref',
'gallery',
'poem',
'pagelist',
]
elif self.getOption('remove'):
separate_line_regex = re.compile(
r'^[*#:]* *{0} *\n'.format(template_regex.pattern),
re.DOTALL | re.MULTILINE)
replacements.append((separate_line_regex, ''))
spaced_regex = re.compile(
r' +{0} +'.format(template_regex.pattern), re.DOTALL)
replacements.append((spaced_regex, ' '))
replacements.append((template_regex, ''))
else:
template = pywikibot.Page(self.site, new, ns=10)
if not template.exists():
pywikibot.warning(u'Template "%s" does not exist.' % new)
if not pywikibot.input_yn('Do you want to proceed anyway?',
default=False,
automatic_quit=False):
continue
replacements.append(
(template_regex, r'{{%s\g<parameters>}}' % new))
super(TemplateRobot,
self).__init__(generator,
replacements,
exceptions,
always=self.getOption('always'),
addedCat=self.getOption('addedCat'),
summary=self.getOption('summary'))
def create_user_config(args=None, force=False):
"""
Create a user-config.py in base_dir.
Create a user-password.py if necessary.
"""
_fnc = os.path.join(base_dir, "user-config.py")
_fncpass = os.path.join(base_dir, 'user-password.py')
if file_exists(_fnc):
return
if args and force and not config.verbose_output:
# main_username may be None, which is used in the next block
main_family, main_code, main_username = args
usernames = [args]
else:
main_family, main_code, main_username = get_site_and_lang(*args,
force=force)
usernames = [(main_family, main_code, main_username)]
while pywikibot.input_yn("Do you want to add any other projects?",
force=force,
default=False, automatic_quit=False):
usernames += [get_site_and_lang(main_family, main_code,
main_username)]
botpasswords = []
if not main_username:
usernames = "# usernames['{0}']['{1}'] = u'MyUsername'".format(
main_family, main_code)
elif not file_exists(_fncpass):
# For each different username entered, ask if user wants to save a
# BotPassword (username, BotPassword name, BotPassword pass)
seen = set()
for username in usernames:
if username[2] in seen:
continue
seen.add(username[2])
if pywikibot.input_yn('Do you want to add a BotPassword for {0}?'
.format(username[2]),
force=force, default=False):
if not botpasswords:
pywikibot.output(
'See https://www.mediawiki.org/wiki/'
'Manual:Pywikibot/BotPasswords to know '
'how to get codes.')
pywikibot.output('Please note that plain text in {0} and '
'anyone with read access to that '
'directory will be able read the file.'
.format(_fncpass))
message = 'BotPassword\'s "bot name" for {0}'.format(
username[2])
botpasswordname = pywikibot.input(message, force=force)
message = 'BotPassword\'s "password" for BotPassword "{0}" ' \
'(no characters will be shown)' \
.format(botpasswordname)
botpasswordpass = pywikibot.input(message, force=force,
password=True)
if botpasswordname and botpasswordpass:
botpasswords.append((username[2], botpasswordname,
botpasswordpass))
usernames = '\n'.join(
u"usernames['{0}']['{1}'] = u'{2}'".format(*username)
for username in usernames)
botpasswords = '\n'.join(
"('{0}', BotPassword('{1}', '{2}'))".format(*botpassword)
for botpassword in botpasswords)
config_text = ''
config_content = SMALL_CONFIG
try:
# config2.py will be in the pywikibot/ directory relative to this
# script (generate_user_files)
install = os.path.dirname(os.path.abspath(__file__))
with codecs.open(os.path.join(install, "pywikibot", "config2.py"),
"r", "utf-8") as config_f:
config_file = config_f.read()
res = re.findall("^(# ############# (?:"
"LOGFILE|"
'EXTERNAL SCRIPT PATH|'
"INTERWIKI|"
"SOLVE_DISAMBIGUATION|"
"IMAGE RELATED|"
"TABLE CONVERSION BOT|"
"WEBLINK CHECKER|"
"DATABASE|"
"SEARCH ENGINE|"
"COPYRIGHT|"
"FURTHER"
") SETTINGS .*)^(?=#####|# =====)",
config_file, re.MULTILINE | re.DOTALL)
if not res:
warn('Extended config extraction failed', UserWarning)
config_text = '\n'.join(res)
if len(config_text.splitlines()) < 350:
warn('Extended config extraction too short: %d'
% len(config_text.splitlines()),
UserWarning)
config_content = EXTENDED_CONFIG
except Exception as e:
# If the warning was explicitly enabled, raise
if isinstance(e, UserWarning):
raise
pywikibot.output('Exception while creating extended user-config; '
'falling back to simple user-config.')
pywikibot.exception()
try:
# Finally save user-config.py
with codecs.open(_fnc, "w", "utf-8") as f:
f.write(config_content.format(main_family=main_family,
main_code=main_code,
usernames=usernames,
config_text=config_text,
botpasswords='password_file = ' +
('"user-password.py"'
if botpasswords
else 'None')))
pywikibot.output(u"'%s' written." % _fnc)
except BaseException:
if os.path.exists(_fnc):
os.remove(_fnc)
raise
if botpasswords:
# Save if necessary user-password.py
try:
# First create an empty file with good permissions, before writing
# in it
with codecs.open(_fncpass, 'w', 'utf-8') as f:
f.write('')
pywikibot.tools.file_mode_checker(_fncpass, mode=0o600,
quiet=True)
with codecs.open(_fncpass, 'w', 'utf-8') as f:
f.write(PASSFILE_CONFIG.format(botpasswords=botpasswords))
pywikibot.tools.file_mode_checker(_fncpass, mode=0o600)
pywikibot.output("'{0}' written.".format(_fncpass))
except EnvironmentError:
os.remove(_fncpass)
raise
def process_filename(self, file_url=None):
"""Return base filename portion of file_url."""
if not file_url:
file_url = self.url
pywikibot.warning("file_url is not given. "
"Set to self.url by default.")
always = self.getOption('always')
# Isolate the pure name
filename = file_url
# Filename may be either a URL or a local file path
if "://" in filename:
# extract the path portion of the URL
filename = urlparse(filename).path
filename = os.path.basename(filename)
if self.useFilename:
filename = self.useFilename
if not self.keepFilename:
pywikibot.output(
u"The filename on the target wiki will default to: %s" %
filename)
assert not always
newfn = pywikibot.input(
u'Enter a better name, or press enter to accept:')
if newfn != "":
filename = newfn
# FIXME: these 2 belong somewhere else, presumably in family
# forbidden characters are handled by pywikibot/page.py
forbidden = ':*?/\\' # to be extended
try:
allowed_formats = self.targetSite.siteinfo.get('fileextensions',
get_default=False)
except KeyError:
allowed_formats = []
else:
allowed_formats = [item['ext'] for item in allowed_formats]
# ask until it's valid
first_check = True
while True:
if not first_check:
if always:
filename = None
else:
filename = pywikibot.input('Enter a better name, or press '
'enter to skip the file:')
if not filename:
return None
first_check = False
ext = os.path.splitext(filename)[1].lower().strip('.')
# are any chars in forbidden also in filename?
invalid = set(forbidden) & set(filename)
if invalid:
c = "".join(invalid)
pywikibot.output('Invalid character(s): %s. Please try again' %
c)
continue
if allowed_formats and ext not in allowed_formats:
if always:
pywikibot.output('File format is not one of '
'[{0}]'.format(' '.join(allowed_formats)))
continue
elif not pywikibot.input_yn(
u"File format is not one of [%s], but %s. Continue?" %
(u' '.join(allowed_formats), ext),
default=False,
automatic_quit=False):
continue
potential_file_page = pywikibot.FilePage(self.targetSite, filename)
if potential_file_page.exists():
overwrite = self._handle_warning('exists')
if overwrite is False:
pywikibot.output(
"File exists and you asked to abort. Skipping.")
return None
if potential_file_page.canBeEdited():
if overwrite is None:
overwrite = not pywikibot.input_yn(
"File with name %s already exists. "
"Would you like to change the name? "
"(Otherwise file will be overwritten.)" % filename,
default=True,
automatic_quit=False)
if not overwrite:
continue
else:
break
else:
pywikibot.output(u"File with name %s already exists and "
"cannot be overwritten." % filename)
continue
else:
try:
if potential_file_page.fileIsShared():
pywikibot.output(
u"File with name %s already exists in shared "
"repository and cannot be overwritten." % filename)
continue
else:
break
except pywikibot.NoPage:
break
# A proper description for the submission.
# Empty descriptions are not accepted.
pywikibot.output(u'The suggested description is:\n%s' %
self.description)
# Description must be set and verified
if not self.description:
self.verifyDescription = True
while not self.description or self.verifyDescription:
if not self.description:
pywikibot.output(
color_format(
'{lightred}It is not possible to upload a file '
'without a summary/description.{default}'))
assert not always
# if no description, default is 'yes'
if pywikibot.input_yn(u'Do you want to change this description?',
default=not self.description):
from pywikibot import editor as editarticle
editor = editarticle.TextEditor()
try:
newDescription = editor.edit(self.description)
except Exception as e:
pywikibot.error(e)
continue
# if user saved / didn't press Cancel
if newDescription:
self.description = newDescription
self.verifyDescription = False
return filename
def run(self):
"""Run the bot."""
commons = self.commons
comment = self.summary
for page in self.generator:
self.current_page = page
try:
local_file_page = pywikibot.FilePage(self.site, page.title())
if local_file_page.file_is_shared():
pywikibot.output('File is already on Commons.')
continue
sha1 = local_file_page.latest_file_info.sha1
file_on_commons = self.find_file_on_commons(local_file_page)
if not file_on_commons:
pywikibot.output('NowCommons template not found.')
continue
commons_file_page = pywikibot.FilePage(
commons, 'File:' + file_on_commons)
if (local_file_page.title(with_ns=False) !=
commons_file_page.title(with_ns=False)):
using_pages = list(local_file_page.using_pages())
if using_pages and using_pages != [local_file_page]:
pywikibot.output(
color_format(
'"{lightred}{0}{default}" '
'is still used in {1} pages.',
local_file_page.title(with_ns=False),
len(using_pages)))
if self.opt.replace:
pywikibot.output(
color_format(
'Replacing "{lightred}{0}{default}" by '
'"{lightgreen}{1}{default}\".',
local_file_page.title(with_ns=False),
commons_file_page.title(with_ns=False)))
bot = ImageBot(
local_file_page.usingPages(),
local_file_page.title(with_ns=False),
commons_file_page.title(with_ns=False),
always=self.opt.replacealways,
loose=self.opt.replaceloose)
bot.run()
# If the image is used with the urlname the
# previous function won't work
is_used = bool(
list(
pywikibot.FilePage(
self.site,
page.title()).using_pages(total=1)))
if is_used and self.opt.replaceloose:
bot = ImageBot(
local_file_page.usimgPages(),
local_file_page.title(with_ns=False,
as_url=True),
commons_file_page.title(with_ns=False),
always=self.opt.replacealways,
loose=self.opt.replaceloose)
bot.run()
# refresh because we want the updated list
using_pages = len(
list(
pywikibot.FilePage(
self.site,
page.title()).using_pages()))
else:
pywikibot.output('Please change them manually.')
continue
pywikibot.output(
color_format(
'No page is using "{lightgreen}{0}{default}" '
'anymore.', local_file_page.title(with_ns=False)))
commons_text = commons_file_page.get()
if not self.opt.replaceonly:
if sha1 == commons_file_page.latest_file_info.sha1:
pywikibot.output(
'The file is identical to the one on Commons.')
if len(local_file_page.get_file_history()) > 1:
pywikibot.output(
'This file has a version history. Please '
'delete it manually after making sure that '
'the old versions are not worth keeping.')
continue
if self.opt.always is False:
format_str = color_format(
'\n\n>>>> Description on {lightpurple}%s'
'{default} <<<<\n')
pywikibot.output(format_str % page.title())
pywikibot.output(local_file_page.get())
pywikibot.output(format_str %
commons_file_page.title())
pywikibot.output(commons_text)
if pywikibot.input_yn(
'Does the description on Commons contain '
'all required source and license\n'
'information?',
default=False,
automatic_quit=False):
local_file_page.delete(
'{} [[:commons:File:{}]]'.format(
comment, file_on_commons),
prompt=False)
else:
local_file_page.delete(
comment + ' [[:commons:File:{}]]'.format(
file_on_commons),
prompt=False)
else:
pywikibot.output('The file is not identical to '
'the one on Commons.')
except (NoPageError, IsRedirectPageError) as e:
pywikibot.output(str(e[0]))
continue
else:
self.counter['read'] += 1
if not self.counter['read']:
pywikibot.output('No transcluded files found for {}.'.format(
self.nc_templates_list()[0]))
self.exit()
def treat(self, refPage, disambPage):
"""
Treat a page.
Parameters:
disambPage - The disambiguation page or redirect we don't want
anything to link to
refPage - A page linking to disambPage
Returns False if the user pressed q to completely quit the program.
Otherwise, returns True.
"""
# TODO: break this function up into subroutines!
include = False
unlink_counter = 0
new_targets = []
try:
text = refPage.get()
ignoreReason = self.checkContents(text)
if ignoreReason:
pywikibot.output(
'\n\nSkipping %s because it contains %s.\n\n' %
(refPage.title(), ignoreReason))
else:
include = True
except pywikibot.IsRedirectPage:
pywikibot.output(u'%s is a redirect to %s' %
(refPage.title(), disambPage.title()))
if disambPage.isRedirectPage():
target = self.alternatives[0]
if pywikibot.input_yn(u'Do you want to make redirect %s point '
'to %s?' % (refPage.title(), target),
default=False,
automatic_quit=False):
redir_text = '#%s [[%s]]' \
% (self.mysite.redirect(), target)
try:
refPage.put_async(redir_text, summary=self.comment)
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Page not saved: %s' % error.args)
else:
choice = pywikibot.input_choice(
u'Do you want to work on pages linking to %s?' %
refPage.title(), [('yes', 'y'), ('no', 'n'),
('change redirect', 'c')],
'n',
automatic_quit=False)
if choice == 'y':
gen = ReferringPageGeneratorWithIgnore(
refPage, self.primary)
preloadingGen = pagegenerators.PreloadingGenerator(gen)
for refPage2 in preloadingGen:
# run until the user selected 'quit'
if not self.treat(refPage2, refPage):
break
elif choice == 'c':
text = refPage.get(get_redirect=True)
include = "redirect"
except pywikibot.NoPage:
pywikibot.output(
u'Page [[%s]] does not seem to exist?! Skipping.' %
refPage.title())
include = False
if include in (True, "redirect"):
# make a backup of the original text so we can show the changes later
original_text = text
n = 0
curpos = 0
dn = False
edited = False
# This loop will run until we have finished the current page
while True:
m = self.linkR.search(text, pos=curpos)
if not m:
if n == 0:
pywikibot.output(u"No changes necessary in %s" %
refPage.title())
return True
else:
# stop loop and save page
break
# Make sure that next time around we will not find this same hit.
curpos = m.start() + 1
try:
foundlink = pywikibot.Link(m.group('title'),
disambPage.site)
foundlink.parse()
except pywikibot.Error:
continue
# ignore interwiki links
if foundlink.site != disambPage.site:
continue
# Check whether the link found is to disambPage.
try:
if foundlink.canonical_title() != disambPage.title():
continue
except pywikibot.Error:
# must be a broken link
pywikibot.log(u"Invalid link [[%s]] in page [[%s]]" %
(m.group('title'), refPage.title()))
continue
n += 1
# how many bytes should be displayed around the current link
context = 60
# check if there's a dn-template here already
if (self.dnSkip and self.dn_template_str
and self.dn_template_str[:-2]
in text[m.end():m.end() + len(self.dn_template_str) +
8]):
continue
# This loop will run while the user doesn't choose an option
# that will actually change the page
while True:
self.current_page = refPage
if not self.always:
# at the beginning of the link, start red color.
# at the end of the link, reset the color to default
pywikibot.output(text[max(0,
m.start() -
context):m.start()] +
'\03{lightred}' +
text[m.start():m.end()] +
'\03{default}' +
text[m.end():m.end() + context])
options = [
'#', 'r#', '[s]kip link', '[e]dit page',
'[n]ext page', '[u]nlink', '[q]uit'
]
if self.dn_template_str:
options.append(u'[t]ag template %s' %
self.dn_template_str)
options.append('[m]ore context')
if not edited:
options.append('show [d]isambiguation page')
options += ['[l]ist', '[a]dd new']
if edited:
options += ['save in this form [x]']
options = concat_options('Option', 72, options)
choice = pywikibot.input(options)
else:
choice = self.always
if choice in ['a', 'A']:
newAlternative = pywikibot.input(u'New alternative:')
self.alternatives.append(newAlternative)
self.listAlternatives()
elif choice in ['e', 'E']:
editor = editarticle.TextEditor()
newText = editor.edit(text,
jumpIndex=m.start(),
highlight=disambPage.title())
# if user didn't press Cancel
if newText and newText != text:
text = newText
break
elif choice in ['d', 'D']:
editor = editarticle.TextEditor()
if disambPage.isRedirectPage():
disambredir = disambPage.getRedirectTarget()
editor.edit(disambredir.get(),
jumpIndex=m.start(),
highlight=disambredir.title())
else:
editor.edit(disambPage.get(),
jumpIndex=m.start(),
highlight=disambPage.title())
elif choice in ['l', 'L']:
self.listAlternatives()
elif choice in ['m', 'M']:
# show more text around the link we're working on
context *= 2
else:
break
if choice in ['e', 'E']:
# user has edited the page and then pressed 'OK'
edited = True
curpos = 0
continue
elif choice in ['n', 'N']:
# skip this page
if self.primary:
# If run with the -primary argument, skip this
# occurrence next time.
self.primaryIgnoreManager.ignore(refPage)
return True
elif choice in ['q', 'Q']:
# quit the program
self.quit()
elif choice in ['s', 'S']:
# Next link on this page
n -= 1
continue
elif choice in ['x', 'X'] and edited:
# Save the page as is
break
# The link looks like this:
# [[page_title|link_text]]trailing_chars
page_title = m.group('title')
link_text = m.group('label')
if not link_text:
# or like this: [[page_title]]trailing_chars
link_text = page_title
if m.group('section') is None:
section = ''
else:
section = m.group('section')
trailing_chars = m.group('linktrail')
if trailing_chars:
link_text += trailing_chars
# '?', '/' for old choice
if choice in ['t', 'T', '?', '/'] and self.dn_template_str:
# small chunk of text to search
search_text = text[m.end():m.end() + context]
# figure out where the link (and sentance) ends, put note
# there
end_of_word_match = re.search(r'\s', search_text)
if end_of_word_match:
position_split = end_of_word_match.start(0)
else:
position_split = 0
# insert dab needed template
text = (text[:m.end() + position_split] +
self.dn_template_str +
text[m.end() + position_split:])
dn = True
continue
elif choice in ['u', 'U']:
# unlink - we remove the section if there's any
text = text[:m.start()] + link_text + text[m.end():]
unlink_counter += 1
continue
else:
if len(choice) > 0 and choice[0] == 'r':
# we want to throw away the original link text
replaceit = link_text == page_title
choice = choice[1:]
elif include == "redirect":
replaceit = True
else:
replaceit = False
try:
choice = int(choice)
except ValueError:
pywikibot.output(u"Unknown option")
# step back to ask the user again what to do with the
# current link
curpos -= 1
continue
if choice >= len(self.alternatives) or choice < 0:
pywikibot.output(
u"Choice out of range. Please select a number "
u"between 0 and %i." %
(len(self.alternatives) - 1))
# show list of possible choices
self.listAlternatives()
# step back to ask the user again what to do with the
# current link
curpos -= 1
continue
new_page_title = self.alternatives[choice]
repPl = pywikibot.Page(
pywikibot.Link(new_page_title, disambPage.site))
if (new_page_title[0].isupper() or link_text[0].isupper()):
new_page_title = repPl.title()
else:
new_page_title = repPl.title()
new_page_title = first_lower(new_page_title)
if new_page_title not in new_targets:
new_targets.append(new_page_title)
if replaceit and trailing_chars:
newlink = "[[%s%s]]%s" % (new_page_title, section,
trailing_chars)
elif replaceit or (new_page_title == link_text
and not section):
newlink = "[[%s]]" % new_page_title
# check if we can create a link with trailing characters
# instead of a pipelink
elif ((len(new_page_title) <= len(link_text))
and (firstcap(link_text[:len(new_page_title)])
== firstcap(new_page_title))
and (re.sub(self.trailR, '',
link_text[len(new_page_title):]) == '')
and (not section)):
newlink = "[[%s]]%s" \
% (link_text[:len(new_page_title)],
link_text[len(new_page_title):])
else:
newlink = "[[%s%s|%s]]" \
% (new_page_title, section, link_text)
text = text[:m.start()] + newlink + text[m.end():]
continue
pywikibot.output(text[max(0, m.start() - 30):m.end() + 30])
if text == original_text:
pywikibot.output(u'\nNo changes have been made:\n')
else:
pywikibot.output(u'\nThe following changes have been made:\n')
pywikibot.showDiff(original_text, text)
pywikibot.output(u'')
# save the page
self.setSummaryMessage(disambPage, new_targets, unlink_counter,
dn)
try:
refPage.put_async(text, summary=self.comment)
except pywikibot.LockedPage:
pywikibot.output(u'Page not saved: page is locked')
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Page not saved: %s' % error.args)
return True
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
add_cat = None
gen = None
# summary message
edit_summary = u""
# Array which will collect commandline parameters.
# First element is original text, second element is replacement text.
commandline_replacements = []
# A list of 2-tuples of original text and replacement text.
replacements = []
# Don't edit pages which contain certain texts.
exceptions = {
'title': [],
'text-contains': [],
'inside': [],
'inside-tags': [],
'require-title': [], # using a seperate requirements dict needs some
} # major refactoring of code.
# Should the elements of 'replacements' and 'exceptions' be interpreted
# as regular expressions?
regex = False
# Predefined fixes from dictionary 'fixes' (see above).
fixes_set = []
# the dump's path, either absolute or relative, which will be used
# if -xml flag is present
xmlFilename = None
useSql = False
# will become True when the user presses a ('yes to all') or uses the
# -always flag.
acceptall = False
# Will become True if the user inputs the commandline parameter -nocase
caseInsensitive = False
# Will become True if the user inputs the commandline parameter -dotall
dotall = False
# Will become True if the user inputs the commandline parameter -multiline
multiline = False
# Do all hits when they overlap
allowoverlap = False
# Do not recurse replacement
recursive = False
# Between a regex and another (using -fix) sleep some time (not to waste
# too much CPU
sleep = None
# Read commandline parameters.
local_args = pywikibot.handle_args(args)
genFactory = pagegenerators.GeneratorFactory()
for arg in local_args:
if genFactory.handleArg(arg):
continue
if arg == '-regex':
regex = True
elif arg.startswith('-xmlstart'):
if len(arg) == 9:
xmlStart = pywikibot.input(
u'Please enter the dumped article to start with:')
else:
xmlStart = arg[10:]
elif arg.startswith('-xml'):
if len(arg) == 4:
xmlFilename = i18n.input('pywikibot-enter-xml-filename')
else:
xmlFilename = arg[5:]
elif arg == '-sql':
useSql = True
elif arg.startswith('-excepttitle:'):
exceptions['title'].append(arg[13:])
elif arg.startswith('-requiretitle:'):
exceptions['require-title'].append(arg[14:])
elif arg.startswith('-excepttext:'):
exceptions['text-contains'].append(arg[12:])
elif arg.startswith('-exceptinside:'):
exceptions['inside'].append(arg[14:])
elif arg.startswith('-exceptinsidetag:'):
exceptions['inside-tags'].append(arg[17:])
elif arg.startswith('-fix:'):
fixes_set += [arg[5:]]
elif arg.startswith('-sleep:'):
sleep = float(arg[7:])
elif arg == '-always':
acceptall = True
elif arg == '-recursive':
recursive = True
elif arg == '-nocase':
caseInsensitive = True
elif arg == '-dotall':
dotall = True
elif arg == '-multiline':
multiline = True
elif arg.startswith('-addcat:'):
add_cat = arg[8:]
elif arg.startswith('-summary:'):
edit_summary = arg[9:]
elif arg.startswith('-allowoverlap'):
allowoverlap = True
else:
commandline_replacements.append(arg)
site = pywikibot.Site()
if (len(commandline_replacements) % 2):
raise pywikibot.Error('require even number of replacements.')
if not commandline_replacements:
if fixes_set:
manual = pywikibot.input_yn('Replacements via -fix: set. Apply '
'also manual replacements?', default=False)
else:
manual = True
if manual:
old = pywikibot.input(u'Please enter the text that should be replaced:')
while old:
new = pywikibot.input(u'Please enter the new text:')
commandline_replacements += [old, new]
old = pywikibot.input(
u'Please enter another text that should be replaced,' +
u'\nor press Enter to start:')
single_summary = None
for i in range(0, len(commandline_replacements), 2):
replacement = Replacement(commandline_replacements[i],
commandline_replacements[i + 1])
if not single_summary:
single_summary = i18n.twtranslate(
site, 'replace-replacing',
{'description':
' (-%s +%s)' % (replacement.old, replacement.new)}
)
replacements.append(replacement)
if not edit_summary:
if single_summary:
pywikibot.output(u'The summary message for the command line '
'replacements will be something like: %s'
% single_summary)
if fixes_set:
pywikibot.output('If a summary is defined for the fix, this '
'default summary won\'t be applied.')
edit_summary = pywikibot.input(
u'Press Enter to use this automatic message, or enter a ' +
u'description of the\nchanges your bot will make:')
# Perform one of the predefined actions.
for fix in fixes_set:
try:
fix = fixes.fixes[fix]
except KeyError:
pywikibot.output(u'Available predefined fixes are: %s'
% ', '.join(fixes.fixes.keys()))
return
if "msg" in fix:
if isinstance(fix['msg'], basestring):
set_summary = i18n.twtranslate(site, str(fix['msg']))
else:
set_summary = i18n.translate(site, fix['msg'], fallback=True)
else:
set_summary = None
for replacement in fix['replacements']:
summary = set_summary if len(replacement) < 3 else replacement[2]
replacements.append(Replacement(
old=replacement[0],
new=replacement[1],
use_regex=fix.get('regex'),
edit_summary=summary,
exceptions=fix.get('exceptions'),
case_insensitive=fix.get('nocase')
))
# Set the regular expression flags
flags = re.UNICODE
if caseInsensitive:
flags = flags | re.IGNORECASE
if dotall:
flags = flags | re.DOTALL
if multiline:
flags = flags | re.MULTILINE
# Pre-compile all regular expressions here to save time later
for replacement in replacements:
replacement.compile(regex, flags)
precompile_exceptions(exceptions, regex, flags)
if xmlFilename:
try:
xmlStart
except NameError:
xmlStart = None
gen = XmlDumpReplacePageGenerator(xmlFilename, xmlStart,
replacements, exceptions, site)
elif useSql:
whereClause = 'WHERE (%s)' % ' OR '.join(
["old_text RLIKE '%s'" % prepareRegexForMySQL(old_regexp.pattern)
for (old_regexp, new_text) in replacements])
if exceptions:
exceptClause = 'AND NOT (%s)' % ' OR '.join(
["old_text RLIKE '%s'" % prepareRegexForMySQL(exc.pattern)
for exc in exceptions])
else:
exceptClause = ''
query = u"""
SELECT page_namespace, page_title
FROM page
JOIN text ON (page_id = old_id)
%s
%s
LIMIT 200""" % (whereClause, exceptClause)
gen = pagegenerators.MySQLPageGenerator(query)
gen = genFactory.getCombinedGenerator(gen)
if not gen:
# syntax error, show help text from the top of this file
pywikibot.showHelp('replace')
return
preloadingGen = pagegenerators.PreloadingGenerator(gen)
bot = ReplaceRobot(preloadingGen, replacements, exceptions, acceptall,
allowoverlap, recursive, add_cat, sleep, edit_summary,
site)
site.login()
bot.run()
# Explicitly call pywikibot.stopme().
# It will make sure the callback is triggered before replace.py is unloaded.
pywikibot.stopme()
pywikibot.output(u'\n%s pages changed.' % bot.changed_pages)
def run(self):
commons = pywikibot.Site('commons', 'commons')
comment = i18n.twtranslate(self.site, 'imagetransfer-nowcommons_notice')
for page in self.getPageGenerator():
if self.getOption('use_hash'):
# Page -> Has the namespace | commons image -> Not
images_list = page # 0 -> local image, 1 -> commons image
page = pywikibot.Page(self.site, images_list[0])
else:
# If use_hash is true, we have already print this before, no need
self.current_page = page
try:
localImagePage = pywikibot.FilePage(self.site, page.title())
if localImagePage.fileIsShared():
pywikibot.output(u'File is already on Commons.')
continue
sha1 = localImagePage.latest_file_info.sha1
if self.getOption('use_hash'):
filenameOnCommons = images_list[1]
else:
filenameOnCommons = self.findFilenameOnCommons(
localImagePage)
if not filenameOnCommons and not self.getOption('use_hash'):
pywikibot.output(u'NowCommons template not found.')
continue
commonsImagePage = pywikibot.FilePage(commons, 'Image:%s'
% filenameOnCommons)
if (localImagePage.title(withNamespace=False) ==
commonsImagePage.title(withNamespace=False) and
self.getOption('use_hash')):
pywikibot.output(
u'The local and the commons images have the same name')
if (localImagePage.title(withNamespace=False) !=
commonsImagePage.title(withNamespace=False)):
usingPages = list(localImagePage.usingPages())
if usingPages and usingPages != [localImagePage]:
pywikibot.output(color_format(
'"{lightred}{0}{default}" is still used in {1} pages.',
localImagePage.title(withNamespace=False),
len(usingPages)))
if self.getOption('replace') is True:
pywikibot.output(color_format(
'Replacing "{lightred}{0}{default}" by '
'"{lightgreen}{1}{default}\".',
localImagePage.title(withNamespace=False),
commonsImagePage.title(withNamespace=False)))
bot = ImageBot(
pg.FileLinksGenerator(localImagePage),
localImagePage.title(withNamespace=False),
commonsImagePage.title(withNamespace=False),
'', self.getOption('replacealways'),
self.getOption('replaceloose'))
bot.run()
# If the image is used with the urlname the
# previous function won't work
is_used = bool(list(pywikibot.FilePage(
self.site, page.title()).usingPages(total=1)))
if is_used and self.getOption('replaceloose'):
bot = ImageBot(
pg.FileLinksGenerator(
localImagePage),
localImagePage.title(
withNamespace=False, asUrl=True),
commonsImagePage.title(
withNamespace=False),
'', self.getOption('replacealways'),
self.getOption('replaceloose'))
bot.run()
# refresh because we want the updated list
usingPages = len(list(pywikibot.FilePage(
self.site, page.title()).usingPages()))
if usingPages > 0 and self.getOption('use_hash'):
# just an enter
pywikibot.input(
u'There are still %s pages with this \
image, confirm the manual removal from them please.'
% usingPages)
else:
pywikibot.output(u'Please change them manually.')
continue
else:
pywikibot.output(color_format(
'No page is using "{lightgreen}{0}{default}" '
'anymore.',
localImagePage.title(withNamespace=False)))
commonsText = commonsImagePage.get()
if self.getOption('replaceonly') is False:
if sha1 == commonsImagePage.latest_file_info.sha1:
pywikibot.output(
u'The image is identical to the one on Commons.')
if (len(localImagePage.getFileVersionHistory()) > 1 and
not self.getOption('use_hash')):
pywikibot.output(
u"This image has a version history. Please \
delete it manually after making sure that the \
old versions are not worth keeping.""")
continue
if self.getOption('always') is False:
format_str = color_format(
'\n\n>>>> Description on {lightpurple}%s'
'{default} <<<<\n')
pywikibot.output(format_str % page.title())
pywikibot.output(localImagePage.get())
pywikibot.output(format_str %
commonsImagePage.title())
pywikibot.output(commonsText)
if pywikibot.input_yn(
u'Does the description on Commons contain '
'all required source and license\n'
'information?',
default=False, automatic_quit=False):
localImagePage.delete(
'%s [[:commons:Image:%s]]'
% (comment, filenameOnCommons), prompt=False)
else:
localImagePage.delete(
comment + ' [[:commons:Image:%s]]'
% filenameOnCommons, prompt=False)
else:
pywikibot.output(
u'The image is not identical to the one on Commons.')
except (pywikibot.NoPage, pywikibot.IsRedirectPage) as e:
pywikibot.output(u'%s' % e[0])
continue
def treat(self, refPage, disambPage):
"""
Treat a page.
Parameters:
disambPage - The disambiguation page or redirect we don't want
anything to link to
refPage - A page linking to disambPage
Returns False if the user pressed q to completely quit the program.
Otherwise, returns True.
"""
# TODO: break this function up into subroutines!
include = False
unlink_counter = 0
new_targets = []
try:
text = refPage.get()
ignoreReason = self.checkContents(text)
if ignoreReason:
pywikibot.output('\n\nSkipping %s because it contains %s.\n\n'
% (refPage.title(), ignoreReason))
else:
include = True
except pywikibot.IsRedirectPage:
pywikibot.output(u'%s is a redirect to %s'
% (refPage.title(), disambPage.title()))
if disambPage.isRedirectPage():
target = self.alternatives[0]
if pywikibot.input_yn(u'Do you want to make redirect %s point '
'to %s?' % (refPage.title(), target),
default=False, automatic_quit=False):
redir_text = '#%s [[%s]]' \
% (self.mysite.redirect(), target)
try:
refPage.put_async(redir_text, summary=self.comment)
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Page not saved: %s' % error.args)
else:
choice = pywikibot.input_choice(
u'Do you want to work on pages linking to %s?'
% refPage.title(),
[('yes', 'y'), ('no', 'n'), ('change redirect', 'c')], 'n',
automatic_quit=False)
if choice == 'y':
gen = ReferringPageGeneratorWithIgnore(refPage,
self.primary)
preloadingGen = pagegenerators.PreloadingGenerator(gen)
for refPage2 in preloadingGen:
# run until the user selected 'quit'
if not self.treat(refPage2, refPage):
break
elif choice == 'c':
text = refPage.get(get_redirect=True)
include = "redirect"
except pywikibot.NoPage:
pywikibot.output(
u'Page [[%s]] does not seem to exist?! Skipping.'
% refPage.title())
include = False
if include in (True, "redirect"):
# make a backup of the original text so we can show the changes later
original_text = text
n = 0
curpos = 0
dn = False
edited = False
# This loop will run until we have finished the current page
while True:
m = self.linkR.search(text, pos=curpos)
if not m:
if n == 0:
pywikibot.output(u"No changes necessary in %s"
% refPage.title())
return True
else:
# stop loop and save page
break
# Make sure that next time around we will not find this same hit.
curpos = m.start() + 1
try:
foundlink = pywikibot.Link(m.group('title'),
disambPage.site)
foundlink.parse()
except pywikibot.Error:
continue
# ignore interwiki links
if foundlink.site != disambPage.site:
continue
# Check whether the link found is to disambPage.
try:
if foundlink.canonical_title() != disambPage.title():
continue
except pywikibot.Error:
# must be a broken link
pywikibot.log(u"Invalid link [[%s]] in page [[%s]]"
% (m.group('title'), refPage.title()))
continue
n += 1
# how many bytes should be displayed around the current link
context = 60
# check if there's a dn-template here already
if (self.dnSkip and self.dn_template_str and
self.dn_template_str[:-2] in text[m.end():m.end() +
len(self.dn_template_str) + 8]):
continue
# This loop will run while the user doesn't choose an option
# that will actually change the page
while True:
self.current_page = refPage
if not self.always:
# at the beginning of the link, start red color.
# at the end of the link, reset the color to default
pywikibot.output(
text[max(0, m.start() - context):m.start()] +
'\03{lightred}' + text[m.start():m.end()] +
'\03{default}' + text[m.end():m.end() + context])
options = ['#', 'r#', '[s]kip link', '[e]dit page',
'[n]ext page', '[u]nlink', '[q]uit']
if self.dn_template_str:
options.append(u'[t]ag template %s' % self.dn_template_str)
options.append('[m]ore context')
if not edited:
options.append('show [d]isambiguation page')
options += ['[l]ist', '[a]dd new']
if edited:
options += ['save in this form [x]']
options = concat_options('Option', 72, options)
choice = pywikibot.input(options)
else:
choice = self.always
if choice in ['a', 'A']:
newAlternative = pywikibot.input(u'New alternative:')
self.alternatives.append(newAlternative)
self.listAlternatives()
elif choice in ['e', 'E']:
editor = editarticle.TextEditor()
newText = editor.edit(text, jumpIndex=m.start(),
highlight=disambPage.title())
# if user didn't press Cancel
if newText and newText != text:
text = newText
break
elif choice in ['d', 'D']:
editor = editarticle.TextEditor()
if disambPage.isRedirectPage():
disambredir = disambPage.getRedirectTarget()
editor.edit(
disambredir.get(),
jumpIndex=m.start(),
highlight=disambredir.title())
else:
editor.edit(
disambPage.get(),
jumpIndex=m.start(),
highlight=disambPage.title())
elif choice in ['l', 'L']:
self.listAlternatives()
elif choice in ['m', 'M']:
# show more text around the link we're working on
context *= 2
else:
break
if choice in ['e', 'E']:
# user has edited the page and then pressed 'OK'
edited = True
curpos = 0
continue
elif choice in ['n', 'N']:
# skip this page
if self.primary:
# If run with the -primary argument, skip this
# occurrence next time.
self.primaryIgnoreManager.ignore(refPage)
return True
elif choice in ['q', 'Q']:
# quit the program
self.quit()
elif choice in ['s', 'S']:
# Next link on this page
n -= 1
continue
elif choice in ['x', 'X'] and edited:
# Save the page as is
break
# The link looks like this:
# [[page_title|link_text]]trailing_chars
page_title = m.group('title')
link_text = m.group('label')
if not link_text:
# or like this: [[page_title]]trailing_chars
link_text = page_title
if m.group('section') is None:
section = ''
else:
section = m.group('section')
trailing_chars = m.group('linktrail')
if trailing_chars:
link_text += trailing_chars
# '?', '/' for old choice
if choice in ['t', 'T', '?', '/'] and self.dn_template_str:
# small chunk of text to search
search_text = text[m.end():m.end() + context]
# figure out where the link (and sentance) ends, put note
# there
end_of_word_match = re.search(r'\s', search_text)
if end_of_word_match:
position_split = end_of_word_match.start(0)
else:
position_split = 0
# insert dab needed template
text = (text[:m.end() + position_split] +
self.dn_template_str +
text[m.end() + position_split:])
dn = True
continue
elif choice in ['u', 'U']:
# unlink - we remove the section if there's any
text = text[:m.start()] + link_text + text[m.end():]
unlink_counter += 1
continue
else:
if len(choice) > 0 and choice[0] == 'r':
# we want to throw away the original link text
replaceit = link_text == page_title
choice = choice[1:]
elif include == "redirect":
replaceit = True
else:
replaceit = False
try:
choice = int(choice)
except ValueError:
pywikibot.output(u"Unknown option")
# step back to ask the user again what to do with the
# current link
curpos -= 1
continue
if choice >= len(self.alternatives) or choice < 0:
pywikibot.output(
u"Choice out of range. Please select a number "
u"between 0 and %i." % (len(self.alternatives) - 1))
# show list of possible choices
self.listAlternatives()
# step back to ask the user again what to do with the
# current link
curpos -= 1
continue
new_page_title = self.alternatives[choice]
repPl = pywikibot.Page(pywikibot.Link(new_page_title,
disambPage.site))
if (new_page_title[0].isupper() or
link_text[0].isupper()):
new_page_title = repPl.title()
else:
new_page_title = repPl.title()
new_page_title = first_lower(new_page_title)
if new_page_title not in new_targets:
new_targets.append(new_page_title)
if replaceit and trailing_chars:
newlink = "[[%s%s]]%s" % (new_page_title,
section,
trailing_chars)
elif replaceit or (new_page_title == link_text and
not section):
newlink = "[[%s]]" % new_page_title
# check if we can create a link with trailing characters
# instead of a pipelink
elif (
(len(new_page_title) <= len(link_text)) and
(firstcap(link_text[:len(new_page_title)]) == firstcap(new_page_title)) and
(re.sub(self.trailR, '', link_text[len(new_page_title):]) == '') and
(not section)
):
newlink = "[[%s]]%s" \
% (link_text[:len(new_page_title)],
link_text[len(new_page_title):])
else:
newlink = "[[%s%s|%s]]" \
% (new_page_title, section, link_text)
text = text[:m.start()] + newlink + text[m.end():]
continue
pywikibot.output(text[max(0, m.start() - 30):m.end() + 30])
if text == original_text:
pywikibot.output(u'\nNo changes have been made:\n')
else:
pywikibot.output(u'\nThe following changes have been made:\n')
pywikibot.showDiff(original_text, text)
pywikibot.output(u'')
# save the page
self.setSummaryMessage(disambPage, new_targets, unlink_counter,
dn)
try:
refPage.put_async(text, summary=self.comment)
except pywikibot.LockedPage:
pywikibot.output(u'Page not saved: page is locked')
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Page not saved: %s' % error.args)
return True
def main(*args):
"""
Process command line arguments and generate user-config.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
global base_dir
# set the config family and mylang values to an invalid state so that
# the script can detect that the command line arguments -family & -lang
# were used and and handle_args has updated these config values,
# and 'force' mode can be activated below.
(config.family, config.mylang) = ('wikipedia', None)
local_args = pywikibot.handle_args(args)
if local_args:
pywikibot.output('Unknown arguments: %s' % ' '.join(local_args))
return False
if config.mylang is not None:
force = True
pywikibot.output(u'Automatically generating user-config.py')
else:
force = False
# Force default site of en.wikipedia
(config.family, config.mylang) = ('wikipedia', 'en')
username = config.usernames[config.family].get(config.mylang)
args = (config.family, config.mylang, username)
while not force or config.verbose_output:
pywikibot.output(u'\nYour default user directory is "%s"' % base_dir)
if pywikibot.input_yn("Do you want to use that directory?",
default=True,
automatic_quit=False,
force=force):
break
else:
new_base = change_base_dir()
if new_base:
base_dir = new_base
break
copied_config = False
copied_fixes = False
while not force or config.verbose_output:
if os.path.exists(os.path.join(base_dir, "user-config.py")):
break
if pywikibot.input_yn(
"Do you want to copy user files from an existing Pywikibot "
"installation?",
default=False,
force=force,
automatic_quit=False):
oldpath = pywikibot.input("Path to existing user-config.py?")
if not os.path.exists(oldpath):
pywikibot.error("Not a valid path")
continue
if os.path.isfile(oldpath):
# User probably typed /user-config.py at the end, so strip it
oldpath = os.path.dirname(oldpath)
if not os.path.isfile(os.path.join(oldpath, "user-config.py")):
pywikibot.error("No user_config.py found in that directory")
continue
shutil.copyfile(os.path.join(oldpath, "user-config.py"),
os.path.join(base_dir, "user-config.py"))
copied_config = True
if os.path.isfile(os.path.join(oldpath, "user-fixes.py")):
shutil.copyfile(os.path.join(oldpath, "user-fixes.py"),
os.path.join(base_dir, "user-fixes.py"))
copied_fixes = True
else:
break
if not os.path.isfile(os.path.join(base_dir, "user-config.py")):
if ((force and not config.verbose_output) or pywikibot.input_yn(
'Create user-config.py file? Required for '
'running bots.',
default=True,
automatic_quit=False,
force=force)):
create_user_config(args, force=force)
elif not copied_config:
pywikibot.output("user-config.py already exists in the directory")
if not os.path.isfile(os.path.join(base_dir, "user-fixes.py")):
if ((force and not config.verbose_output) or pywikibot.input_yn(
'Create user-fixes.py file? Optional and '
'for advanced users.',
force=force,
default=False,
automatic_quit=False)):
create_user_fixes()
elif not copied_fixes:
pywikibot.output("user-fixes.py already exists in the directory")
def __init__(self, generator, templates, **kwargs):
"""
Constructor.
@param generator: the pages to work on
@type generator: iterable
@param templates: a dictionary which maps old template names to
their replacements. If remove or subst is True, it maps the
names of the templates that should be removed/resolved to None.
@type templates: dict
"""
self.availableOptions.update({
'subst': False,
'remove': False,
'summary': None,
'addedCat': None,
})
Bot.__init__(self, generator=generator, **kwargs)
self.templates = templates
# get edit summary message if it's empty
if not self.getOption('summary'):
comma = self.site.mediawiki_message('comma-separator')
params = {'list': comma.join(self.templates.keys()),
'num': len(self.templates)}
site = self.site
if self.getOption('remove'):
self.options['summary'] = i18n.twtranslate(
site, 'template-removing', params)
elif self.getOption('subst'):
self.options['summary'] = i18n.twtranslate(
site, 'template-substituting', params)
else:
self.options['summary'] = i18n.twtranslate(
site, 'template-changing', params)
# regular expression to find the original template.
# {{vfd}} does the same thing as {{Vfd}}, so both will be found.
# The old syntax, {{msg:vfd}}, will also be found.
# The group 'parameters' will either match the parameters, or an
# empty string if there are none.
replacements = []
exceptions = {}
builder = textlib._MultiTemplateMatchBuilder(site)
for old, new in self.templates.items():
templateRegex = builder.pattern(old)
if self.getOption('subst') and self.getOption('remove'):
replacements.append((templateRegex,
r'{{{{subst:{0!s}\g<parameters>}}}}'.format(new)))
exceptions['inside-tags'] = ['ref', 'gallery']
elif self.getOption('subst'):
replacements.append((templateRegex,
r'{{{{subst:{0!s}\g<parameters>}}}}'.format(old)))
exceptions['inside-tags'] = ['ref', 'gallery']
elif self.getOption('remove'):
replacements.append((templateRegex, ''))
else:
template = pywikibot.Page(self.site, new, ns=10)
if not template.exists():
pywikibot.warning(u'Template "{0!s}" does not exist.'.format(new))
if not pywikibot.input_yn('Do you want to proceed anyway?',
default=False, automatic_quit=False):
continue
replacements.append((templateRegex,
r'{{{{{0!s}\g<parameters>}}}}'.format(new)))
super(TemplateRobot, self).__init__(
generator, replacements, exceptions,
always=self.getOption('always'),
addedCat=self.getOption('addedCat'),
summary=self.getOption('summary'))
def get_site_and_lang(default_family='wikipedia',
default_lang='en',
default_username=None,
force=False):
"""
Ask the user for the family, language and username.
@param default_family: The default family which should be chosen.
@type default_family: None or str
@param default_lang: The default language which should be chosen, if the
family supports this language.
@type default_lang: None or str
@param default_username: The default username which should be chosen.
@type default_username: None or str
@return: The family, language and username
@rtype: tuple of three str
"""
known_families = sorted(pywikibot.config2.family_files.keys())
if default_family not in known_families:
default_family = None
fam = pywikibot.bot.input_list_choice(
u"Select family of sites we are working on, "
u"just enter the number or name",
known_families,
force=force,
default=default_family)
fam = pywikibot.family.Family.load(fam)
if hasattr(fam, "langs"):
if hasattr(fam, "languages_by_size"):
by_size = [
code for code in fam.languages_by_size
if code in fam.langs.keys()
]
else:
by_size = []
known_langs = by_size + sorted(
set(fam.langs.keys()).difference(by_size))
else:
known_langs = []
if len(known_langs) == 0:
pywikibot.output('There were no known languages found in {0}.'.format(
fam.name))
default_lang = None
elif len(known_langs) == 1:
pywikibot.output('The only known language: {0}'.format(known_langs[0]))
default_lang = known_langs[0]
else:
pywikibot.output("This is the list of known languages:")
pywikibot.output(u", ".join(known_langs))
if default_lang not in known_langs:
if default_lang != 'en' and 'en' in known_langs:
default_lang = 'en'
else:
default_lang = None
message = "The language code of the site we're working on"
mylang = None
while not mylang:
mylang = pywikibot.input(message, default=default_lang, force=force)
if known_langs and mylang and mylang not in known_langs:
if not pywikibot.input_yn("The language code {0} is not in the "
"list of known languages. Do you want "
"to continue?".format(mylang),
default=False,
automatic_quit=False):
mylang = None
message = u"Username on {0}:{1}".format(mylang, fam.name)
username = pywikibot.input(message, default=default_username, force=force)
# Escape ''s
if username:
username = username.replace("'", "\\'")
return fam.name, mylang, username
def treat(self, refPage, disambPage):
"""Treat a page.
@param disambPage: the disambiguation page or redirect we don't want
anything to link to
@type disambPage: pywikibot.Page
@param refPage: a page linking to disambPage
@type refPage: pywikibot.Page
@return: False if the user pressed q to completely quit the program,
True otherwise
@rtype: bool
"""
# TODO: break this function up into subroutines!
self.current_page = refPage
include = False
unlink_counter = 0
new_targets = []
try:
text = refPage.get()
ignoreReason = self.checkContents(text)
if ignoreReason:
pywikibot.output('\n\nSkipping %s because it contains %s.\n\n'
% (refPage.title(), ignoreReason))
else:
include = True
except pywikibot.IsRedirectPage:
pywikibot.output(u'%s is a redirect to %s'
% (refPage.title(), disambPage.title()))
if disambPage.isRedirectPage():
target = self.alternatives[0]
if pywikibot.input_yn(u'Do you want to make redirect %s point '
'to %s?' % (refPage.title(), target),
default=False, automatic_quit=False):
redir_text = '#%s [[%s]]' \
% (self.mysite.redirect(), target)
try:
refPage.put_async(redir_text, summary=self.comment)
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Page not saved: %s' % error.args)
else:
choice = pywikibot.input_choice(
u'Do you want to work on pages linking to %s?'
% refPage.title(),
[('yes', 'y'), ('no', 'n'), ('change redirect', 'c')], 'n',
automatic_quit=False)
if choice == 'y':
gen = ReferringPageGeneratorWithIgnore(
refPage, self.primary, main_only=self.main_only
)
preloadingGen = pagegenerators.PreloadingGenerator(gen)
for refPage2 in preloadingGen:
# run until the user selected 'quit'
if not self.treat(refPage2, refPage):
break
elif choice == 'c':
text = refPage.get(get_redirect=True)
include = "redirect"
except pywikibot.NoPage:
pywikibot.output(
u'Page [[%s]] does not seem to exist?! Skipping.'
% refPage.title())
include = False
if include in (True, "redirect"):
# make a backup of the original text so we can show the changes later
original_text = text
n = 0
curpos = 0
dn = False
edited = False
# This loop will run until we have finished the current page
while True:
m = self.linkR.search(text, pos=curpos)
if not m:
if n == 0:
pywikibot.output(u"No changes necessary in %s"
% refPage.title())
return True
else:
# stop loop and save page
break
# Make sure that next time around we will not find this same hit.
curpos = m.start() + 1
try:
foundlink = pywikibot.Link(m.group('title'),
disambPage.site)
foundlink.parse()
except pywikibot.Error:
continue
# ignore interwiki links
if foundlink.site != disambPage.site:
continue
# Check whether the link found is to disambPage.
try:
if foundlink.canonical_title() != disambPage.title():
continue
except pywikibot.Error:
# must be a broken link
pywikibot.log(u"Invalid link [[%s]] in page [[%s]]"
% (m.group('title'), refPage.title()))
continue
n += 1
# how many bytes should be displayed around the current link
context = 60
# check if there's a dn-template here already
if (self.dnSkip and self.dn_template_str and
self.dn_template_str[:-2] in text[m.end():m.end() +
len(self.dn_template_str) + 8]):
continue
edit = EditOption('edit page', 'e', text, m.start(), disambPage.title())
context_option = HighlightContextOption(
'more context', 'm', text, 60, start=m.start(), end=m.end())
context_option.before_question = True
options = [ListOption(self.alternatives, ''),
ListOption(self.alternatives, 'r'),
StandardOption('skip link', 's'),
edit,
StandardOption('next page', 'n'),
StandardOption('unlink', 'u')]
if self.dn_template_str:
# '?', '/' for old choice
options += [AliasOption('tag template %s' % self.dn_template_str,
['t', '?', '/'])]
options += [context_option]
if not edited:
options += [ShowPageOption('show disambiguation page', 'd',
m.start(), disambPage)]
options += [
OutputProxyOption('list', 'l',
SequenceOutputter(self.alternatives)),
AddAlternativeOption('add new', 'a',
SequenceOutputter(self.alternatives))]
if edited:
options += [StandardOption('save in this form', 'x')]
# TODO: Output context on each question
answer = pywikibot.input_choice('Option', options,
default=self.always,
force=bool(self.always))
if answer == 'x':
assert edited, 'invalid option before editing'
break
elif answer == 's':
n -= 1 # TODO what's this for?
continue
elif answer == 'e':
text = edit.new_text
edited = True
curpos = 0
continue
elif answer == 'n':
# skip this page
if self.primary:
# If run with the -primary argument, skip this
# occurrence next time.
self.primaryIgnoreManager.ignore(refPage)
return True
# The link looks like this:
# [[page_title|link_text]]trailing_chars
page_title = m.group('title')
link_text = m.group('label')
if not link_text:
# or like this: [[page_title]]trailing_chars
link_text = page_title
if m.group('section') is None:
section = ''
else:
section = m.group('section')
trailing_chars = m.group('linktrail')
if trailing_chars:
link_text += trailing_chars
if answer == 't':
assert self.dn_template_str
# small chunk of text to search
search_text = text[m.end():m.end() + context]
# figure out where the link (and sentance) ends, put note
# there
end_of_word_match = re.search(r'\s', search_text)
if end_of_word_match:
position_split = end_of_word_match.start(0)
else:
position_split = 0
# insert dab needed template
text = (text[:m.end() + position_split] +
self.dn_template_str +
text[m.end() + position_split:])
dn = True
continue
elif answer == 'u':
# unlink - we remove the section if there's any
text = text[:m.start()] + link_text + text[m.end():]
unlink_counter += 1
continue
else:
# Check that no option from above was missed
assert isinstance(answer, tuple), 'only tuple answer left.'
assert answer[0] in ['r', ''], 'only valid tuple answers.'
if answer[0] == 'r':
# we want to throw away the original link text
replaceit = link_text == page_title
elif include == "redirect":
replaceit = True
else:
replaceit = False
new_page_title = answer[1]
repPl = pywikibot.Page(pywikibot.Link(new_page_title,
disambPage.site))
if (new_page_title[0].isupper() or
link_text[0].isupper()):
new_page_title = repPl.title()
else:
new_page_title = repPl.title()
new_page_title = first_lower(new_page_title)
if new_page_title not in new_targets:
new_targets.append(new_page_title)
if replaceit and trailing_chars:
newlink = "[[%s%s]]%s" % (new_page_title,
section,
trailing_chars)
elif replaceit or (new_page_title == link_text and
not section):
newlink = "[[%s]]" % new_page_title
# check if we can create a link with trailing characters
# instead of a pipelink
elif (
(len(new_page_title) <= len(link_text)) and
(firstcap(link_text[:len(new_page_title)]) == firstcap(new_page_title)) and
(re.sub(self.trailR, '', link_text[len(new_page_title):]) == '') and
(not section)
):
newlink = "[[%s]]%s" \
% (link_text[:len(new_page_title)],
link_text[len(new_page_title):])
else:
newlink = "[[%s%s|%s]]" \
% (new_page_title, section, link_text)
text = text[:m.start()] + newlink + text[m.end():]
continue
pywikibot.output(text[max(0, m.start() - 30):m.end() + 30])
if text == original_text:
pywikibot.output(u'\nNo changes have been made:\n')
else:
pywikibot.output(u'\nThe following changes have been made:\n')
pywikibot.showDiff(original_text, text)
pywikibot.output(u'')
# save the page
self.setSummaryMessage(disambPage, new_targets, unlink_counter,
dn)
try:
refPage.put_async(text, summary=self.comment)
except pywikibot.LockedPage:
pywikibot.output(u'Page not saved: page is locked')
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Page not saved: %s' % error.args)
return True
def process_filename(self, file_url):
"""Return base filename portion of file_url."""
# Isolate the pure name
filename = file_url
# Filename may be either a URL or a local file path
if '://' in filename:
# extract the path portion of the URL
filename = urlparse(filename).path
filename = os.path.basename(filename)
if self.use_filename:
filename = self.use_filename
if self.filename_prefix:
filename = self.filename_prefix + filename
if not self.keep_filename:
pywikibot.output(
'\nThe filename on the target wiki will default to: {}\n'.
format(filename))
assert not self.opt.always
newfn = pywikibot.input(
'Enter a better name, or press enter to accept:')
if newfn != '':
filename = newfn
# FIXME: these 2 belong somewhere else, presumably in family
# forbidden characters are handled by pywikibot/page.py
forbidden = ':*?/\\' # to be extended
try:
allowed_formats = self.target_site.siteinfo.get('fileextensions',
get_default=False)
except KeyError:
allowed_formats = []
else:
allowed_formats = [item['ext'] for item in allowed_formats]
# ask until it's valid
first_check = True
while True:
if not first_check:
if self.opt.always:
filename = None
else:
filename = pywikibot.input('Enter a better name, or press '
'enter to skip the file:')
if not filename:
return None
first_check = False
ext = os.path.splitext(filename)[1].lower().strip('.')
# are any chars in forbidden also in filename?
invalid = set(forbidden) & set(filename)
if invalid:
c = ''.join(invalid)
pywikibot.output(
'Invalid character(s): {}. Please try again'.format(c))
continue
if allowed_formats and ext not in allowed_formats:
if self.opt.always:
pywikibot.output('File format is not one of '
'[{}]'.format(' '.join(allowed_formats)))
continue
if not pywikibot.input_yn(
'File format is not one of [{}], but {!r}. Continue?'.
format(' '.join(allowed_formats), ext),
default=False):
continue
potential_file_page = pywikibot.FilePage(self.target_site,
filename)
if potential_file_page.exists():
overwrite = self._handle_warning('exists')
if overwrite is False:
pywikibot.output(
'File exists and you asked to abort. Skipping.')
return None
if potential_file_page.has_permission():
if overwrite is None:
overwrite = not pywikibot.input_yn(
'File with name {} already exists. '
'Would you like to change the name? '
'(Otherwise file will be overwritten.)'.format(
filename),
default=True,
automatic_quit=False)
if not overwrite:
continue
break
pywikibot.output('File with name {} already exists and '
'cannot be overwritten.'.format(filename))
continue
with suppress(NoPageError):
if potential_file_page.file_is_shared():
pywikibot.output(
'File with name {} already exists in shared '
'repository and cannot be overwritten.'.format(
filename))
continue
break
# A proper description for the submission.
# Empty descriptions are not accepted.
if self.description:
pywikibot.output('The suggested description is:\n{}'.format(
self.description))
while not self.description or self.verify_description:
if not self.description:
pywikibot.output(
color_format(
'{lightred}It is not possible to upload a file '
'without a description.{default}'))
assert not self.opt.always
# if no description, ask if user want to add one or quit,
# and loop until one is filled.
# if self.verify_description, ask if user want to change it
# or continue.
if self.description:
question = 'Do you want to change this description?'
else:
question = 'No description was given. Add one?'
if pywikibot.input_yn(question,
default=not self.description,
automatic_quit=self.description):
from pywikibot import editor as editarticle
editor = editarticle.TextEditor()
try:
new_description = editor.edit(self.description)
except ImportError:
raise
except Exception as e:
pywikibot.error(e)
continue
# if user saved / didn't press Cancel
if new_description:
self.description = new_description
elif not self.description:
raise QuitKeyboardInterrupt
self.verify_description = False
return filename
def __init__(self, generator, templates, **kwargs):
"""
Constructor.
@param generator: the pages to work on
@type generator: iterable
@param templates: a dictionary which maps old template names to
their replacements. If remove or subst is True, it maps the
names of the templates that should be removed/resolved to None.
@type templates: dict
"""
self.availableOptions.update({
'subst': False,
'remove': False,
'summary': None,
'addedCat': None,
})
Bot.__init__(self, generator=generator, **kwargs)
self.templates = templates
# get edit summary message if it's empty
if not self.getOption('summary'):
comma = self.site.mediawiki_message('comma-separator')
params = {'list': comma.join(self.templates.keys()),
'num': len(self.templates)}
if self.getOption('remove'):
self.options['summary'] = i18n.twtranslate(
self.site, 'template-removing', params)
elif self.getOption('subst'):
self.options['summary'] = i18n.twtranslate(
self.site, 'template-substituting', params)
else:
self.options['summary'] = i18n.twtranslate(
self.site, 'template-changing', params)
replacements = []
exceptions = {}
builder = textlib._MultiTemplateMatchBuilder(self.site)
for old, new in self.templates.items():
templateRegex = builder.pattern(old)
if self.getOption('subst') and self.getOption('remove'):
replacements.append((templateRegex,
r'{{subst:%s\g<parameters>}}' % new))
exceptions['inside-tags'] = ['ref', 'gallery', 'poem',
'pagelist', ]
elif self.getOption('subst'):
replacements.append((templateRegex,
r'{{subst:%s\g<parameters>}}' % old))
exceptions['inside-tags'] = ['ref', 'gallery', 'poem',
'pagelist', ]
elif self.getOption('remove'):
separate_line_regex = re.compile(
r'^[*#:]* *{0} *\n'.format(templateRegex.pattern),
re.DOTALL | re.MULTILINE)
replacements.append((separate_line_regex, ''))
spaced_regex = re.compile(
r' +{0} +'.format(templateRegex.pattern),
re.DOTALL)
replacements.append((spaced_regex, ' '))
replacements.append((templateRegex, ''))
else:
template = pywikibot.Page(self.site, new, ns=10)
if not template.exists():
pywikibot.warning(u'Template "%s" does not exist.' % new)
if not pywikibot.input_yn('Do you want to proceed anyway?',
default=False,
automatic_quit=False):
continue
replacements.append((templateRegex,
r'{{%s\g<parameters>}}' % new))
super(TemplateRobot, self).__init__(
generator, replacements, exceptions,
always=self.getOption('always'),
addedCat=self.getOption('addedCat'),
summary=self.getOption('summary'))
def main(*args):
"""
Process command line arguments and generate user-config.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
global base_dir
# set the config family and mylang values to an invalid state so that
# the script can detect that the command line arguments -family & -lang
# were used and and handle_args has updated these config values,
# and 'force' mode can be activated below.
(config.family, config.mylang) = ('wikipedia', None)
local_args = pywikibot.handle_args(args)
if local_args:
pywikibot.output('Unknown arguments: %s' % ' '.join(local_args))
return False
if config.mylang is not None:
force = True
pywikibot.output(u'Automatically generating user-config.py')
else:
force = False
# Force default site of en.wikipedia
(config.family, config.mylang) = ('wikipedia', 'en')
username = config.usernames[config.family].get(config.mylang)
args = (config.family, config.mylang, username)
while not force or config.verbose_output:
pywikibot.output(u'\nYour default user directory is "%s"' % base_dir)
if pywikibot.input_yn("Do you want to use that directory?",
default=True, automatic_quit=False,
force=force):
break
else:
new_base = change_base_dir()
if new_base:
base_dir = new_base
break
copied_config = False
copied_fixes = False
while not force or config.verbose_output:
if os.path.exists(os.path.join(base_dir, "user-config.py")):
break
if pywikibot.input_yn(
"Do you want to copy user files from an existing Pywikibot "
"installation?",
default=False, force=force,
automatic_quit=False):
oldpath = pywikibot.input("Path to existing user-config.py?")
if not os.path.exists(oldpath):
pywikibot.error("Not a valid path")
continue
if os.path.isfile(oldpath):
# User probably typed /user-config.py at the end, so strip it
oldpath = os.path.dirname(oldpath)
if not os.path.isfile(os.path.join(oldpath, "user-config.py")):
pywikibot.error("No user_config.py found in that directory")
continue
shutil.copyfile(os.path.join(oldpath, "user-config.py"),
os.path.join(base_dir, "user-config.py"))
copied_config = True
if os.path.isfile(os.path.join(oldpath, "user-fixes.py")):
shutil.copyfile(os.path.join(oldpath, "user-fixes.py"),
os.path.join(base_dir, "user-fixes.py"))
copied_fixes = True
else:
break
if not os.path.isfile(os.path.join(base_dir, "user-config.py")):
if ((force and not config.verbose_output) or
pywikibot.input_yn('Create user-config.py file? Required for '
'running bots.',
default=True, automatic_quit=False,
force=force)):
create_user_config(args, force=force)
elif not copied_config:
pywikibot.output("user-config.py already exists in the directory")
if not os.path.isfile(os.path.join(base_dir, "user-fixes.py")):
if ((force and not config.verbose_output) or
pywikibot.input_yn('Create user-fixes.py file? Optional and '
'for advanced users.',
force=force,
default=False, automatic_quit=False)):
create_user_fixes()
elif not copied_fixes:
pywikibot.output("user-fixes.py already exists in the directory")
def treat_disamb_only(self, refPage, disambPage):
"""Resolve the links to disambPage but don't look for its redirects.
@param disambPage: the disambiguation page or redirect we don't want
anything to link to
@type disambPage: pywikibot.Page
@param refPage: a page linking to disambPage
@type refPage: pywikibot.Page
@return: "nextpage" if the user enters "n" to skip this page,
"nochange" if the page needs no change, and
"done" if the page is processed successfully
@rtype: str
"""
# TODO: break this function up into subroutines!
self.current_page = refPage
include = False
unlink_counter = 0
new_targets = []
try:
text = refPage.get()
ignoreReason = self.checkContents(text)
if ignoreReason:
pywikibot.output(
'\n\nSkipping %s because it contains %s.\n\n' %
(refPage.title(), ignoreReason))
else:
include = True
except pywikibot.IsRedirectPage:
pywikibot.output(u'%s is a redirect to %s' %
(refPage.title(), disambPage.title()))
if disambPage.isRedirectPage():
target = self.alternatives[0]
if pywikibot.input_yn(u'Do you want to make redirect %s point '
'to %s?' % (refPage.title(), target),
default=False,
automatic_quit=False):
redir_text = '#%s [[%s]]' \
% (self.mysite.redirect(), target)
try:
refPage.put(redir_text,
summary=self.comment,
asynchronous=True)
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Page not saved: %s' % error.args)
else:
choice = pywikibot.input_choice(
u'Do you want to work on pages linking to %s?' %
refPage.title(), [('yes', 'y'), ('no', 'n'),
('change redirect', 'c')],
'n',
automatic_quit=False)
if choice == 'y':
gen = ReferringPageGeneratorWithIgnore(
refPage, self.primary, main_only=self.main_only)
preloadingGen = pagegenerators.PreloadingGenerator(gen)
for refPage2 in preloadingGen:
# run until the user selected 'quit'
self.treat(refPage2, refPage)
elif choice == 'c':
text = refPage.get(get_redirect=True)
include = "redirect"
except pywikibot.NoPage:
pywikibot.output(
u'Page [[%s]] does not seem to exist?! Skipping.' %
refPage.title())
include = False
if include in (True, "redirect"):
# save the original text so we can show the changes later
original_text = text
n = 0
curpos = 0
dn = False
edited = False
# This loop will run until we have finished the current page
while True:
m = self.linkR.search(text, pos=curpos)
if not m:
if n == 0:
# No changes necessary for this disambiguation title.
return 'nochange'
else:
# stop loop and save page
break
# Ensure that next time around we will not find this same hit.
curpos = m.start() + 1
try:
foundlink = pywikibot.Link(m.group('title'),
disambPage.site)
foundlink.parse()
except pywikibot.Error:
continue
# ignore interwiki links
if foundlink.site != disambPage.site:
continue
# Check whether the link found is to disambPage.
try:
if foundlink.canonical_title() != disambPage.title():
continue
except pywikibot.Error:
# must be a broken link
pywikibot.log(u"Invalid link [[%s]] in page [[%s]]" %
(m.group('title'), refPage.title()))
continue
n += 1
# how many bytes should be displayed around the current link
context = 60
# check if there's a dn-template here already
if (self.dnSkip and self.dn_template_str
and self.dn_template_str[:-2]
in text[m.end():m.end() + len(self.dn_template_str) +
8]):
continue
edit = EditOption('edit page', 'e', text, m.start(),
disambPage.title())
context_option = HighlightContextOption('more context',
'm',
text,
60,
start=m.start(),
end=m.end())
context_option.before_question = True
options = [
ListOption(self.alternatives, ''),
ListOption(self.alternatives, 'r'),
StandardOption('skip link', 's'), edit,
StandardOption('next page', 'n'),
StandardOption('unlink', 'u')
]
if self.dn_template_str:
# '?', '/' for old choice
options += [
AliasOption('tag template %s' % self.dn_template_str,
['t', '?', '/'])
]
options += [context_option]
if not edited:
options += [
ShowPageOption('show disambiguation page', 'd',
m.start(), disambPage)
]
options += [
OutputProxyOption('list', 'l',
SequenceOutputter(self.alternatives)),
AddAlternativeOption('add new', 'a',
SequenceOutputter(self.alternatives))
]
if edited:
options += [StandardOption('save in this form', 'x')]
# TODO: Output context on each question
answer = pywikibot.input_choice('Option',
options,
default=self.always,
force=bool(self.always))
if answer == 'x':
assert edited, 'invalid option before editing'
break
elif answer == 's':
n -= 1 # TODO what's this for?
continue
elif answer == 'e':
text = edit.new_text
edited = True
curpos = 0
continue
elif answer == 'n':
# skip this page
if self.primary:
# If run with the -primary argument, skip this
# occurrence next time.
self.primaryIgnoreManager.ignore(refPage)
return 'nextpage'
# The link looks like this:
# [[page_title|link_text]]trailing_chars
page_title = m.group('title')
link_text = m.group('label')
if not link_text:
# or like this: [[page_title]]trailing_chars
link_text = page_title
if m.group('section') is None:
section = ''
else:
section = m.group('section')
trailing_chars = m.group('linktrail')
if trailing_chars:
link_text += trailing_chars
if answer == 't':
assert self.dn_template_str
# small chunk of text to search
search_text = text[m.end():m.end() + context]
# figure out where the link (and sentance) ends, put note
# there
end_of_word_match = re.search(r'\s', search_text)
if end_of_word_match:
position_split = end_of_word_match.start(0)
else:
position_split = 0
# insert dab needed template
text = (text[:m.end() + position_split] +
self.dn_template_str +
text[m.end() + position_split:])
dn = True
continue
elif answer == 'u':
# unlink - we remove the section if there's any
text = text[:m.start()] + link_text + text[m.end():]
unlink_counter += 1
continue
else:
# Check that no option from above was missed
assert isinstance(answer, tuple), 'only tuple answer left.'
assert answer[0] in ['r', ''], 'only valid tuple answers.'
if answer[0] == 'r':
# we want to throw away the original link text
replaceit = link_text == page_title
elif include == "redirect":
replaceit = True
else:
replaceit = False
new_page_title = answer[1]
repPl = pywikibot.Page(
pywikibot.Link(new_page_title, disambPage.site))
if (new_page_title[0].isupper() or link_text[0].isupper()):
new_page_title = repPl.title()
else:
new_page_title = repPl.title()
new_page_title = first_lower(new_page_title)
if new_page_title not in new_targets:
new_targets.append(new_page_title)
if replaceit and trailing_chars:
newlink = "[[%s%s]]%s" % (new_page_title, section,
trailing_chars)
elif replaceit or (new_page_title == link_text
and not section):
newlink = "[[%s]]" % new_page_title
# check if we can create a link with trailing characters
# instead of a pipelink
elif ((len(new_page_title) <= len(link_text))
and (firstcap(link_text[:len(new_page_title)])
== firstcap(new_page_title))
and (re.sub(self.trailR, '',
link_text[len(new_page_title):]) == '')
and (not section)):
newlink = "[[%s]]%s" \
% (link_text[:len(new_page_title)],
link_text[len(new_page_title):])
else:
newlink = "[[%s%s|%s]]" \
% (new_page_title, section, link_text)
text = text[:m.start()] + newlink + text[m.end():]
continue
# Todo: This line is unreachable (T155337)
pywikibot.output(text[max(0, m.start() - 30):m.end() + 30])
if text == original_text:
pywikibot.output(u'\nNo changes have been made:\n')
else:
pywikibot.output(u'\nThe following changes have been made:\n')
pywikibot.showDiff(original_text, text)
pywikibot.output(u'')
# save the page
self.setSummaryMessage(disambPage, new_targets, unlink_counter,
dn)
try:
refPage.put(text, summary=self.comment, asynchronous=True)
except pywikibot.LockedPage:
pywikibot.output(u'Page not saved: page is locked')
except pywikibot.PageNotSaved as error:
pywikibot.output(u'Page not saved: %s' % error.args)
return 'done'
def get_site_and_lang(default_family='wikipedia', default_lang='en',
default_username=None, force=False):
"""
Ask the user for the family, language and username.
@param default_family: The default family which should be chosen.
@type default_family: None or str
@param default_lang: The default language which should be chosen, if the
family supports this language.
@type default_lang: None or str
@param default_username: The default username which should be chosen.
@type default_username: None or str
@return: The family, language and username
@rtype: tuple of three str
"""
known_families = sorted(pywikibot.config2.family_files.keys())
if default_family not in known_families:
default_family = None
fam = pywikibot.bot.input_list_choice(
u"Select family of sites we are working on, "
u"just enter the number or name",
known_families,
force=force,
default=default_family)
fam = pywikibot.family.Family.load(fam)
if hasattr(fam, "langs"):
if hasattr(fam, "languages_by_size"):
by_size = [code for code in fam.languages_by_size
if code in fam.langs.keys()]
else:
by_size = []
known_langs = by_size + sorted(
set(fam.langs.keys()).difference(by_size))
else:
known_langs = []
if len(known_langs) == 0:
pywikibot.output('There were no known languages found in {0}.'.format(fam.name))
default_lang = None
elif len(known_langs) == 1:
pywikibot.output('The only known language: {0}'.format(known_langs[0]))
default_lang = known_langs[0]
else:
pywikibot.output("This is the list of known languages:")
pywikibot.output(u", ".join(known_langs))
if default_lang not in known_langs:
if default_lang != 'en' and 'en' in known_langs:
default_lang = 'en'
else:
default_lang = None
message = "The language code of the site we're working on"
mylang = None
while not mylang:
mylang = pywikibot.input(message, default=default_lang, force=force)
if known_langs and mylang and mylang not in known_langs:
if not pywikibot.input_yn("The language code {0} is not in the "
"list of known languages. Do you want "
"to continue?".format(mylang),
default=False, automatic_quit=False):
mylang = None
message = u"Username on {0}:{1}".format(mylang, fam.name)
username = pywikibot.input(message, default=default_username, force=force)
# Escape ''s
if username:
username = username.replace("'", "\\'")
return fam.name, mylang, username
def upload_image(self, debug=False):
"""Upload the image at self.url to the target wiki.
Return the filename that was used to upload the image.
If the upload fails, ask the user whether to try again or not.
If the user chooses not to retry, return null.
"""
filename = self.process_filename()
site = self.targetSite
imagepage = pywikibot.FilePage(site, filename) # normalizes filename
imagepage.text = self.description
pywikibot.output(u'Uploading file to %s via API....' % site)
try:
apiIgnoreWarnings = False
if self.ignoreWarning is True:
apiIgnoreWarnings = True
if self.uploadByUrl:
site.upload(imagepage, source_url=self.url,
ignore_warnings=apiIgnoreWarnings)
else:
if "://" in self.url:
temp = self.read_file_content()
else:
temp = self.url
site.upload(imagepage, source_filename=temp,
ignore_warnings=apiIgnoreWarnings,
chunk_size=self.chunk_size)
except pywikibot.data.api.UploadWarning as warn:
pywikibot.output(
u'We got a warning message: {0} - {1}'.format(warn.code, warn.message))
if self.abort_on_warn(warn.code):
answer = False
elif self.ignore_on_warn(warn.code):
answer = True
else:
answer = pywikibot.input_yn(u"Do you want to ignore?",
default=False, automatic_quit=False)
if answer:
self.ignoreWarning = True
self.keepFilename = True
return self.upload_image(debug)
else:
pywikibot.output(u"Upload aborted.")
return
except pywikibot.data.api.APIError as error:
if error.code == u'uploaddisabled':
pywikibot.error("Upload error: Local file uploads are disabled on %s."
% site)
else:
pywikibot.error("Upload error: ", exc_info=True)
except Exception:
pywikibot.error("Upload error: ", exc_info=True)
else:
# No warning, upload complete.
pywikibot.output(u"Upload successful.")
return filename # data['filename']
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
# the option that's always selected when the bot wonders what to do with
# a link. If it's None, the user is prompted (default behaviour).
always = None
alternatives = []
getAlternatives = True
dnSkip = False
generator = None
primary = False
main_only = False
# For sorting the linked pages, case can be ignored
minimum = 0
local_args = pywikibot.handle_args(args)
generator_factory = pagegenerators.GeneratorFactory(
positional_arg_name='page')
for arg in local_args:
if arg.startswith('-primary:'):
primary = True
getAlternatives = False
alternatives.append(arg[9:])
elif arg == '-primary':
primary = True
elif arg.startswith('-always:'):
always = arg[8:]
elif arg.startswith('-pos:'):
if arg[5] != ':':
mysite = pywikibot.Site()
page = pywikibot.Page(pywikibot.Link(arg[5:], mysite))
if page.exists():
alternatives.append(page.title())
else:
if pywikibot.input_yn(
u'Possibility %s does not actually exist. Use it '
'anyway?' % page.title(),
default=False, automatic_quit=False):
alternatives.append(page.title())
else:
alternatives.append(arg[5:])
elif arg == '-just':
getAlternatives = False
elif arg == '-dnskip':
dnSkip = True
elif arg == '-main':
main_only = True
elif arg.startswith('-min:'):
minimum = int(arg[5:])
elif arg.startswith('-start'):
try:
generator = pagegenerators.CategorizedPageGenerator(
pywikibot.Site().disambcategory(),
start=arg[7:], namespaces=[0])
except pywikibot.NoPage:
pywikibot.output("Disambiguation category for your wiki is not known.")
raise
else:
generator_factory.handleArg(arg)
site = pywikibot.Site()
generator = generator_factory.getCombinedGenerator(generator)
if not generator:
pywikibot.bot.suggest_help(missing_generator=True)
return False
site.login()
bot = DisambiguationRobot(always, alternatives, getAlternatives, dnSkip,
generator, primary, main_only,
minimum=minimum)
bot.run()
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
gen = None
notitle = False
fmt = '1'
outputlang = None
page_get = False
base_dir = None
encoding = config.textfile_encoding
# Process global args and prepare generator args parser
local_args = pywikibot.handle_args(args)
genFactory = GeneratorFactory()
for arg in local_args:
if arg == '-notitle':
notitle = True
elif arg.startswith('-format:'):
fmt = arg[len('-format:'):]
fmt = fmt.replace(u'\\03{{', u'\03{{')
elif arg.startswith('-outputlang:'):
outputlang = arg[len('-outputlang:'):]
elif arg == '-get':
page_get = True
elif arg.startswith('-save'):
base_dir = arg.partition(':')[2] or '.'
elif arg.startswith('-encode:'):
encoding = arg.partition(':')[2]
else:
genFactory.handleArg(arg)
if base_dir:
base_dir = os.path.expanduser(base_dir)
if not os.path.isabs(base_dir):
base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir))
if not os.path.exists(base_dir):
pywikibot.output(u'Directory "%s" does not exist.' % base_dir)
choice = pywikibot.input_yn(
u'Do you want to create it ("No" to continue without saving)?')
if choice:
os.makedirs(base_dir, mode=0o744)
else:
base_dir = None
elif not os.path.isdir(base_dir):
# base_dir is a file.
pywikibot.warning(u'Not a directory: "%s"\n'
u'Skipping saving ...'
% base_dir)
base_dir = None
gen = genFactory.getCombinedGenerator()
if gen:
i = 0
for i, page in enumerate(gen, start=1):
if not notitle:
page_fmt = Formatter(page, outputlang)
pywikibot.stdout(page_fmt.output(num=i, fmt=fmt))
if page_get:
try:
pywikibot.output(page.text, toStdout=True)
except pywikibot.Error as err:
pywikibot.output(err)
if base_dir:
filename = os.path.join(base_dir, page.title(as_filename=True))
pywikibot.output(u'Saving %s to %s' % (page.title(), filename))
with open(filename, mode='wb') as f:
f.write(page.text.encode(encoding))
pywikibot.output(u"%i page(s) found" % i)
else:
pywikibot.showHelp()
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: str
"""
gen = None
notitle = False
fmt = '1'
outputlang = None
page_get = False
base_dir = None
encoding = config.textfile_encoding
page_target = None
overwrite = False
summary = 'listpages-save-list'
# Process global args and prepare generator args parser
local_args = pywikibot.handle_args(args)
gen_factory = GeneratorFactory()
for arg in local_args:
option, sep, value = arg.partition(':')
if option == '-notitle':
notitle = True
elif option == '-format':
fmt = value.replace('\\03{{', '\03{{')
if not fmt.strip():
notitle = True
elif option == '-outputlang:':
outputlang = value
elif option == '-get':
page_get = True
elif option == '-save':
base_dir = value or '.'
elif option == '-encode':
encoding = value
elif option == '-put':
page_target = value
elif option == '-overwrite':
overwrite = True
elif option == '-summary':
summary = value
else:
gen_factory.handleArg(arg)
if base_dir:
base_dir = os.path.expanduser(base_dir)
if not os.path.isabs(base_dir):
base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir))
if not os.path.exists(base_dir):
pywikibot.output(
'Directory "{0}" does not exist.'.format(base_dir))
choice = pywikibot.input_yn(
'Do you want to create it ("No" to continue without saving)?')
if choice:
os.makedirs(base_dir, mode=0o744)
else:
base_dir = None
elif not os.path.isdir(base_dir):
# base_dir is a file.
pywikibot.warning('Not a directory: "{0}"\n'
'Skipping saving ...'.format(base_dir))
base_dir = None
if page_target:
site = pywikibot.Site()
page_target = pywikibot.Page(site, page_target)
if not overwrite and page_target.exists():
pywikibot.bot.suggest_help(
additional_text='Page {0} already exists.\n'
'You can use the -overwrite argument to '
'replace the content of this page.'.format(
page_target.title(as_link=True)))
return False
if re.match('^[a-z_-]+$', summary):
summary = i18n.twtranslate(site, summary)
gen = gen_factory.getCombinedGenerator()
if gen:
i = 0
output_list = []
for i, page in enumerate(gen, start=1):
if not notitle:
page_fmt = Formatter(page, outputlang)
output_list += [page_fmt.output(num=i, fmt=fmt)]
pywikibot.stdout(output_list[-1])
if page_get:
try:
pywikibot.stdout(page.text)
except pywikibot.Error as err:
pywikibot.output(err)
if base_dir:
filename = os.path.join(base_dir, page.title(as_filename=True))
pywikibot.output('Saving {0} to {1}'.format(
page.title(), filename))
with open(filename, mode='wb') as f:
f.write(page.text.encode(encoding))
pywikibot.output('{0} page(s) found'.format(i))
if page_target:
page_target.text = '\n'.join(output_list)
page_target.save(summary=summary)
return True
else:
pywikibot.bot.suggest_help(missing_generator=True)
return False
def run_bot(give_url, image_url, desc):
"""Run the bot."""
url = give_url
image_url = ''
if url == '':
if image_url:
url = pywikibot.input(u"What URL range should I check "
u"(use $ for the part that is changeable)")
else:
url = pywikibot.input(u"From what URL should I get the images?")
if image_url:
minimum = 1
maximum = 99
answer = pywikibot.input(
u"What is the first number to check (default: 1)")
if answer:
minimum = int(answer)
answer = pywikibot.input(
u"What is the last number to check (default: 99)")
if answer:
maximum = int(answer)
if not desc:
basicdesc = pywikibot.input(
u"What text should be added at the end of "
u"the description of each image from this url?")
else:
basicdesc = desc
if image_url:
ilinks = []
i = minimum
while i <= maximum:
ilinks += [url.replace("$", str(i))]
i += 1
else:
ilinks = get_imagelinks(url)
for image in ilinks:
if pywikibot.input_yn('Include image %s?' % image,
default=False,
automatic_quit=False):
desc = pywikibot.input(u"Give the description of this image:")
categories = []
while True:
cat = pywikibot.input(u"Specify a category (or press enter to "
u"end adding categories)")
if not cat.strip():
break
if ":" in cat:
categories.append(u"[[%s]]" % cat)
else:
categories.append(u"[[%s:%s]]" %
(mysite.namespace(14), cat))
desc += "\r\n\r\n" + basicdesc + "\r\n\r\n" + \
"\r\n".join(categories)
uploadBot = UploadRobot(image, description=desc)
uploadBot.run()
elif answer == 's':
break
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
# the option that's always selected when the bot wonders what to do with
# a link. If it's None, the user is prompted (default behaviour).
always = None
alternatives = []
getAlternatives = True
dnSkip = False
generator = None
primary = False
main_only = False
# For sorting the linked pages, case can be ignored
minimum = 0
local_args = pywikibot.handle_args(args)
generator_factory = pagegenerators.GeneratorFactory(
positional_arg_name='page')
for arg in local_args:
if arg.startswith('-primary:'):
primary = True
getAlternatives = False
alternatives.append(arg[9:])
elif arg == '-primary':
primary = True
elif arg.startswith('-always:'):
always = arg[8:]
elif arg.startswith('-pos:'):
if arg[5] != ':':
mysite = pywikibot.Site()
page = pywikibot.Page(pywikibot.Link(arg[5:], mysite))
if page.exists():
alternatives.append(page.title())
else:
if pywikibot.input_yn(
u'Possibility %s does not actually exist. Use it '
'anyway?' % page.title(),
default=False,
automatic_quit=False):
alternatives.append(page.title())
else:
alternatives.append(arg[5:])
elif arg == '-just':
getAlternatives = False
elif arg == '-dnskip':
dnSkip = True
elif arg == '-main':
main_only = True
elif arg.startswith('-min:'):
minimum = int(arg[5:])
elif arg.startswith('-start'):
try:
generator = pagegenerators.CategorizedPageGenerator(
pywikibot.Site().disambcategory(),
start=arg[7:],
namespaces=[0])
except pywikibot.NoPage:
pywikibot.output(
'Disambiguation category for your wiki is not known.')
raise
else:
generator_factory.handleArg(arg)
site = pywikibot.Site()
generator = generator_factory.getCombinedGenerator(generator)
if not generator:
pywikibot.bot.suggest_help(missing_generator=True)
return False
site.login()
bot = DisambiguationRobot(always,
alternatives,
getAlternatives,
dnSkip,
generator,
primary,
main_only,
minimum=minimum)
bot.run()
def __init__(self, generator, templates, **kwargs):
"""
Constructor.
@param generator: the pages to work on
@type generator: iterable
@param templates: a dictionary which maps old template names to
their replacements. If remove or subst is True, it maps the
names of the templates that should be removed/resolved to None.
@type templates: dict
"""
self.availableOptions.update({
'subst': False,
'remove': False,
'summary': None,
'addedCat': None,
})
Bot.__init__(self, generator=generator, **kwargs)
self.templates = templates
# get edit summary message if it's empty
if not self.getOption('summary'):
comma = self.site.mediawiki_message('comma-separator')
params = {'list': comma.join(self.templates.keys()),
'num': len(self.templates)}
site = self.site
if self.getOption('remove'):
self.options['summary'] = i18n.twntranslate(
site, 'template-removing', params)
elif self.getOption('subst'):
self.options['summary'] = i18n.twntranslate(
site, 'template-substituting', params)
else:
self.options['summary'] = i18n.twntranslate(
site, 'template-changing', params)
# regular expression to find the original template.
# {{vfd}} does the same thing as {{Vfd}}, so both will be found.
# The old syntax, {{msg:vfd}}, will also be found.
# The group 'parameters' will either match the parameters, or an
# empty string if there are none.
replacements = []
exceptions = {}
namespace = self.site.namespaces[10]
for old, new in self.templates.items():
if namespace.case == 'first-letter':
pattern = '[' + \
re.escape(old[0].upper()) + \
re.escape(old[0].lower()) + \
']' + re.escape(old[1:])
else:
pattern = re.escape(old)
pattern = re.sub(r'_|\\ ', r'[_ ]', pattern)
templateRegex = re.compile(r'\{\{ *(' + ':|'.join(namespace) +
r':|[mM][sS][gG]:)?' + pattern +
r'(?P<parameters>\s*\|.+?|) *}}',
re.DOTALL)
if self.getOption('subst') and self.getOption('remove'):
replacements.append((templateRegex,
r'{{subst:%s\g<parameters>}}' % new))
exceptions['inside-tags'] = ['ref', 'gallery']
elif self.getOption('subst'):
replacements.append((templateRegex,
r'{{subst:%s\g<parameters>}}' % old))
exceptions['inside-tags'] = ['ref', 'gallery']
elif self.getOption('remove'):
replacements.append((templateRegex, ''))
else:
template = pywikibot.Page(self.site, new, ns=10)
if not template.exists():
pywikibot.warning(u'Template "%s" does not exist.' % new)
if not pywikibot.input_yn('Do you want to proceed anyway?',
default=False, automatic_quit=False):
continue
replacements.append((templateRegex,
r'{{%s\g<parameters>}}' % new))
super(TemplateRobot, self).__init__(
generator, replacements, exceptions,
always=self.getOption('always'),
addedCat=self.getOption('addedCat'),
summary=self.getOption('summary'))
def treat(self, page):
"""It loads the given page, does some changes, and saves it."""
choice = False
try:
# page: title, date, username, comment, loginfo, rcid, token
username = page['user']
# when the feed isnt from the API, it used to contain
# '(not yet written)' or '(page does not exist)' when it was
# a redlink
rcid = page['rcid']
title = page['title']
if not rcid:
raise Exception('rcid not present')
# check whether we have wrapped around to higher rcids
# which indicates a new RC feed is being processed
if rcid > self.last_rcid:
# refresh the whitelist
self.load_whitelist()
self.repeat_start_ts = time.time()
if pywikibot.config.verbose_output or self.getOption('ask'):
pywikibot.output(u'User %s has created or modified page %s' %
(username, title))
if self.getOption('autopatroluserns') and (page['ns'] == 2
or page['ns'] == 3):
# simple rule to whitelist any user editing their own userspace
if title.partition(':')[2].split('/')[0].startswith(username):
if pywikibot.config.verbose_output:
pywikibot.output(u'%s is whitelisted to modify %s' %
(username, title))
choice = True
if not choice and username in self.whitelist:
if self.in_list(self.whitelist[username], title):
if pywikibot.config.verbose_output:
pywikibot.output(u'%s is whitelisted to modify %s' %
(username, title))
choice = True
if self.getOption('ask'):
choice = pywikibot.input_yn(
'Do you want to mark page as patrolled?')
# Patrol the page
if choice:
# list() iterates over patrol() which returns a generator
list(self.site.patrol(rcid))
self.patrol_counter = self.patrol_counter + 1
pywikibot.output(u'Patrolled %s (rcid %d) by user %s' %
(title, rcid, username))
else:
if pywikibot.config.verbose_output:
pywikibot.output(u'Skipped')
if rcid > self.highest_rcid:
self.highest_rcid = rcid
self.last_rcid = rcid
self.rc_item_counter = self.rc_item_counter + 1
except pywikibot.NoPage:
pywikibot.output(u'Page %s does not exist; skipping.' %
title(asLink=True))
except pywikibot.IsRedirectPage:
pywikibot.output(u'Page %s is a redirect; skipping.' %
title(asLink=True))
def run(self):
"""Run the bot."""
commons = self.commons
comment = self.summary
for page in self.generator:
self.current_page = page
try:
localImagePage = pywikibot.FilePage(self.site, page.title())
if localImagePage.fileIsShared():
pywikibot.output(u'File is already on Commons.')
continue
sha1 = localImagePage.latest_file_info.sha1
filenameOnCommons = self.findFilenameOnCommons(localImagePage)
if not filenameOnCommons:
pywikibot.output(u'NowCommons template not found.')
continue
commonsImagePage = pywikibot.FilePage(
commons, 'Image:%s' % filenameOnCommons)
if (localImagePage.title(withNamespace=False) !=
commonsImagePage.title(withNamespace=False)):
usingPages = list(localImagePage.usingPages())
if usingPages and usingPages != [localImagePage]:
pywikibot.output(
color_format(
'"{lightred}{0}{default}" is still used in {1} pages.',
localImagePage.title(withNamespace=False),
len(usingPages)))
if self.getOption('replace') is True:
pywikibot.output(
color_format(
'Replacing "{lightred}{0}{default}" by '
'"{lightgreen}{1}{default}\".',
localImagePage.title(withNamespace=False),
commonsImagePage.title(
withNamespace=False)))
bot = ImageBot(
pg.FileLinksGenerator(localImagePage),
localImagePage.title(withNamespace=False),
commonsImagePage.title(withNamespace=False),
'', self.getOption('replacealways'),
self.getOption('replaceloose'))
bot.run()
# If the image is used with the urlname the
# previous function won't work
is_used = bool(
list(
pywikibot.FilePage(
self.site,
page.title()).usingPages(total=1)))
if is_used and self.getOption('replaceloose'):
bot = ImageBot(
pg.FileLinksGenerator(localImagePage),
localImagePage.title(withNamespace=False,
asUrl=True),
commonsImagePage.title(
withNamespace=False), '',
self.getOption('replacealways'),
self.getOption('replaceloose'))
bot.run()
# refresh because we want the updated list
usingPages = len(
list(
pywikibot.FilePage(
self.site, page.title()).usingPages()))
else:
pywikibot.output(u'Please change them manually.')
continue
else:
pywikibot.output(
color_format(
'No page is using "{lightgreen}{0}{default}" '
'anymore.',
localImagePage.title(withNamespace=False)))
commonsText = commonsImagePage.get()
if self.getOption('replaceonly') is False:
if sha1 == commonsImagePage.latest_file_info.sha1:
pywikibot.output(
u'The image is identical to the one on Commons.')
if len(localImagePage.getFileVersionHistory()) > 1:
pywikibot.output(
'This image has a version history. Please '
'delete it manually after making sure that the '
'old versions are not worth keeping.')
continue
if self.getOption('always') is False:
format_str = color_format(
'\n\n>>>> Description on {lightpurple}%s'
'{default} <<<<\n')
pywikibot.output(format_str % page.title())
pywikibot.output(localImagePage.get())
pywikibot.output(format_str %
commonsImagePage.title())
pywikibot.output(commonsText)
if pywikibot.input_yn(
u'Does the description on Commons contain '
'all required source and license\n'
'information?',
default=False,
automatic_quit=False):
localImagePage.delete(
'%s [[:commons:Image:%s]]' %
(comment, filenameOnCommons),
prompt=False)
else:
localImagePage.delete(
comment +
' [[:commons:Image:%s]]' % filenameOnCommons,
prompt=False)
else:
pywikibot.output(
u'The image is not identical to the one on Commons.'
)
except (pywikibot.NoPage, pywikibot.IsRedirectPage) as e:
pywikibot.output(u'%s' % e[0])
continue
else:
self._treat_counter += 1
if not self._treat_counter:
pywikibot.output('No transcluded files found for %s.' %
self.ncTemplates()[0])
self.exit()
def add_template(self, source, dest, task, fromsite):
"""Place or remove the Link_GA/FA template on/from a page."""
def compile_link(site, templates):
"""Compile one link template list."""
findtemplate = '(%s)' % '|'.join(templates)
return re.compile(
r"\{\{%s\|%s\}\}" %
(findtemplate.replace(u' ', u'[ _]'), site.code),
re.IGNORECASE)
tosite = dest.site
add_tl, remove_tl = self.getTemplateList(tosite.code, task)
re_link_add = compile_link(fromsite, add_tl)
re_link_remove = compile_link(fromsite, remove_tl)
text = dest.text
m1 = add_tl and re_link_add.search(text)
m2 = remove_tl and re_link_remove.search(text)
changed = False
interactive = self.getOption('interactive')
if add_tl:
if m1:
pywikibot.output(u"(already added)")
else:
# insert just before interwiki
if (not interactive or
pywikibot.input_yn(u'Connecting %s -> %s. Proceed?' %
(source.title(), dest.title()),
default=False,
automatic_quit=False)):
if self.getOption('side'):
# Placing {{Link FA|xx}} right next to
# corresponding interwiki
text = (text[:m1.end()] + u" {{%s|%s}}" %
(add_tl[0], fromsite.code) + text[m1.end():])
else:
# Moving {{Link FA|xx}} to top of interwikis
iw = textlib.getLanguageLinks(text, tosite)
text = textlib.removeLanguageLinks(text, tosite)
text += u"%s{{%s|%s}}%s" % (config.LS, add_tl[0],
fromsite.code, config.LS)
text = textlib.replaceLanguageLinks(text, iw, tosite)
changed = True
if remove_tl:
if m2:
if (changed or # Don't force the user to say "Y" twice
not interactive or
pywikibot.input_yn(u'Connecting %s -> %s. Proceed?' %
(source.title(), dest.title()),
default=False,
automatic_quit=False)):
text = re.sub(re_link_remove, '', text)
changed = True
elif task == 'former':
pywikibot.output(u"(already removed)")
if changed:
comment = i18n.twtranslate(tosite, 'featured-' + task,
{'page': source})
try:
dest.put(text, comment)
self._save_counter += 1
except pywikibot.LockedPage:
pywikibot.output(u'Page %s is locked!' % dest.title())
except pywikibot.PageNotSaved:
pywikibot.output(u"Page not saved")
def process_filename(self, file_url=None):
"""Return base filename portion of file_url."""
if not file_url:
file_url = self.url
pywikibot.warning("file_url is not given. "
"Set to self.url by default.")
always = self.getOption('always')
# Isolate the pure name
filename = file_url
# Filename may be either a URL or a local file path
if "://" in filename:
# extract the path portion of the URL
filename = urlparse(filename).path
filename = os.path.basename(filename)
if self.useFilename:
filename = self.useFilename
if not self.keepFilename:
pywikibot.output(
u"The filename on the target wiki will default to: %s"
% filename)
assert not always
newfn = pywikibot.input(
u'Enter a better name, or press enter to accept:')
if newfn != "":
filename = newfn
# FIXME: these 2 belong somewhere else, presumably in family
# forbidden characters are handled by pywikibot/page.py
forbidden = ':*?/\\' # to be extended
try:
allowed_formats = self.targetSite.siteinfo.get(
'fileextensions', get_default=False)
except KeyError:
allowed_formats = []
else:
allowed_formats = [item['ext'] for item in allowed_formats]
# ask until it's valid
first_check = True
while True:
if not first_check:
if always:
filename = None
else:
filename = pywikibot.input('Enter a better name, or press '
'enter to skip the file:')
if not filename:
return None
first_check = False
ext = os.path.splitext(filename)[1].lower().strip('.')
# are any chars in forbidden also in filename?
invalid = set(forbidden) & set(filename)
if invalid:
c = "".join(invalid)
pywikibot.output(
'Invalid character(s): %s. Please try again' % c)
continue
if allowed_formats and ext not in allowed_formats:
if always:
pywikibot.output('File format is not one of '
'[{0}]'.format(' '.join(allowed_formats)))
continue
elif not pywikibot.input_yn(
u"File format is not one of [%s], but %s. Continue?"
% (u' '.join(allowed_formats), ext),
default=False, automatic_quit=False):
continue
potential_file_page = pywikibot.FilePage(self.targetSite, filename)
if potential_file_page.exists():
overwrite = self._handle_warning('exists')
if overwrite is False:
pywikibot.output(
'File exists and you asked to abort. Skipping.')
return None
if potential_file_page.canBeEdited():
if overwrite is None:
overwrite = not pywikibot.input_yn(
"File with name %s already exists. "
"Would you like to change the name? "
"(Otherwise file will be overwritten.)"
% filename, default=True,
automatic_quit=False)
if not overwrite:
continue
else:
break
else:
pywikibot.output(u"File with name %s already exists and "
"cannot be overwritten." % filename)
continue
else:
try:
if potential_file_page.fileIsShared():
pywikibot.output(
'File with name %s already exists in shared '
'repository and cannot be overwritten.' % filename)
continue
else:
break
except pywikibot.NoPage:
break
# A proper description for the submission.
# Empty descriptions are not accepted.
if self.description:
pywikibot.output('The suggested description is:\n%s'
% self.description)
while not self.description or self.verifyDescription:
if not self.description:
pywikibot.output(color_format(
'{lightred}It is not possible to upload a file '
'without a description.{default}'))
assert not always
# if no description, ask if user want to add one or quit,
# and loop until one is filled.
# if self.verifyDescription, ask if user want to change it
# or continue.
if self.description:
question = 'Do you want to change this description?'
else:
question = 'No description was given. Add one?'
if pywikibot.input_yn(question, default=not self.description,
automatic_quit=self.description):
from pywikibot import editor as editarticle
editor = editarticle.TextEditor()
try:
newDescription = editor.edit(self.description)
except ImportError:
raise
except Exception as e:
pywikibot.error(e)
continue
# if user saved / didn't press Cancel
if newDescription:
self.description = newDescription
elif not self.description:
raise QuitKeyboardInterrupt
self.verifyDescription = False
return filename
def __init__(self, generator, templates: dict, **kwargs) -> None:
"""
Initializer.
:param generator: the pages to work on
:type generator: iterable
:param templates: a dictionary which maps old template names to
their replacements. If remove or subst is True, it maps the
names of the templates that should be removed/resolved to None.
"""
SingleSiteBot.__init__(self, **kwargs)
self.templates = templates
# get edit summary message if it's empty
if not self.opt.summary:
comma = self.site.mediawiki_message('comma-separator')
params = {'list': comma.join(self.templates.keys()),
'num': len(self.templates)}
if self.opt.remove:
tw_key = 'template-removing'
elif self.opt.subst:
tw_key = 'template-substituting'
else:
tw_key = 'template-changing'
self.opt.summary = i18n.twtranslate(self.site, tw_key, params)
replacements = []
exceptions = {}
builder = textlib.MultiTemplateMatchBuilder(self.site)
for old, new in self.templates.items():
template_regex = builder.pattern(old)
if self.opt.subst and self.opt.remove:
replacements.append((template_regex,
r'{{subst:%s\g<parameters>}}' % new))
exceptions['inside-tags'] = ['ref', 'gallery', 'poem',
'pagelist', ]
elif self.opt.subst:
replacements.append(
(template_regex, r'{{%s:%s\g<parameters>}}' %
(self.opt.subst, old)))
exceptions['inside-tags'] = ['ref', 'gallery', 'poem',
'pagelist', ]
elif self.opt.remove:
separate_line_regex = re.compile(
r'^[*#:]* *{} *\n'.format(template_regex.pattern),
re.DOTALL | re.MULTILINE)
replacements.append((separate_line_regex, ''))
spaced_regex = re.compile(
r' +{} +'.format(template_regex.pattern),
re.DOTALL)
replacements.append((spaced_regex, ' '))
replacements.append((template_regex, ''))
else:
template = pywikibot.Page(self.site, new, ns=10)
if not template.exists():
pywikibot.warning('Template "{}" does not exist.'
.format(new))
if not pywikibot.input_yn('Do you want to proceed anyway?',
default=False,
automatic_quit=False):
continue
replacements.append((template_regex,
r'{{%s\g<parameters>}}' % new))
super().__init__(
generator, replacements, exceptions,
always=self.opt.always,
addcat=self.opt.addcat,
summary=self.opt.summary)
def get_site_and_lang(default_family='wikipedia',
default_lang='en',
default_username=None):
"""
Ask the user for the family, language and username.
@param default_family: The default family which should be chosen.
@type default_family: None or str
@param default_lang: The default language which should be chosen, if the
family supports this language.
@type default_lang: None or str
@param default_username: The default username which should be chosen.
@type default_username: None or str
@return: The family, language and username
@rtype: tuple of three str
"""
known_families = sorted(pywikibot.config2.family_files.keys())
if default_family not in known_families:
default_family = None
fam = listchoice(known_families,
u"Select family of sites we are working on, "
u"just enter the number or name",
default=default_family)
fam = pywikibot.family.Family.load(fam)
if hasattr(fam, "langs"):
if hasattr(fam, "languages_by_size"):
key = {lang: i for i, lang in enumerate(fam.languages_by_size)}.get
by_size = sorted(set(fam.langs.keys()).intersection(
fam.languages_by_size),
key=key)
else:
by_size = []
known_langs = by_size + sorted(
set(fam.langs.keys()).difference(by_size))
else:
known_langs = []
if len(known_langs) == 0:
pywikibot.output('There were no known languages found in {0}.'.format(
fam.name))
default_lang = None
elif len(known_langs) == 1:
pywikibot.output('The only known language: {0}'.format(known_langs[0]))
default_lang = known_langs[0]
else:
pywikibot.output("This is the list of known languages:")
pywikibot.output(u", ".join(known_langs))
if default_lang not in known_langs:
if default_lang != 'en' and 'en' in known_langs:
default_lang = 'en'
else:
default_lang = None
message = "The language code of the site we're working on"
if default_lang:
message += " (default: '{0}')".format(default_lang)
message += ":"
mylang = None
while not mylang:
mylang = pywikibot.input(message) or default_lang
if known_langs and mylang and mylang not in known_langs:
if not pywikibot.input_yn("The language code {0} is not in the "
"list of known languages. Do you want "
"to continue?".format(mylang),
default=False,
automatic_quit=False):
mylang = None
username = None
message = u"Username on {0}:{1}".format(mylang, fam.name)
if default_username:
message += " (default: '{0}')".format(default_username)
message += ":"
while not username:
username = pywikibot.input(message) or default_username
if not username:
pywikibot.error('The username may not be empty.')
if sys.version_info == 2:
username = username.decode(console_encoding)
# Escape ''s
username = username.replace("'", "\\'")
return fam.name, mylang, username
