def useHashGenerator(self): """Use hash generator.""" # https://toolserver.org/~multichill/nowcommons.php?language=it&page=2&filter= lang = self.site.lang num_page = 0 word_to_skip_translated = i18n.translate(self.site, word_to_skip) images_processed = list() while 1: url = ('https://toolserver.org/~multichill/nowcommons.php?' 'language=%s&page=%s&filter=') % (lang, num_page) HTML_text = self.site.getUrl(url, no_hostname=True) reg = r'<[Aa] href="(?P<urllocal>.*?)">(?P<imagelocal>.*?)</[Aa]> +?</td><td>\n\s*?' reg += r'<[Aa] href="(?P<urlcommons>http[s]?://commons.wikimedia.org/.*?)" \ >Image:(?P<imagecommons>.*?)</[Aa]> +?</td><td>' regex = re.compile(reg, re.UNICODE) found_something = False change_page = True for x in regex.finditer(HTML_text): found_something = True image_local = x.group('imagelocal') image_commons = x.group('imagecommons') if image_local in images_processed: continue change_page = False images_processed.append(image_local) # Skip images that have something in the title (useful for it.wiki) image_to_skip = False for word in word_to_skip_translated: if word.lower() in image_local.lower(): image_to_skip = True if image_to_skip: continue url_local = x.group('urllocal') url_commons = x.group('urlcommons') pywikibot.output(color_format( '\n\n>>> {lightpurple}{0}{default} <<<', image_local)) pywikibot.output(u'Local: %s\nCommons: %s\n' % (url_local, url_commons)) webbrowser.open(url_local, 0, 1) webbrowser.open(url_commons, 0, 1) if image_local.split('Image:')[1] == image_commons: choice = pywikibot.input_yn( u'The local and the commons images have the same name, ' 'continue?', default=False, automatic_quit=False) else: choice = pywikibot.input_yn( u'Are the two images equal?', default=False, automatic_quit=False) if choice: yield [image_local, image_commons] else: continue # The page is dinamically updated, so we may don't need to change it if change_page: num_page += 1 # If no image found means that there aren't anymore, break. if not found_something: break
def create_user_config(): """Create a user-config.py in base_dir.""" _fnc = os.path.join(base_dir, "user-config.py") if not file_exists(_fnc): main_family, main_lang, main_username = get_site_and_lang() usernames = [(main_family, main_lang, main_username)] while pywikibot.input_yn("Do you want to add any other projects?", default=False, automatic_quit=False): usernames += [get_site_and_lang(main_family, main_lang, main_username)] usernames = '\n'.join( u"usernames['{0}']['{1}'] = u'{2}'".format(*username) for username in usernames) extended = pywikibot.input_yn("Would you like the extended version of " "user-config.py, with explanations " "included?", automatic_quit=False) if extended: # config2.py will be in the pywikibot/ directory relative to this # script (generate_user_files) install = os.path.dirname(os.path.abspath(__file__)) with codecs.open(os.path.join(install, "pywikibot", "config2.py"), "r", "utf-8") as config_f: config = config_f.read() res = re.findall("^(############## (?:" "LOGFILE|" "INTERWIKI|" "SOLVE_DISAMBIGUATION|" "IMAGE RELATED|" "TABLE CONVERSION BOT|" "WEBLINK CHECKER|" "DATABASE|" "SEARCH ENGINE|" "COPYRIGHT|" "FURTHER" ") SETTINGS .*?)^(?=#####|# =====)", config, re.MULTILINE | re.DOTALL) config_text = '\n'.join(res) config_content = EXTENDED_CONFIG else: config_content = SMALL_CONFIG with codecs.open(_fnc, "w", "utf-8") as f: f.write(config_content.format(**locals())) pywikibot.output(u"'%s' written." % _fnc)
def save(self, text, page, comment=None, minorEdit=True, botflag=True): """ Update the given page with new text. """ # only save if something was changed if text != page.get(): # Show the title of the page we're working on. # Highlight the title in purple. pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<" % page.title()) # show what was changed pywikibot.showDiff(page.get(), text) pywikibot.output(u'Comment: %s' % comment) if not self.dry: if pywikibot.input_yn( u'Do you want to accept these changes?', default=False, automatic_quit=False): try: page.text = text # Save the page page.save(comment=comment or self.comment, minor=minorEdit, botflag=botflag) except pywikibot.LockedPage: pywikibot.output(u"Page %s is locked; skipping." % page.title(asLink=True)) except pywikibot.EditConflict: pywikibot.output( u'Skipping %s because of edit conflict' % (page.title())) except pywikibot.SpamfilterError as error: pywikibot.output( u'Cannot change %s because of spam blacklist entry %s' % (page.title(), error.url)) else: return True return False
def main(give_url, image_url, desc): """Run the bot.""" url = give_url image_url = '' if url == '': if image_url: url = pywikibot.input(u"What URL range should I check " u"(use $ for the part that is changeable)") else: url = pywikibot.input(u"From what URL should I get the images?") if image_url: minimum = 1 maximum = 99 answer = pywikibot.input( u"What is the first number to check (default: 1)") if answer: minimum = int(answer) answer = pywikibot.input( u"What is the last number to check (default: 99)") if answer: maximum = int(answer) if not desc: basicdesc = pywikibot.input( u"What text should be added at the end of " u"the description of each image from this url?") else: basicdesc = desc if image_url: ilinks = [] i = minimum while i <= maximum: ilinks += [url.replace("$", str(i))] i += 1 else: ilinks = get_imagelinks(url) for image in ilinks: if pywikibot.input_yn('Include image %s?' % image, default=False, automatic_quit=False): desc = pywikibot.input(u"Give the description of this image:") categories = [] while True: cat = pywikibot.input(u"Specify a category (or press enter to " u"end adding categories)") if not cat.strip(): break if ":" in cat: categories.append(u"[[%s]]" % cat) else: categories.append(u"[[%s:%s]]" % (mysite.namespace(14), cat)) desc += "\r\n\r\n" + basicdesc + "\r\n\r\n" + \ "\r\n".join(categories) uploadBot = UploadRobot(image, description=desc) uploadBot.run() elif answer == 's': break
def upload_file(self, file_url, debug=False): """Upload the image at file_url to the target wiki. Return the filename that was used to upload the image. If the upload fails, ask the user whether to try again or not. If the user chooses not to retry, return null. """ filename = self.process_filename(file_url) if not filename: return None site = self.targetSite imagepage = pywikibot.FilePage(site, filename) # normalizes filename imagepage.text = self.description pywikibot.output("Uploading file to %s via API..." % site) try: apiIgnoreWarnings = False if self.ignoreWarning is True: apiIgnoreWarnings = True if self.uploadByUrl: site.upload(imagepage, source_url=file_url, ignore_warnings=apiIgnoreWarnings) else: if "://" in file_url: temp = self.read_file_content(file_url) else: temp = file_url site.upload( imagepage, source_filename=temp, ignore_warnings=apiIgnoreWarnings, chunk_size=self.chunk_size ) except pywikibot.data.api.UploadWarning as warn: pywikibot.output("We got a warning message: {0} - {1}".format(warn.code, warn.message)) if self.abort_on_warn(warn.code): answer = False elif self.ignore_on_warn(warn.code): answer = True else: answer = pywikibot.input_yn("Do you want to ignore?", default=False, automatic_quit=False) if answer: self.ignoreWarning = True self.keepFilename = True return self.upload_file(file_url, debug) else: pywikibot.output("Upload aborted.") return except pywikibot.data.api.APIError as error: if error.code == "uploaddisabled": pywikibot.error("Upload error: Local file uploads are disabled on %s." % site) else: pywikibot.error("Upload error: ", exc_info=True) except Exception: pywikibot.error("Upload error: ", exc_info=True) else: # No warning, upload complete. pywikibot.output("Upload of %s successful." % filename) return filename # data['filename']
def get_dest(self): self.dest = u'%s-core.%s' % tuple(self.source.rsplit(u'.', 1)) if not self.warnonly and not pywikibot.input_yn( u'Destination file is %s.' % self.dest, default=True, automatic_quit=False): pywikibot.output('Quitting...') exit()
def save(self, text, page, comment, minorEdit=True, botflag=True): """Save the text.""" if text != page.text: # Show the title of the page we're working on. # Highlight the title in purple. pywikibot.output(color_format( '\n\n>>> {lightpurple}{0}{default} <<<', page.title())) # show what was changed pywikibot.showDiff(page.get(), text) pywikibot.output(u'Comment: %s' % comment) if not self.dry: if pywikibot.input_yn( u'Do you want to accept these changes?', default=False, automatic_quit=False): page.text = text try: # Save the page page.save(summary=comment, minorEdit=minorEdit, botflag=botflag) except pywikibot.LockedPage: pywikibot.output(u"Page %s is locked; skipping." % page.title(asLink=True)) except pywikibot.EditConflict: pywikibot.output( u'Skipping %s because of edit conflict' % (page.title())) except pywikibot.SpamfilterError as error: pywikibot.output( u'Cannot change %s because of spam blacklist entry ' u'%s' % (page.title(), error.url)) else: return True
def treat(self, page): """It loads the given page, does some changes, and saves it.""" choice = False try: # page: title, date, username, comment, loginfo, rcid, token username = page["user"] # when the feed isnt from the API, it used to contain # '(not yet written)' or '(page does not exist)' when it was # a redlink rcid = page["rcid"] title = page["title"] if not rcid: raise Exception("rcid not present") # check whether we have wrapped around to higher rcids # which indicates a new RC feed is being processed if rcid > self.last_rcid: # refresh the whitelist self.load_whitelist() self.repeat_start_ts = time.time() if pywikibot.config.verbose_output or self.getOption("ask"): pywikibot.output("User %s has created or modified page %s" % (username, title)) if self.getOption("autopatroluserns") and (page["ns"] == 2 or page["ns"] == 3): # simple rule to whitelist any user editing their own userspace if title.partition(":")[2].split("/")[0].startswith(username): if pywikibot.config.verbose_output: pywikibot.output("%s is whitelisted to modify %s" % (username, title)) choice = True if not choice and username in self.whitelist: if self.in_list(self.whitelist[username], title): if pywikibot.config.verbose_output: pywikibot.output("%s is whitelisted to modify %s" % (username, title)) choice = True if self.getOption("ask"): choice = pywikibot.input_yn("Do you want to mark page as patrolled?", automatic_quit=False) # Patrol the page if choice: # list() iterates over patrol() which returns a generator list(self.site.patrol(rcid)) self.patrol_counter = self.patrol_counter + 1 pywikibot.output("Patrolled %s (rcid %d) by user %s" % (title, rcid, username)) else: if pywikibot.config.verbose_output: pywikibot.output("Skipped") if rcid > self.highest_rcid: self.highest_rcid = rcid self.last_rcid = rcid self.rc_item_counter = self.rc_item_counter + 1 except pywikibot.NoPage: pywikibot.output("Page %s does not exist; skipping." % title(asLink=True)) except pywikibot.IsRedirectPage: pywikibot.output("Page %s is a redirect; skipping." % title(asLink=True))
def main(*args): """ Process command line arguments and generate user-config. If args is an empty list, sys.argv is used. @param args: command line arguments @type args: list of unicode """ global base_dir # set the config family and mylang values to an invalid state so that # the script can detect that the command line arguments -family & -lang # were used and and handle_args has updated these config values, # and 'force' mode can be activated below. (config.family, config.mylang) = ('wikipedia', None) pywikibot.output('You can abort at any time by pressing ctrl-c') local_args = pywikibot.handle_args(args) if local_args: pywikibot.output('Unknown arguments: %s' % ' '.join(local_args)) return False if config.mylang is not None: force = True pywikibot.output(u'Automatically generating user-config.py') else: force = False # Force default site of en.wikipedia (config.family, config.mylang) = ('wikipedia', 'en') username = config.usernames[config.family].get(config.mylang) args = (config.family, config.mylang, username) # Only give option for directory change if user-config.py already exists # in the directory. This will repeat if user-config.py also exists in # the requested directory. if not force or config.verbose_output: pywikibot.output(u'\nYour default user directory is "%s"' % base_dir) while os.path.isfile(os.path.join(base_dir, "user-config.py")): pywikibot.output('user-config.py already exists' ' in the target directory.') if pywikibot.input_yn('Would you like to change the directory?', default=True, automatic_quit=False, force=force): new_base = change_base_dir() if new_base: base_dir = new_base else: break # user-fixes.py also used to be created here, but has # been replaced by an example file. if not os.path.isfile(os.path.join(base_dir, "user-config.py")): create_user_config(args, force=force) else: pywikibot.output('user-config.py already exists in the target ' 'directory "{0}".'.format(base_dir))
def get_dest(self): """Ask for destination script name.""" self.dest = u'{0!s}-core.{1!s}'.format(*tuple(self.source.rsplit(u'.', 1))) if not self.warnonly and not pywikibot.input_yn( u'Destination file is {0!s}.'.format(self.dest), default=True, automatic_quit=False): pywikibot.output('Quitting...') exit()
def main(*args): """ Process command line arguments and invoke bot. If args is an empty list, sys.argv is used. @param args: command line arguments @type args: list of unicode """ options = {} # Process global args and prepare generator args parser local_args = pywikibot.handle_args(args) genFactory = pagegenerators.GeneratorFactory() for arg in local_args: if arg.startswith('-summary:'): options['summary'] = arg[len('-summary:'):] elif arg == '-always': options['always'] = True elif arg == '-async': options['async'] = True elif arg.startswith('-ignore:'): ignore_mode = arg[len('-ignore:'):].lower() if ignore_mode == 'method': options['ignore'] = cosmetic_changes.CANCEL_METHOD elif ignore_mode == 'page': options['ignore'] = cosmetic_changes.CANCEL_PAGE elif ignore_mode == 'match': options['ignore'] = cosmetic_changes.CANCEL_MATCH else: raise ValueError('Unknown ignore mode "{0}"!'.format(ignore_mode)) else: genFactory.handleArg(arg) site = pywikibot.Site() if not options.get('summary'): # Load default summary message. options['summary'] = i18n.twtranslate(site, 'cosmetic_changes-standalone') gen = genFactory.getCombinedGenerator() if gen: if options.get('always') or pywikibot.input_yn( warning + '\nDo you really want to continue?', default=False, automatic_quit=False): site.login() preloadingGen = pagegenerators.PreloadingGenerator(gen) bot = CosmeticChangesBot(preloadingGen, **options) bot.run() return True else: pywikibot.bot.suggest_help(missing_generator=True) return False
def treat_page(self): """Suggest redirects by reordering names in titles.""" if self.current_page.isRedirectPage(): return page_t = self.current_page.title() split_title = page_t.split(' (') name = split_title[0] site = self.current_page.site possible_names = [] if self.getOption('surnames_last'): name_parts = name.split(', ') if len(name_parts) == 2 and len(name.split(' ')) <= 3: possible_names.append(name_parts[1] + ' ' + name_parts[0]) else: words = name.split() if len(words) == 2 and name == name.title(): possible_names.append(words[1] + ', ' + words[0]) elif len(words) == 3: # title may have at most one non-titlecased word if len(SequenceMatcher(None, name, name.title()).get_matching_blocks()) <= 3: possible_names.append(words[1] + ' ' + words[2] + ', ' + words[0]) possible_names.append(words[2] + ', ' + words[0] + ' ' + words[1]) for possible_name in possible_names: # append disambiguation inside parenthesis if there is one if len(split_title) == 2: possible_name += ' (' + split_title[1] new_page = pywikibot.Page(site, possible_name) if new_page.exists(): pywikibot.output('%s already exists, skipping...' % new_page.title(asLink=True)) else: pywikibot.output('%s doesn\'t exist' % new_page.title(asLink=True)) choice = pywikibot.input_yn( 'Do you want to create a redirect?') if choice: comment = i18n.twtranslate( site, 'capitalize_redirects-create-redirect', {'to': page_t}) new_page.set_redirect_target(self.current_page, create=True, summary=comment)
def run(self): """Start the robot's action.""" # regular expression to find the original template. # {{vfd}} does the same thing as {{Vfd}}, so both will be found. # The old syntax, {{msg:vfd}}, will also be found. # The group 'parameters' will either match the parameters, or an # empty string if there are none. replacements = [] exceptions = {} site = pywikibot.Site() for old, new in self.templates.items(): namespaces = list(site.namespace(10, all=True)) if not site.nocapitalize: pattern = '[' + \ re.escape(old[0].upper()) + \ re.escape(old[0].lower()) + \ ']' + re.escape(old[1:]) else: pattern = re.escape(old) pattern = re.sub(r'_|\\ ', r'[_ ]', pattern) templateRegex = re.compile(r'\{\{ *(' + ':|'.join(namespaces) + r':|[mM][sS][gG]:)?' + pattern + r'(?P<parameters>\s*\|.+?|) *}}', re.DOTALL) if self.getOption('subst') and self.getOption('remove'): replacements.append((templateRegex, '{{subst:%s\g<parameters>}}' % new)) exceptions['inside-tags'] = ['ref', 'gallery'] elif self.getOption('subst'): replacements.append((templateRegex, '{{subst:%s\g<parameters>}}' % old)) exceptions['inside-tags'] = ['ref', 'gallery'] elif self.getOption('remove'): replacements.append((templateRegex, '')) else: template = pywikibot.Page(site, new, ns=10) if not template.exists(): pywikibot.warning(u'Template "%s" does not exist.' % new) if not pywikibot.input_yn('Do you want to proceed anyway?', default=False, automatic_quit=False): continue replacements.append((templateRegex, '{{%s\g<parameters>}}' % new)) replaceBot = replace.ReplaceRobot(self.generator, replacements, exceptions, acceptall=self.getOption('always'), addedCat=self.getOption('addedCat'), summary=self.getOption('summary')) replaceBot.run()
def run(self): """Start the robot's action.""" # regular expression to find the original template. # {{vfd}} does the same thing as {{Vfd}}, so both will be found. # The old syntax, {{msg:vfd}}, will also be found. # The group 'parameters' will either match the parameters, or an # empty string if there are none. replacements = [] exceptions = {} namespace = self.site.namespaces[10] for old, new in self.templates.items(): if namespace.case == "first-letter": pattern = "[" + re.escape(old[0].upper()) + re.escape(old[0].lower()) + "]" + re.escape(old[1:]) else: pattern = re.escape(old) pattern = re.sub(r"_|\\ ", r"[_ ]", pattern) templateRegex = re.compile( r"\{\{ *(" + ":|".join(namespace) + r":|[mM][sS][gG]:)?" + pattern + r"(?P<parameters>\s*\|.+?|) *}}", re.DOTALL, ) if self.getOption("subst") and self.getOption("remove"): replacements.append((templateRegex, r"{{subst:%s\g<parameters>}}" % new)) exceptions["inside-tags"] = ["ref", "gallery"] elif self.getOption("subst"): replacements.append((templateRegex, r"{{subst:%s\g<parameters>}}" % old)) exceptions["inside-tags"] = ["ref", "gallery"] elif self.getOption("remove"): replacements.append((templateRegex, "")) else: template = pywikibot.Page(self.site, new, ns=10) if not template.exists(): pywikibot.warning('Template "%s" does not exist.' % new) if not pywikibot.input_yn("Do you want to proceed anyway?", default=False, automatic_quit=False): continue replacements.append((templateRegex, r"{{%s\g<parameters>}}" % new)) replaceBot = replace.ReplaceRobot( self.generator, replacements, exceptions, acceptall=self.getOption("always"), addedCat=self.getOption("addedCat"), summary=self.getOption("summary"), ) replaceBot.run()
def treat(self, page): """Re-directing process. Check if pages are in the given form Something, State, and if so, create a redirect from Something, ST.. """ for sn in self.abbrev: R = re.compile(r', %s$' % sn) if R.search(page.title()): pl = pywikibot.Page(self.site, page.title().replace(sn, self.abbrev[sn])) # A bit hacking here - the real work is done in the # 'except pywikibot.NoPage' part rather than the 'try'. try: pl.get(get_redirect=True) goal = pl.getRedirectTarget().title() if pywikibot.Page(self.site, goal).exists(): pywikibot.output( u"Not creating %s - redirect already exists." % goal) else: pywikibot.warning( u"%s already exists but redirects elsewhere!" % goal) except pywikibot.IsNotRedirectPage: pywikibot.warning( u"Page %s already exists and is not a redirect " u"Please check page!" % pl.title()) except pywikibot.NoPage: if page.isRedirectPage(): p2 = page.getRedirectTarget() pywikibot.output( u'Note: goal page is redirect.\nCreating redirect ' u'to "%s" to avoid double redirect.' % p2.title()) else: p2 = page if self.force or pywikibot.input_yn('Create redirect {0}?' .format(pl.title())): pl.set_redirect_target( p2, create=True, summary=i18n.twtranslate( self.site, 'states_redirect-comment'))
def _handle_warnings(self, warnings): messages = '\n'.join('{0.code}: {0.info}'.format(warning) for warning in sorted(warnings, key=lambda w: w.code)) if len(warnings) > 1: messages = '\n' + messages pywikibot.output('We got the following warning(s): ' + messages) answer = True for warning in warnings: this_answer = self._handle_warning(warning.code) if this_answer is False: answer = False break elif this_answer is None: answer = None if answer is None: answer = pywikibot.input_yn(u"Do you want to ignore?", default=False, automatic_quit=False) return answer
def _handle_warnings(self, warnings): messages = '\n'.join( '{0.code}: {0.info}'.format(warning) for warning in sorted(warnings, key=lambda w: w.code)) if len(warnings) > 1: messages = '\n' + messages pywikibot.output('We got the following warning(s): ' + messages) answer = True for warning in warnings: this_answer = self._handle_warning(warning.code) if this_answer is False: answer = False break if this_answer is None: answer = None if answer is None: answer = pywikibot.input_yn('Do you want to ignore?', default=False, automatic_quit=False) return answer
def change_base_dir(): """Create a new user directory.""" while True: new_base = pywikibot.input("New user directory? ") new_base = os.path.abspath(new_base) if os.path.exists(new_base): if os.path.isfile(new_base): pywikibot.error("there is an existing file with that name.") continue # make sure user can read and write this directory if not os.access(new_base, os.R_OK | os.W_OK): pywikibot.error("directory access restricted") continue pywikibot.output("Using existing directory") break else: try: os.mkdir(new_base, pywikibot.config2.private_files_permission) except Exception: pywikibot.error("ERROR: directory creation failed") continue pywikibot.output("Created new directory.") break if new_base == pywikibot.config2.get_base_dir(new_base): # config would find that file return new_base msg = wrap(u"""WARNING: Your user files will be created in the directory '%(new_base)s' you have chosen. To access these files, you will either have to use the argument "-dir:%(new_base)s" every time you run the bot, or set the environment variable "PYWIKIBOT2_DIR" equal to this directory name in your operating system. See your operating system documentation for how to set environment variables.""" % {'new_base': new_base}, width=76) for line in msg: pywikibot.output(line) if pywikibot.input_yn('Is this OK?', default=False, automatic_quit=False): return new_base pywikibot.output("Aborting changes.") return False
def run_bot(give_url, image_url, desc, shown): """Run the bot.""" if not give_url and image_url: url = pywikibot.input('What URL range should I check ' '(use $ for the part that is changeable)') minimum = int( pywikibot.input('What is the first number to check (default: 1)') or 1) maximum = int( pywikibot.input('What is the last number to check (default: 99)') or 99) ilinks = (url.replace('$', str(i)) for i in range(minimum, maximum + 1)) else: url = (give_url or pywikibot.input('From what URL should I get the images?')) ilinks = get_imagelinks(url, shown) basicdesc = desc or pywikibot.input( 'What text should be added at the end of ' 'the description of each image from this url?') mysite = pywikibot.Site() for image in ilinks: try: include = pywikibot.input_yn('Include image {}?'.format(image), default=False) except QuitKeyboardInterrupt: break if not include: continue categories = get_categories(mysite) desc = pywikibot.input('Give the description of this image:') desc += '\n\n' + basicdesc + '\n\n' + '\n'.join(categories) UploadRobot(image, description=desc).run()
def main(*args: str) -> None: """ Process command line arguments and invoke bot. If args is an empty list, sys.argv is used. :param args: command line arguments """ options = {} # Process global args and prepare generator args parser gen_factory = pagegenerators.GeneratorFactory() local_args = pywikibot.handle_args(args) local_args = gen_factory.handle_args(local_args) for arg in local_args: opt, _, value = arg.partition(':') if opt == '-summary': options['summary'] = value elif opt in ('-always', '-async'): options[opt[1:]] = True elif opt == '-ignore': value = value.upper() try: options['ignore'] = getattr(CANCEL, value) except AttributeError: raise ValueError('Unknown ignore mode {!r}!'.format(value)) gen = gen_factory.getCombinedGenerator(preload=True) if not pywikibot.bot.suggest_help(missing_generator=not gen) \ and (options.get('always') or config.simulate or pywikibot.input_yn( warning + '\nDo you really want to continue?', default=False, automatic_quit=False)): bot = CosmeticChangesBot(generator=gen, **options) bot.run()
def create_user_config(args=None, force=False): """Create a user-config.py in base_dir.""" _fnc = os.path.join(base_dir, "user-config.py") if file_exists(_fnc): return if args and force and not config.verbose_output: # main_username may be None, which is used in the next block main_family, main_lang, main_username = args usernames = [args] else: main_family, main_lang, main_username = get_site_and_lang(*args, force=force) usernames = [(main_family, main_lang, main_username)] while pywikibot.input_yn("Do you want to add any other projects?", force=force, default=False, automatic_quit=False): usernames += [get_site_and_lang(main_family, main_lang, main_username)] if not main_username: usernames = "# usernames['{0}']['{1}'] = u'MyUsername'".format( main_family, main_lang) else: usernames = '\n'.join( u"usernames['{0}']['{1}'] = u'{2}'".format(*username) for username in usernames) config_text = '' config_content = SMALL_CONFIG if ((force and not config.verbose_output) or pywikibot.input_yn('Would you like the extended version of ' 'user-config.py, with explanations ' 'included?', automatic_quit=False, default=True, force=force)): try: # config2.py will be in the pywikibot/ directory relative to this # script (generate_user_files) install = os.path.dirname(os.path.abspath(__file__)) with codecs.open(os.path.join(install, "pywikibot", "config2.py"), "r", "utf-8") as config_f: config_file = config_f.read() res = re.findall("^(# ############# (?:" "LOGFILE|" "INTERWIKI|" "SOLVE_DISAMBIGUATION|" "IMAGE RELATED|" "TABLE CONVERSION BOT|" "WEBLINK CHECKER|" "DATABASE|" "SEARCH ENGINE|" "COPYRIGHT|" "FURTHER" ") SETTINGS .*)^(?=#####|# =====)", config_file, re.MULTILINE | re.DOTALL) if not res: warn('Extended config extraction failed', UserWarning) config_text = '\n'.join(res) if len(config_text.splitlines()) < 350: warn('Extended config extraction too short: %d' % len(config_text.splitlines()), UserWarning) config_content = EXTENDED_CONFIG except Exception as e: # If the warning was explicitly enabled, raise if isinstance(e, UserWarning): raise pywikibot.output('Exception while creating extended user-config; ' 'falling back to simple user-config.') pywikibot.exception() try: with codecs.open(_fnc, "w", "utf-8") as f: f.write(config_content.format(main_family=main_family, main_lang=main_lang, usernames=usernames, config_text=config_text)) pywikibot.output(u"'%s' written." % _fnc) except: try: os.remove(_fnc) except: pass raise
def main(*args): """ Process command line arguments and invoke bot. If args is an empty list, sys.argv is used. @param args: command line arguments @type args: list of unicode """ gen = None notitle = False fmt = '1' outputlang = None page_get = False base_dir = None encoding = config.textfile_encoding page_target = None overwrite = False summary = 'listpages-save-list' # Process global args and prepare generator args parser local_args = pywikibot.handle_args(args) genFactory = GeneratorFactory() for arg in local_args: if arg == '-notitle': notitle = True elif arg.startswith('-format:'): fmt = arg[len('-format:'):] fmt = fmt.replace(u'\\03{{', u'\03{{') elif arg.startswith('-outputlang:'): outputlang = arg[len('-outputlang:'):] elif arg == '-get': page_get = True elif arg.startswith('-save'): base_dir = arg.partition(':')[2] or '.' elif arg.startswith('-encode:'): encoding = arg.partition(':')[2] elif arg.startswith('-put:'): page_target = arg.partition(':')[2] elif arg.startswith('-overwrite'): overwrite = True elif arg.startswith('-summary:'): summary = arg.partition(':')[2] else: genFactory.handleArg(arg) if base_dir: base_dir = os.path.expanduser(base_dir) if not os.path.isabs(base_dir): base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir)) if not os.path.exists(base_dir): pywikibot.output(u'Directory "{0!s}" does not exist.'.format(base_dir)) choice = pywikibot.input_yn( u'Do you want to create it ("No" to continue without saving)?') if choice: os.makedirs(base_dir, mode=0o744) else: base_dir = None elif not os.path.isdir(base_dir): # base_dir is a file. pywikibot.warning(u'Not a directory: "%s"\n' u'Skipping saving ...' % base_dir) base_dir = None if page_target: site = pywikibot.Site() page_target = pywikibot.Page(site, page_target) if not overwrite and page_target.exists(): pywikibot.bot.suggest_help( additional_text='Page "{0}" already exists.'.format( page_target.title())) return False if re.match('^[a-z_-]+$', summary): summary = i18n.twtranslate(site, summary) gen = genFactory.getCombinedGenerator() if gen: i = 0 output_list = [] for i, page in enumerate(gen, start=1): if not notitle: page_fmt = Formatter(page, outputlang) output_list += [page_fmt.output(num=i, fmt=fmt)] pywikibot.stdout(output_list[-1]) if page_get: try: pywikibot.stdout(page.text) except pywikibot.Error as err: pywikibot.output(err) if base_dir: filename = os.path.join(base_dir, page.title(as_filename=True)) pywikibot.output(u'Saving {0!s} to {1!s}'.format(page.title(), filename)) with open(filename, mode='wb') as f: f.write(page.text.encode(encoding)) pywikibot.output(u"{0:d} page(s) found".format(i)) if page_target: page_target.text = '\n'.join(output_list) page_target.save(summary=summary) return True else: pywikibot.bot.suggest_help(missing_generator=True) return False
def create_user_config(args=None, force=False): """Create a user-config.py in base_dir.""" _fnc = os.path.join(base_dir, "user-config.py") if file_exists(_fnc): return if args and force and not config.verbose_output: # main_username may be None, which is used in the next block main_family, main_lang, main_username = args usernames = [args] else: main_family, main_lang, main_username = get_site_and_lang(*args, force=force) usernames = [(main_family, main_lang, main_username)] while pywikibot.input_yn("Do you want to add any other projects?", force=force, default=False, automatic_quit=False): usernames += [ get_site_and_lang(main_family, main_lang, main_username) ] if not main_username: usernames = "# usernames['{0}']['{1}'] = u'MyUsername'".format( main_family, main_lang) else: usernames = '\n'.join( u"usernames['{0}']['{1}'] = u'{2}'".format(*username) for username in usernames) config_text = '' config_content = SMALL_CONFIG if ((force and not config.verbose_output) or pywikibot.input_yn( 'Would you like the extended version of ' 'user-config.py, with explanations ' 'included?', automatic_quit=False, default=True, force=force)): try: # config2.py will be in the pywikibot/ directory relative to this # script (generate_user_files) install = os.path.dirname(os.path.abspath(__file__)) with codecs.open(os.path.join(install, "pywikibot", "config2.py"), "r", "utf-8") as config_f: config_file = config_f.read() res = re.findall( "^(# ############# (?:" "LOGFILE|" "INTERWIKI|" "SOLVE_DISAMBIGUATION|" "IMAGE RELATED|" "TABLE CONVERSION BOT|" "WEBLINK CHECKER|" "DATABASE|" "SEARCH ENGINE|" "COPYRIGHT|" "FURTHER" ") SETTINGS .*)^(?=#####|# =====)", config_file, re.MULTILINE | re.DOTALL) if not res: warn('Extended config extraction failed', UserWarning) config_text = '\n'.join(res) if len(config_text.splitlines()) < 350: warn( 'Extended config extraction too short: %d' % len(config_text.splitlines()), UserWarning) config_content = EXTENDED_CONFIG except Exception as e: # If the warning was explicitly enabled, raise if isinstance(e, UserWarning): raise pywikibot.output('Exception while creating extended user-config; ' 'falling back to simple user-config.') pywikibot.exception() try: with codecs.open(_fnc, "w", "utf-8") as f: f.write( config_content.format(main_family=main_family, main_lang=main_lang, usernames=usernames, config_text=config_text)) pywikibot.output(u"'%s' written." % _fnc) except: try: os.remove(_fnc) except: pass raise
def process_filename(self): """Return base filename portion of self.url.""" # Isolate the pure name filename = self.url # Filename may be either a local file path or a URL if "://" in filename: # extract the path portion of the URL filename = urlparse(filename).path filename = os.path.basename(filename) if self.useFilename: filename = self.useFilename if not self.keepFilename: pywikibot.output( u"The filename on the target wiki will default to: %s" % filename) # FIXME: these 2 belong somewhere else, presumably in family forbidden = '/' # to be extended allowed_formats = (u'gif', u'jpg', u'jpeg', u'mid', u'midi', u'ogg', u'png', u'svg', u'xcf', u'djvu', u'ogv', u'oga', u'tif', u'tiff') # ask until it's valid while True: newfn = pywikibot.input( u'Enter a better name, or press enter to accept:') if newfn == "": newfn = filename break ext = os.path.splitext(newfn)[1].lower().strip('.') # are any chars in forbidden also in newfn? invalid = set(forbidden) & set(newfn) if invalid: c = "".join(invalid) pywikibot.output( 'Invalid character(s): %s. Please try again' % c) continue if ext not in allowed_formats: if not pywikibot.input_yn( u"File format is not one of [%s], but %s. Continue?" % (u' '.join(allowed_formats), ext), default=False, automatic_quit=False): continue break if newfn != '': filename = newfn # A proper description for the submission. # Empty descriptions are not accepted. pywikibot.output(u'The suggested description is:\n%s' % self.description) # Description must be set and verified if not self.description: self.verifyDescription = True while not self.description or self.verifyDescription: if not self.description: pywikibot.output( u'\03{lightred}It is not possible to upload a file ' 'without a summary/description.\03{default}') # if no description, default is 'yes' if pywikibot.input_yn( u'Do you want to change this description?', default=not self.description): from pywikibot import editor as editarticle editor = editarticle.TextEditor() try: newDescription = editor.edit(self.description) except Exception as e: pywikibot.error(e) continue # if user saved / didn't press Cancel if newDescription: self.description = newDescription self.verifyDescription = False return filename
def treat_page(self): """Treat a single page.""" page = self.current_page pagetitle = page.title(with_ns=False) namesp = page.site.namespace(page.namespace()) if self.appendAll: newPageTitle = '{}{}{}'.format(self.pagestart, pagetitle, self.pageend) if not self.noNamespace and namesp: newPageTitle = '{}:{}'.format(namesp, newPageTitle) elif self.regexAll: newPageTitle = self.regex.sub(self.replacePattern, pagetitle) if not self.noNamespace and namesp: newPageTitle = '{}:{}'.format(namesp, newPageTitle) if self.opt.prefix: newPageTitle = '{}{}'.format(self.opt.prefix, pagetitle) if self.opt.prefix or self.appendAll or self.regexAll: if self.user_confirm( 'Change the page title to {!r}?'.format(newPageTitle)): self.moveOne(page, newPageTitle) return # else: choice = pywikibot.input_choice('What do you want to do?', [('change page name', 'c'), ('append to page name', 'a'), ('use a regular expression', 'r'), ('next page', 'n')]) if choice == 'c': newPageTitle = pywikibot.input('New page name:') self.moveOne(page, newPageTitle) elif choice == 'a': self.pagestart = pywikibot.input('Append this to the start:') self.pageend = pywikibot.input('Append this to the end:') newPageTitle = ('{}{}{}'.format(self.pagestart, pagetitle, self.pageend)) if namesp: if pywikibot.input_yn('Do you want to remove the ' 'namespace prefix "{}:"?'.format(namesp), automatic_quit=False): self.noNamespace = True else: newPageTitle = ('{}:{}'.format(namesp, newPageTitle)) choice2 = pywikibot.input_choice( 'Change the page title to {!r}?'.format(newPageTitle), [('yes', 'y'), ('no', 'n'), ('all', 'a')]) if choice2 == 'y': self.moveOne(page, newPageTitle) elif choice2 == 'a': self.appendAll = True self.moveOne(page, newPageTitle) elif choice == 'r': searchPattern = pywikibot.input('Enter the search pattern:') self.replacePattern = pywikibot.input('Enter the replace pattern:') self.regex = re.compile(searchPattern) if page.title() == page.title(with_ns=False): newPageTitle = self.regex.sub(self.replacePattern, page.title()) else: if pywikibot.input_yn('Do you want to remove the ' 'namespace prefix "{}:"?'.format(namesp), automatic_quit=False): newPageTitle = self.regex.sub(self.replacePattern, page.title(with_ns=False)) self.noNamespace = True else: newPageTitle = self.regex.sub(self.replacePattern, page.title()) choice2 = pywikibot.input_choice( 'Change the page title to {!r}?'.format(newPageTitle), [('yes', 'y'), ('no', 'n'), ('all', 'a')]) if choice2 == 'y': self.moveOne(page, newPageTitle) elif choice2 == 'a': self.regexAll = True self.moveOne(page, newPageTitle)
def treat(self, page): self.current_page = page if self.getOption('skipredirects') and page.isRedirectPage(): pywikibot.output(u'Page %s is a redirect; skipping.' % page.title()) return pagetitle = page.title(withNamespace=False) namesp = page.site.namespace(page.namespace()) if self.appendAll: newPageTitle = (u'%s%s%s' % (self.pagestart, pagetitle, self.pageend)) if not self.noNamespace and namesp: newPageTitle = (u'%s:%s' % (namesp, newPageTitle)) elif self.regexAll: newPageTitle = self.regex.sub(self.replacePattern, pagetitle) if not self.noNamespace and namesp: newPageTitle = (u'%s:%s' % (namesp, newPageTitle)) if self.getOption('prefix'): newPageTitle = (u'%s%s' % (self.getOption('prefix'), pagetitle)) if self.getOption('prefix') or self.appendAll or self.regexAll: if not self.getOption('always'): choice2 = pywikibot.input_choice( u'Change the page title to "%s"?' % newPageTitle, [('yes', 'y'), ('no', 'n'), ('all', 'a')]) if choice2 == 'y': self.moveOne(page, newPageTitle) elif choice2 == 'a': self.options['always'] = True self.moveOne(page, newPageTitle) elif choice2 == 'n': pass else: self.treat(page) else: self.moveOne(page, newPageTitle) else: choice = pywikibot.input_choice(u'What do you want to do?', [('change page name', 'c'), ('append to page name', 'a'), ('use a regular expression', 'r'), ('next page', 'n')]) if choice == 'c': newPageTitle = pywikibot.input(u'New page name:') self.moveOne(page, newPageTitle) elif choice == 'a': self.pagestart = pywikibot.input(u'Append this to the start:') self.pageend = pywikibot.input(u'Append this to the end:') newPageTitle = (u'%s%s%s' % (self.pagestart, pagetitle, self.pageend)) if namesp: if pywikibot.input_yn(u'Do you want to remove the ' 'namespace prefix "%s:"?' % namesp, automatic_quit=False): self.noNamespace = True else: newPageTitle = (u'%s:%s' % (namesp, newPageTitle)) choice2 = pywikibot.input_choice( u'Change the page title to "%s"?' % newPageTitle, [('yes', 'y'), ('no', 'n'), ('all', 'a')]) if choice2 == 'y': self.moveOne(page, newPageTitle) elif choice2 == 'a': self.appendAll = True self.moveOne(page, newPageTitle) elif choice2 == 'n': pass else: self.treat(page) elif choice == 'r': searchPattern = pywikibot.input(u'Enter the search pattern:') self.replacePattern = pywikibot.input( u'Enter the replace pattern:') self.regex = re.compile(searchPattern) if page.title() == page.title(withNamespace=False): newPageTitle = self.regex.sub(self.replacePattern, page.title()) else: if pywikibot.input_yn(u'Do you want to remove the ' 'namespace prefix "%s:"?' % namesp, automatic_quit=False): newPageTitle = self.regex.sub( self.replacePattern, page.title(withNamespace=False)) self.noNamespace = True else: newPageTitle = self.regex.sub(self.replacePattern, page.title()) choice2 = pywikibot.input_choice( u'Change the page title to "%s"?' % newPageTitle, [('yes', 'y'), ('no', 'n'), ('all', 'a')]) if choice2 == 'y': self.moveOne(page, newPageTitle) elif choice2 == 'a': self.regexAll = True self.moveOne(page, newPageTitle) elif choice2 == 'n': pass else: self.treat(page) elif choice == 'n': pass else: self.treat(page)
def __init__(self, generator, templates, **kwargs): """ Constructor. @param generator: the pages to work on @type generator: iterable @param templates: a dictionary which maps old template names to their replacements. If remove or subst is True, it maps the names of the templates that should be removed/resolved to None. @type templates: dict """ self.availableOptions.update({ 'subst': False, 'remove': False, 'summary': None, 'addedCat': None, }) Bot.__init__(self, generator=generator, **kwargs) self.templates = templates # get edit summary message if it's empty if not self.getOption('summary'): comma = self.site.mediawiki_message('comma-separator') params = { 'list': comma.join(self.templates.keys()), 'num': len(self.templates) } site = self.site if self.getOption('remove'): self.options['summary'] = i18n.twtranslate( site, 'template-removing', params) elif self.getOption('subst'): self.options['summary'] = i18n.twtranslate( site, 'template-substituting', params) else: self.options['summary'] = i18n.twtranslate( site, 'template-changing', params) # regular expression to find the original template. # {{vfd}} does the same thing as {{Vfd}}, so both will be found. # The old syntax, {{msg:vfd}}, will also be found. # The group 'parameters' will either match the parameters, or an # empty string if there are none. replacements = [] exceptions = {} builder = textlib._MultiTemplateMatchBuilder(site) for old, new in self.templates.items(): templateRegex = builder.pattern(old) if self.getOption('subst') and self.getOption('remove'): replacements.append( (templateRegex, r'{{subst:%s\g<parameters>}}' % new)) exceptions['inside-tags'] = ['ref', 'gallery'] elif self.getOption('subst'): replacements.append( (templateRegex, r'{{subst:%s\g<parameters>}}' % old)) exceptions['inside-tags'] = ['ref', 'gallery'] elif self.getOption('remove'): replacements.append((templateRegex, '')) else: template = pywikibot.Page(self.site, new, ns=10) if not template.exists(): pywikibot.warning(u'Template "%s" does not exist.' % new) if not pywikibot.input_yn('Do you want to proceed anyway?', default=False, automatic_quit=False): continue replacements.append( (templateRegex, r'{{%s\g<parameters>}}' % new)) super(TemplateRobot, self).__init__(generator, replacements, exceptions, always=self.getOption('always'), addedCat=self.getOption('addedCat'), summary=self.getOption('summary'))
def run(self): commons = pywikibot.Site('commons', 'commons') comment = i18n.translate(self.site, nowCommonsMessage, fallback=True) for page in self.getPageGenerator(): if self.getOption('use_hash'): # Page -> Has the namespace | commons image -> Not images_list = page # 0 -> local image, 1 -> commons image page = pywikibot.Page(self.site, images_list[0]) else: # If use_hash is true, we have already print this before, no need self.current_page = page try: localImagePage = pywikibot.FilePage(self.site, page.title()) if localImagePage.fileIsShared(): pywikibot.output(u'File is already on Commons.') continue sha1 = localImagePage.latest_file_info.sha1 if self.getOption('use_hash'): filenameOnCommons = images_list[1] else: filenameOnCommons = self.findFilenameOnCommons( localImagePage) if not filenameOnCommons and not self.getOption('use_hash'): pywikibot.output(u'NowCommons template not found.') continue commonsImagePage = pywikibot.FilePage( commons, 'Image:%s' % filenameOnCommons) if localImagePage.title(withNamespace=False) == \ commonsImagePage.title(withNamespace=False) and self.getOption('use_hash'): pywikibot.output( u'The local and the commons images have the same name') if localImagePage.title(withNamespace=False) != \ commonsImagePage.title(withNamespace=False): usingPages = list(localImagePage.usingPages()) if usingPages and usingPages != [localImagePage]: pywikibot.output( u'\"\03{lightred}%s\03{default}\" is still used in %i pages.' % (localImagePage.title(withNamespace=False), len(usingPages))) if self.getOption('replace') is True: pywikibot.output( u'Replacing \"\03{lightred}%s\03{default}\" by \ \"\03{lightgreen}%s\03{default}\".' % (localImagePage.title(withNamespace=False), commonsImagePage.title(withNamespace=False))) oImageRobot = image.ImageRobot( pg.FileLinksGenerator(localImagePage), localImagePage.title(withNamespace=False), commonsImagePage.title(withNamespace=False), '', self.getOption('replacealways'), self.getOption('replaceloose')) oImageRobot.run() # If the image is used with the urlname the # previous function won't work if len(list(pywikibot.FilePage(self.site, page.title()).usingPages())) > 0 and \ self.getOption('replaceloose'): oImageRobot = image.ImageRobot( pg.FileLinksGenerator(localImagePage), localImagePage.title(withNamespace=False, asUrl=True), commonsImagePage.title( withNamespace=False), '', self.getOption('replacealways'), self.getOption('replaceloose')) oImageRobot.run() # refresh because we want the updated list usingPages = len( list( pywikibot.FilePage( self.site, page.title()).usingPages())) if usingPages > 0 and self.getOption('use_hash'): # just an enter pywikibot.input( u'There are still %s pages with this \ image, confirm the manual removal from them please.' % usingPages) else: pywikibot.output(u'Please change them manually.') continue else: pywikibot.output( u'No page is using \"\03{lightgreen}%s\03{default}\" anymore.' % localImagePage.title(withNamespace=False)) commonsText = commonsImagePage.get() if self.getOption('replaceonly') is False: if sha1 == commonsImagePage.latest_file_info.sha1: pywikibot.output( u'The image is identical to the one on Commons.') if len(localImagePage.getFileVersionHistory() ) > 1 and not self.getOption('use_hash'): pywikibot.output( u"This image has a version history. Please \ delete it manually after making sure that the \ old versions are not worth keeping." "") continue if self.getOption('always') is False: pywikibot.output( u'\n\n>>>> Description on \03{lightpurple}%s\03{default} <<<<\n' % page.title()) pywikibot.output(localImagePage.get()) pywikibot.output( u'\n\n>>>> Description on \03{lightpurple}%s\03{default} <<<<\n' % commonsImagePage.title()) pywikibot.output(commonsText) if pywikibot.input_yn( u'Does the description on Commons contain ' 'all required source and license\n' 'information?', default=False, automatic_quit=False): localImagePage.delete( '%s [[:commons:Image:%s]]' % (comment, filenameOnCommons), prompt=False) else: localImagePage.delete( comment + ' [[:commons:Image:%s]]' % filenameOnCommons, prompt=False) else: pywikibot.output( u'The image is not identical to the one on Commons.' ) except (pywikibot.NoPage, pywikibot.IsRedirectPage) as e: pywikibot.output(u'%s' % e[0]) continue
def useHashGenerator(self): # https://toolserver.org/~multichill/nowcommons.php?language=it&page=2&filter= lang = self.site.lang num_page = 0 word_to_skip_translated = i18n.translate(self.site, word_to_skip) images_processed = list() while 1: url = ('https://toolserver.org/~multichill/nowcommons.php?' 'language=%s&page=%s&filter=') % (lang, num_page) HTML_text = self.site.getUrl(url, no_hostname=True) reg = r'<[Aa] href="(?P<urllocal>.*?)">(?P<imagelocal>.*?)</[Aa]> +?</td><td>\n\s*?' reg += r'<[Aa] href="(?P<urlcommons>http[s]?://commons.wikimedia.org/.*?)" \ >Image:(?P<imagecommons>.*?)</[Aa]> +?</td><td>' regex = re.compile(reg, re.UNICODE) found_something = False change_page = True for x in regex.finditer(HTML_text): found_something = True image_local = x.group('imagelocal') image_commons = x.group('imagecommons') if image_local in images_processed: continue change_page = False images_processed.append(image_local) # Skip images that have something in the title (useful for it.wiki) image_to_skip = False for word in word_to_skip_translated: if word.lower() in image_local.lower(): image_to_skip = True if image_to_skip: continue url_local = x.group('urllocal') url_commons = x.group('urlcommons') pywikibot.output( u"\n\n>>> \03{lightpurple}%s\03{default} <<<" % image_local) pywikibot.output(u'Local: %s\nCommons: %s\n' % (url_local, url_commons)) webbrowser.open(url_local, 0, 1) webbrowser.open(url_commons, 0, 1) if image_local.split('Image:')[1] == image_commons: choice = pywikibot.input_yn( u'The local and the commons images have the same name, ' 'continue?', default=False, automatic_quit=False) else: choice = pywikibot.input_yn(u'Are the two images equal?', default=False, automatic_quit=False) if choice: yield [image_local, image_commons] else: continue # The page is dinamically updated, so we may don't need to change it if change_page: num_page += 1 # If no image found means that there aren't anymore, break. if not found_something: break
def __init__(self, generator, templates, **kwargs): """ Initializer. @param generator: the pages to work on @type generator: iterable @param templates: a dictionary which maps old template names to their replacements. If remove or subst is True, it maps the names of the templates that should be removed/resolved to None. @type templates: dict """ self.availableOptions.update({ 'subst': False, 'remove': False, 'summary': None, 'addedCat': None, }) Bot.__init__(self, generator=generator, **kwargs) self.templates = templates # get edit summary message if it's empty if not self.getOption('summary'): comma = self.site.mediawiki_message('comma-separator') params = { 'list': comma.join(self.templates.keys()), 'num': len(self.templates) } if self.getOption('remove'): self.options['summary'] = i18n.twtranslate( self.site, 'template-removing', params) elif self.getOption('subst'): self.options['summary'] = i18n.twtranslate( self.site, 'template-substituting', params) else: self.options['summary'] = i18n.twtranslate( self.site, 'template-changing', params) replacements = [] exceptions = {} builder = textlib._MultiTemplateMatchBuilder(self.site) for old, new in self.templates.items(): template_regex = builder.pattern(old) if self.getOption('subst') and self.getOption('remove'): replacements.append( (template_regex, r'{{subst:%s\g<parameters>}}' % new)) exceptions['inside-tags'] = [ 'ref', 'gallery', 'poem', 'pagelist', ] elif self.getOption('subst'): replacements.append( (template_regex, r'{{subst:%s\g<parameters>}}' % old)) exceptions['inside-tags'] = [ 'ref', 'gallery', 'poem', 'pagelist', ] elif self.getOption('remove'): separate_line_regex = re.compile( r'^[*#:]* *{0} *\n'.format(template_regex.pattern), re.DOTALL | re.MULTILINE) replacements.append((separate_line_regex, '')) spaced_regex = re.compile( r' +{0} +'.format(template_regex.pattern), re.DOTALL) replacements.append((spaced_regex, ' ')) replacements.append((template_regex, '')) else: template = pywikibot.Page(self.site, new, ns=10) if not template.exists(): pywikibot.warning(u'Template "%s" does not exist.' % new) if not pywikibot.input_yn('Do you want to proceed anyway?', default=False, automatic_quit=False): continue replacements.append( (template_regex, r'{{%s\g<parameters>}}' % new)) super(TemplateRobot, self).__init__(generator, replacements, exceptions, always=self.getOption('always'), addedCat=self.getOption('addedCat'), summary=self.getOption('summary'))
def create_user_config(args=None, force=False): """ Create a user-config.py in base_dir. Create a user-password.py if necessary. """ _fnc = os.path.join(base_dir, "user-config.py") _fncpass = os.path.join(base_dir, 'user-password.py') if file_exists(_fnc): return if args and force and not config.verbose_output: # main_username may be None, which is used in the next block main_family, main_code, main_username = args usernames = [args] else: main_family, main_code, main_username = get_site_and_lang(*args, force=force) usernames = [(main_family, main_code, main_username)] while pywikibot.input_yn("Do you want to add any other projects?", force=force, default=False, automatic_quit=False): usernames += [get_site_and_lang(main_family, main_code, main_username)] botpasswords = [] if not main_username: usernames = "# usernames['{0}']['{1}'] = u'MyUsername'".format( main_family, main_code) elif not file_exists(_fncpass): # For each different username entered, ask if user wants to save a # BotPassword (username, BotPassword name, BotPassword pass) seen = set() for username in usernames: if username[2] in seen: continue seen.add(username[2]) if pywikibot.input_yn('Do you want to add a BotPassword for {0}?' .format(username[2]), force=force, default=False): if not botpasswords: pywikibot.output( 'See https://www.mediawiki.org/wiki/' 'Manual:Pywikibot/BotPasswords to know ' 'how to get codes.') pywikibot.output('Please note that plain text in {0} and ' 'anyone with read access to that ' 'directory will be able read the file.' .format(_fncpass)) message = 'BotPassword\'s "bot name" for {0}'.format( username[2]) botpasswordname = pywikibot.input(message, force=force) message = 'BotPassword\'s "password" for BotPassword "{0}" ' \ '(no characters will be shown)' \ .format(botpasswordname) botpasswordpass = pywikibot.input(message, force=force, password=True) if botpasswordname and botpasswordpass: botpasswords.append((username[2], botpasswordname, botpasswordpass)) usernames = '\n'.join( u"usernames['{0}']['{1}'] = u'{2}'".format(*username) for username in usernames) botpasswords = '\n'.join( "('{0}', BotPassword('{1}', '{2}'))".format(*botpassword) for botpassword in botpasswords) config_text = '' config_content = SMALL_CONFIG try: # config2.py will be in the pywikibot/ directory relative to this # script (generate_user_files) install = os.path.dirname(os.path.abspath(__file__)) with codecs.open(os.path.join(install, "pywikibot", "config2.py"), "r", "utf-8") as config_f: config_file = config_f.read() res = re.findall("^(# ############# (?:" "LOGFILE|" 'EXTERNAL SCRIPT PATH|' "INTERWIKI|" "SOLVE_DISAMBIGUATION|" "IMAGE RELATED|" "TABLE CONVERSION BOT|" "WEBLINK CHECKER|" "DATABASE|" "SEARCH ENGINE|" "COPYRIGHT|" "FURTHER" ") SETTINGS .*)^(?=#####|# =====)", config_file, re.MULTILINE | re.DOTALL) if not res: warn('Extended config extraction failed', UserWarning) config_text = '\n'.join(res) if len(config_text.splitlines()) < 350: warn('Extended config extraction too short: %d' % len(config_text.splitlines()), UserWarning) config_content = EXTENDED_CONFIG except Exception as e: # If the warning was explicitly enabled, raise if isinstance(e, UserWarning): raise pywikibot.output('Exception while creating extended user-config; ' 'falling back to simple user-config.') pywikibot.exception() try: # Finally save user-config.py with codecs.open(_fnc, "w", "utf-8") as f: f.write(config_content.format(main_family=main_family, main_code=main_code, usernames=usernames, config_text=config_text, botpasswords='password_file = ' + ('"user-password.py"' if botpasswords else 'None'))) pywikibot.output(u"'%s' written." % _fnc) except BaseException: if os.path.exists(_fnc): os.remove(_fnc) raise if botpasswords: # Save if necessary user-password.py try: # First create an empty file with good permissions, before writing # in it with codecs.open(_fncpass, 'w', 'utf-8') as f: f.write('') pywikibot.tools.file_mode_checker(_fncpass, mode=0o600, quiet=True) with codecs.open(_fncpass, 'w', 'utf-8') as f: f.write(PASSFILE_CONFIG.format(botpasswords=botpasswords)) pywikibot.tools.file_mode_checker(_fncpass, mode=0o600) pywikibot.output("'{0}' written.".format(_fncpass)) except EnvironmentError: os.remove(_fncpass) raise
def process_filename(self, file_url=None): """Return base filename portion of file_url.""" if not file_url: file_url = self.url pywikibot.warning("file_url is not given. " "Set to self.url by default.") always = self.getOption('always') # Isolate the pure name filename = file_url # Filename may be either a URL or a local file path if "://" in filename: # extract the path portion of the URL filename = urlparse(filename).path filename = os.path.basename(filename) if self.useFilename: filename = self.useFilename if not self.keepFilename: pywikibot.output( u"The filename on the target wiki will default to: %s" % filename) assert not always newfn = pywikibot.input( u'Enter a better name, or press enter to accept:') if newfn != "": filename = newfn # FIXME: these 2 belong somewhere else, presumably in family # forbidden characters are handled by pywikibot/page.py forbidden = ':*?/\\' # to be extended try: allowed_formats = self.targetSite.siteinfo.get('fileextensions', get_default=False) except KeyError: allowed_formats = [] else: allowed_formats = [item['ext'] for item in allowed_formats] # ask until it's valid first_check = True while True: if not first_check: if always: filename = None else: filename = pywikibot.input('Enter a better name, or press ' 'enter to skip the file:') if not filename: return None first_check = False ext = os.path.splitext(filename)[1].lower().strip('.') # are any chars in forbidden also in filename? invalid = set(forbidden) & set(filename) if invalid: c = "".join(invalid) pywikibot.output('Invalid character(s): %s. Please try again' % c) continue if allowed_formats and ext not in allowed_formats: if always: pywikibot.output('File format is not one of ' '[{0}]'.format(' '.join(allowed_formats))) continue elif not pywikibot.input_yn( u"File format is not one of [%s], but %s. Continue?" % (u' '.join(allowed_formats), ext), default=False, automatic_quit=False): continue potential_file_page = pywikibot.FilePage(self.targetSite, filename) if potential_file_page.exists(): overwrite = self._handle_warning('exists') if overwrite is False: pywikibot.output( "File exists and you asked to abort. Skipping.") return None if potential_file_page.canBeEdited(): if overwrite is None: overwrite = not pywikibot.input_yn( "File with name %s already exists. " "Would you like to change the name? " "(Otherwise file will be overwritten.)" % filename, default=True, automatic_quit=False) if not overwrite: continue else: break else: pywikibot.output(u"File with name %s already exists and " "cannot be overwritten." % filename) continue else: try: if potential_file_page.fileIsShared(): pywikibot.output( u"File with name %s already exists in shared " "repository and cannot be overwritten." % filename) continue else: break except pywikibot.NoPage: break # A proper description for the submission. # Empty descriptions are not accepted. pywikibot.output(u'The suggested description is:\n%s' % self.description) # Description must be set and verified if not self.description: self.verifyDescription = True while not self.description or self.verifyDescription: if not self.description: pywikibot.output( color_format( '{lightred}It is not possible to upload a file ' 'without a summary/description.{default}')) assert not always # if no description, default is 'yes' if pywikibot.input_yn(u'Do you want to change this description?', default=not self.description): from pywikibot import editor as editarticle editor = editarticle.TextEditor() try: newDescription = editor.edit(self.description) except Exception as e: pywikibot.error(e) continue # if user saved / didn't press Cancel if newDescription: self.description = newDescription self.verifyDescription = False return filename
def run(self): """Run the bot.""" commons = self.commons comment = self.summary for page in self.generator: self.current_page = page try: local_file_page = pywikibot.FilePage(self.site, page.title()) if local_file_page.file_is_shared(): pywikibot.output('File is already on Commons.') continue sha1 = local_file_page.latest_file_info.sha1 file_on_commons = self.find_file_on_commons(local_file_page) if not file_on_commons: pywikibot.output('NowCommons template not found.') continue commons_file_page = pywikibot.FilePage( commons, 'File:' + file_on_commons) if (local_file_page.title(with_ns=False) != commons_file_page.title(with_ns=False)): using_pages = list(local_file_page.using_pages()) if using_pages and using_pages != [local_file_page]: pywikibot.output( color_format( '"{lightred}{0}{default}" ' 'is still used in {1} pages.', local_file_page.title(with_ns=False), len(using_pages))) if self.opt.replace: pywikibot.output( color_format( 'Replacing "{lightred}{0}{default}" by ' '"{lightgreen}{1}{default}\".', local_file_page.title(with_ns=False), commons_file_page.title(with_ns=False))) bot = ImageBot( local_file_page.usingPages(), local_file_page.title(with_ns=False), commons_file_page.title(with_ns=False), always=self.opt.replacealways, loose=self.opt.replaceloose) bot.run() # If the image is used with the urlname the # previous function won't work is_used = bool( list( pywikibot.FilePage( self.site, page.title()).using_pages(total=1))) if is_used and self.opt.replaceloose: bot = ImageBot( local_file_page.usimgPages(), local_file_page.title(with_ns=False, as_url=True), commons_file_page.title(with_ns=False), always=self.opt.replacealways, loose=self.opt.replaceloose) bot.run() # refresh because we want the updated list using_pages = len( list( pywikibot.FilePage( self.site, page.title()).using_pages())) else: pywikibot.output('Please change them manually.') continue pywikibot.output( color_format( 'No page is using "{lightgreen}{0}{default}" ' 'anymore.', local_file_page.title(with_ns=False))) commons_text = commons_file_page.get() if not self.opt.replaceonly: if sha1 == commons_file_page.latest_file_info.sha1: pywikibot.output( 'The file is identical to the one on Commons.') if len(local_file_page.get_file_history()) > 1: pywikibot.output( 'This file has a version history. Please ' 'delete it manually after making sure that ' 'the old versions are not worth keeping.') continue if self.opt.always is False: format_str = color_format( '\n\n>>>> Description on {lightpurple}%s' '{default} <<<<\n') pywikibot.output(format_str % page.title()) pywikibot.output(local_file_page.get()) pywikibot.output(format_str % commons_file_page.title()) pywikibot.output(commons_text) if pywikibot.input_yn( 'Does the description on Commons contain ' 'all required source and license\n' 'information?', default=False, automatic_quit=False): local_file_page.delete( '{} [[:commons:File:{}]]'.format( comment, file_on_commons), prompt=False) else: local_file_page.delete( comment + ' [[:commons:File:{}]]'.format( file_on_commons), prompt=False) else: pywikibot.output('The file is not identical to ' 'the one on Commons.') except (NoPageError, IsRedirectPageError) as e: pywikibot.output(str(e[0])) continue else: self.counter['read'] += 1 if not self.counter['read']: pywikibot.output('No transcluded files found for {}.'.format( self.nc_templates_list()[0])) self.exit()
def treat(self, refPage, disambPage): """ Treat a page. Parameters: disambPage - The disambiguation page or redirect we don't want anything to link to refPage - A page linking to disambPage Returns False if the user pressed q to completely quit the program. Otherwise, returns True. """ # TODO: break this function up into subroutines! include = False unlink_counter = 0 new_targets = [] try: text = refPage.get() ignoreReason = self.checkContents(text) if ignoreReason: pywikibot.output( '\n\nSkipping %s because it contains %s.\n\n' % (refPage.title(), ignoreReason)) else: include = True except pywikibot.IsRedirectPage: pywikibot.output(u'%s is a redirect to %s' % (refPage.title(), disambPage.title())) if disambPage.isRedirectPage(): target = self.alternatives[0] if pywikibot.input_yn(u'Do you want to make redirect %s point ' 'to %s?' % (refPage.title(), target), default=False, automatic_quit=False): redir_text = '#%s [[%s]]' \ % (self.mysite.redirect(), target) try: refPage.put_async(redir_text, summary=self.comment) except pywikibot.PageNotSaved as error: pywikibot.output(u'Page not saved: %s' % error.args) else: choice = pywikibot.input_choice( u'Do you want to work on pages linking to %s?' % refPage.title(), [('yes', 'y'), ('no', 'n'), ('change redirect', 'c')], 'n', automatic_quit=False) if choice == 'y': gen = ReferringPageGeneratorWithIgnore( refPage, self.primary) preloadingGen = pagegenerators.PreloadingGenerator(gen) for refPage2 in preloadingGen: # run until the user selected 'quit' if not self.treat(refPage2, refPage): break elif choice == 'c': text = refPage.get(get_redirect=True) include = "redirect" except pywikibot.NoPage: pywikibot.output( u'Page [[%s]] does not seem to exist?! Skipping.' % refPage.title()) include = False if include in (True, "redirect"): # make a backup of the original text so we can show the changes later original_text = text n = 0 curpos = 0 dn = False edited = False # This loop will run until we have finished the current page while True: m = self.linkR.search(text, pos=curpos) if not m: if n == 0: pywikibot.output(u"No changes necessary in %s" % refPage.title()) return True else: # stop loop and save page break # Make sure that next time around we will not find this same hit. curpos = m.start() + 1 try: foundlink = pywikibot.Link(m.group('title'), disambPage.site) foundlink.parse() except pywikibot.Error: continue # ignore interwiki links if foundlink.site != disambPage.site: continue # Check whether the link found is to disambPage. try: if foundlink.canonical_title() != disambPage.title(): continue except pywikibot.Error: # must be a broken link pywikibot.log(u"Invalid link [[%s]] in page [[%s]]" % (m.group('title'), refPage.title())) continue n += 1 # how many bytes should be displayed around the current link context = 60 # check if there's a dn-template here already if (self.dnSkip and self.dn_template_str and self.dn_template_str[:-2] in text[m.end():m.end() + len(self.dn_template_str) + 8]): continue # This loop will run while the user doesn't choose an option # that will actually change the page while True: self.current_page = refPage if not self.always: # at the beginning of the link, start red color. # at the end of the link, reset the color to default pywikibot.output(text[max(0, m.start() - context):m.start()] + '\03{lightred}' + text[m.start():m.end()] + '\03{default}' + text[m.end():m.end() + context]) options = [ '#', 'r#', '[s]kip link', '[e]dit page', '[n]ext page', '[u]nlink', '[q]uit' ] if self.dn_template_str: options.append(u'[t]ag template %s' % self.dn_template_str) options.append('[m]ore context') if not edited: options.append('show [d]isambiguation page') options += ['[l]ist', '[a]dd new'] if edited: options += ['save in this form [x]'] options = concat_options('Option', 72, options) choice = pywikibot.input(options) else: choice = self.always if choice in ['a', 'A']: newAlternative = pywikibot.input(u'New alternative:') self.alternatives.append(newAlternative) self.listAlternatives() elif choice in ['e', 'E']: editor = editarticle.TextEditor() newText = editor.edit(text, jumpIndex=m.start(), highlight=disambPage.title()) # if user didn't press Cancel if newText and newText != text: text = newText break elif choice in ['d', 'D']: editor = editarticle.TextEditor() if disambPage.isRedirectPage(): disambredir = disambPage.getRedirectTarget() editor.edit(disambredir.get(), jumpIndex=m.start(), highlight=disambredir.title()) else: editor.edit(disambPage.get(), jumpIndex=m.start(), highlight=disambPage.title()) elif choice in ['l', 'L']: self.listAlternatives() elif choice in ['m', 'M']: # show more text around the link we're working on context *= 2 else: break if choice in ['e', 'E']: # user has edited the page and then pressed 'OK' edited = True curpos = 0 continue elif choice in ['n', 'N']: # skip this page if self.primary: # If run with the -primary argument, skip this # occurrence next time. self.primaryIgnoreManager.ignore(refPage) return True elif choice in ['q', 'Q']: # quit the program self.quit() elif choice in ['s', 'S']: # Next link on this page n -= 1 continue elif choice in ['x', 'X'] and edited: # Save the page as is break # The link looks like this: # [[page_title|link_text]]trailing_chars page_title = m.group('title') link_text = m.group('label') if not link_text: # or like this: [[page_title]]trailing_chars link_text = page_title if m.group('section') is None: section = '' else: section = m.group('section') trailing_chars = m.group('linktrail') if trailing_chars: link_text += trailing_chars # '?', '/' for old choice if choice in ['t', 'T', '?', '/'] and self.dn_template_str: # small chunk of text to search search_text = text[m.end():m.end() + context] # figure out where the link (and sentance) ends, put note # there end_of_word_match = re.search(r'\s', search_text) if end_of_word_match: position_split = end_of_word_match.start(0) else: position_split = 0 # insert dab needed template text = (text[:m.end() + position_split] + self.dn_template_str + text[m.end() + position_split:]) dn = True continue elif choice in ['u', 'U']: # unlink - we remove the section if there's any text = text[:m.start()] + link_text + text[m.end():] unlink_counter += 1 continue else: if len(choice) > 0 and choice[0] == 'r': # we want to throw away the original link text replaceit = link_text == page_title choice = choice[1:] elif include == "redirect": replaceit = True else: replaceit = False try: choice = int(choice) except ValueError: pywikibot.output(u"Unknown option") # step back to ask the user again what to do with the # current link curpos -= 1 continue if choice >= len(self.alternatives) or choice < 0: pywikibot.output( u"Choice out of range. Please select a number " u"between 0 and %i." % (len(self.alternatives) - 1)) # show list of possible choices self.listAlternatives() # step back to ask the user again what to do with the # current link curpos -= 1 continue new_page_title = self.alternatives[choice] repPl = pywikibot.Page( pywikibot.Link(new_page_title, disambPage.site)) if (new_page_title[0].isupper() or link_text[0].isupper()): new_page_title = repPl.title() else: new_page_title = repPl.title() new_page_title = first_lower(new_page_title) if new_page_title not in new_targets: new_targets.append(new_page_title) if replaceit and trailing_chars: newlink = "[[%s%s]]%s" % (new_page_title, section, trailing_chars) elif replaceit or (new_page_title == link_text and not section): newlink = "[[%s]]" % new_page_title # check if we can create a link with trailing characters # instead of a pipelink elif ((len(new_page_title) <= len(link_text)) and (firstcap(link_text[:len(new_page_title)]) == firstcap(new_page_title)) and (re.sub(self.trailR, '', link_text[len(new_page_title):]) == '') and (not section)): newlink = "[[%s]]%s" \ % (link_text[:len(new_page_title)], link_text[len(new_page_title):]) else: newlink = "[[%s%s|%s]]" \ % (new_page_title, section, link_text) text = text[:m.start()] + newlink + text[m.end():] continue pywikibot.output(text[max(0, m.start() - 30):m.end() + 30]) if text == original_text: pywikibot.output(u'\nNo changes have been made:\n') else: pywikibot.output(u'\nThe following changes have been made:\n') pywikibot.showDiff(original_text, text) pywikibot.output(u'') # save the page self.setSummaryMessage(disambPage, new_targets, unlink_counter, dn) try: refPage.put_async(text, summary=self.comment) except pywikibot.LockedPage: pywikibot.output(u'Page not saved: page is locked') except pywikibot.PageNotSaved as error: pywikibot.output(u'Page not saved: %s' % error.args) return True
def main(*args): """ Process command line arguments and invoke bot. If args is an empty list, sys.argv is used. @param args: command line arguments @type args: list of unicode """ add_cat = None gen = None # summary message edit_summary = u"" # Array which will collect commandline parameters. # First element is original text, second element is replacement text. commandline_replacements = [] # A list of 2-tuples of original text and replacement text. replacements = [] # Don't edit pages which contain certain texts. exceptions = { 'title': [], 'text-contains': [], 'inside': [], 'inside-tags': [], 'require-title': [], # using a seperate requirements dict needs some } # major refactoring of code. # Should the elements of 'replacements' and 'exceptions' be interpreted # as regular expressions? regex = False # Predefined fixes from dictionary 'fixes' (see above). fixes_set = [] # the dump's path, either absolute or relative, which will be used # if -xml flag is present xmlFilename = None useSql = False # will become True when the user presses a ('yes to all') or uses the # -always flag. acceptall = False # Will become True if the user inputs the commandline parameter -nocase caseInsensitive = False # Will become True if the user inputs the commandline parameter -dotall dotall = False # Will become True if the user inputs the commandline parameter -multiline multiline = False # Do all hits when they overlap allowoverlap = False # Do not recurse replacement recursive = False # Between a regex and another (using -fix) sleep some time (not to waste # too much CPU sleep = None # Read commandline parameters. local_args = pywikibot.handle_args(args) genFactory = pagegenerators.GeneratorFactory() for arg in local_args: if genFactory.handleArg(arg): continue if arg == '-regex': regex = True elif arg.startswith('-xmlstart'): if len(arg) == 9: xmlStart = pywikibot.input( u'Please enter the dumped article to start with:') else: xmlStart = arg[10:] elif arg.startswith('-xml'): if len(arg) == 4: xmlFilename = i18n.input('pywikibot-enter-xml-filename') else: xmlFilename = arg[5:] elif arg == '-sql': useSql = True elif arg.startswith('-excepttitle:'): exceptions['title'].append(arg[13:]) elif arg.startswith('-requiretitle:'): exceptions['require-title'].append(arg[14:]) elif arg.startswith('-excepttext:'): exceptions['text-contains'].append(arg[12:]) elif arg.startswith('-exceptinside:'): exceptions['inside'].append(arg[14:]) elif arg.startswith('-exceptinsidetag:'): exceptions['inside-tags'].append(arg[17:]) elif arg.startswith('-fix:'): fixes_set += [arg[5:]] elif arg.startswith('-sleep:'): sleep = float(arg[7:]) elif arg == '-always': acceptall = True elif arg == '-recursive': recursive = True elif arg == '-nocase': caseInsensitive = True elif arg == '-dotall': dotall = True elif arg == '-multiline': multiline = True elif arg.startswith('-addcat:'): add_cat = arg[8:] elif arg.startswith('-summary:'): edit_summary = arg[9:] elif arg.startswith('-allowoverlap'): allowoverlap = True else: commandline_replacements.append(arg) site = pywikibot.Site() if (len(commandline_replacements) % 2): raise pywikibot.Error('require even number of replacements.') if not commandline_replacements: if fixes_set: manual = pywikibot.input_yn('Replacements via -fix: set. Apply ' 'also manual replacements?', default=False) else: manual = True if manual: old = pywikibot.input(u'Please enter the text that should be replaced:') while old: new = pywikibot.input(u'Please enter the new text:') commandline_replacements += [old, new] old = pywikibot.input( u'Please enter another text that should be replaced,' + u'\nor press Enter to start:') single_summary = None for i in range(0, len(commandline_replacements), 2): replacement = Replacement(commandline_replacements[i], commandline_replacements[i + 1]) if not single_summary: single_summary = i18n.twtranslate( site, 'replace-replacing', {'description': ' (-%s +%s)' % (replacement.old, replacement.new)} ) replacements.append(replacement) if not edit_summary: if single_summary: pywikibot.output(u'The summary message for the command line ' 'replacements will be something like: %s' % single_summary) if fixes_set: pywikibot.output('If a summary is defined for the fix, this ' 'default summary won\'t be applied.') edit_summary = pywikibot.input( u'Press Enter to use this automatic message, or enter a ' + u'description of the\nchanges your bot will make:') # Perform one of the predefined actions. for fix in fixes_set: try: fix = fixes.fixes[fix] except KeyError: pywikibot.output(u'Available predefined fixes are: %s' % ', '.join(fixes.fixes.keys())) return if "msg" in fix: if isinstance(fix['msg'], basestring): set_summary = i18n.twtranslate(site, str(fix['msg'])) else: set_summary = i18n.translate(site, fix['msg'], fallback=True) else: set_summary = None for replacement in fix['replacements']: summary = set_summary if len(replacement) < 3 else replacement[2] replacements.append(Replacement( old=replacement[0], new=replacement[1], use_regex=fix.get('regex'), edit_summary=summary, exceptions=fix.get('exceptions'), case_insensitive=fix.get('nocase') )) # Set the regular expression flags flags = re.UNICODE if caseInsensitive: flags = flags | re.IGNORECASE if dotall: flags = flags | re.DOTALL if multiline: flags = flags | re.MULTILINE # Pre-compile all regular expressions here to save time later for replacement in replacements: replacement.compile(regex, flags) precompile_exceptions(exceptions, regex, flags) if xmlFilename: try: xmlStart except NameError: xmlStart = None gen = XmlDumpReplacePageGenerator(xmlFilename, xmlStart, replacements, exceptions, site) elif useSql: whereClause = 'WHERE (%s)' % ' OR '.join( ["old_text RLIKE '%s'" % prepareRegexForMySQL(old_regexp.pattern) for (old_regexp, new_text) in replacements]) if exceptions: exceptClause = 'AND NOT (%s)' % ' OR '.join( ["old_text RLIKE '%s'" % prepareRegexForMySQL(exc.pattern) for exc in exceptions]) else: exceptClause = '' query = u""" SELECT page_namespace, page_title FROM page JOIN text ON (page_id = old_id) %s %s LIMIT 200""" % (whereClause, exceptClause) gen = pagegenerators.MySQLPageGenerator(query) gen = genFactory.getCombinedGenerator(gen) if not gen: # syntax error, show help text from the top of this file pywikibot.showHelp('replace') return preloadingGen = pagegenerators.PreloadingGenerator(gen) bot = ReplaceRobot(preloadingGen, replacements, exceptions, acceptall, allowoverlap, recursive, add_cat, sleep, edit_summary, site) site.login() bot.run() # Explicitly call pywikibot.stopme(). # It will make sure the callback is triggered before replace.py is unloaded. pywikibot.stopme() pywikibot.output(u'\n%s pages changed.' % bot.changed_pages)
def run(self): commons = pywikibot.Site('commons', 'commons') comment = i18n.twtranslate(self.site, 'imagetransfer-nowcommons_notice') for page in self.getPageGenerator(): if self.getOption('use_hash'): # Page -> Has the namespace | commons image -> Not images_list = page # 0 -> local image, 1 -> commons image page = pywikibot.Page(self.site, images_list[0]) else: # If use_hash is true, we have already print this before, no need self.current_page = page try: localImagePage = pywikibot.FilePage(self.site, page.title()) if localImagePage.fileIsShared(): pywikibot.output(u'File is already on Commons.') continue sha1 = localImagePage.latest_file_info.sha1 if self.getOption('use_hash'): filenameOnCommons = images_list[1] else: filenameOnCommons = self.findFilenameOnCommons( localImagePage) if not filenameOnCommons and not self.getOption('use_hash'): pywikibot.output(u'NowCommons template not found.') continue commonsImagePage = pywikibot.FilePage(commons, 'Image:%s' % filenameOnCommons) if (localImagePage.title(withNamespace=False) == commonsImagePage.title(withNamespace=False) and self.getOption('use_hash')): pywikibot.output( u'The local and the commons images have the same name') if (localImagePage.title(withNamespace=False) != commonsImagePage.title(withNamespace=False)): usingPages = list(localImagePage.usingPages()) if usingPages and usingPages != [localImagePage]: pywikibot.output(color_format( '"{lightred}{0}{default}" is still used in {1} pages.', localImagePage.title(withNamespace=False), len(usingPages))) if self.getOption('replace') is True: pywikibot.output(color_format( 'Replacing "{lightred}{0}{default}" by ' '"{lightgreen}{1}{default}\".', localImagePage.title(withNamespace=False), commonsImagePage.title(withNamespace=False))) bot = ImageBot( pg.FileLinksGenerator(localImagePage), localImagePage.title(withNamespace=False), commonsImagePage.title(withNamespace=False), '', self.getOption('replacealways'), self.getOption('replaceloose')) bot.run() # If the image is used with the urlname the # previous function won't work is_used = bool(list(pywikibot.FilePage( self.site, page.title()).usingPages(total=1))) if is_used and self.getOption('replaceloose'): bot = ImageBot( pg.FileLinksGenerator( localImagePage), localImagePage.title( withNamespace=False, asUrl=True), commonsImagePage.title( withNamespace=False), '', self.getOption('replacealways'), self.getOption('replaceloose')) bot.run() # refresh because we want the updated list usingPages = len(list(pywikibot.FilePage( self.site, page.title()).usingPages())) if usingPages > 0 and self.getOption('use_hash'): # just an enter pywikibot.input( u'There are still %s pages with this \ image, confirm the manual removal from them please.' % usingPages) else: pywikibot.output(u'Please change them manually.') continue else: pywikibot.output(color_format( 'No page is using "{lightgreen}{0}{default}" ' 'anymore.', localImagePage.title(withNamespace=False))) commonsText = commonsImagePage.get() if self.getOption('replaceonly') is False: if sha1 == commonsImagePage.latest_file_info.sha1: pywikibot.output( u'The image is identical to the one on Commons.') if (len(localImagePage.getFileVersionHistory()) > 1 and not self.getOption('use_hash')): pywikibot.output( u"This image has a version history. Please \ delete it manually after making sure that the \ old versions are not worth keeping.""") continue if self.getOption('always') is False: format_str = color_format( '\n\n>>>> Description on {lightpurple}%s' '{default} <<<<\n') pywikibot.output(format_str % page.title()) pywikibot.output(localImagePage.get()) pywikibot.output(format_str % commonsImagePage.title()) pywikibot.output(commonsText) if pywikibot.input_yn( u'Does the description on Commons contain ' 'all required source and license\n' 'information?', default=False, automatic_quit=False): localImagePage.delete( '%s [[:commons:Image:%s]]' % (comment, filenameOnCommons), prompt=False) else: localImagePage.delete( comment + ' [[:commons:Image:%s]]' % filenameOnCommons, prompt=False) else: pywikibot.output( u'The image is not identical to the one on Commons.') except (pywikibot.NoPage, pywikibot.IsRedirectPage) as e: pywikibot.output(u'%s' % e[0]) continue
def treat(self, refPage, disambPage): """ Treat a page. Parameters: disambPage - The disambiguation page or redirect we don't want anything to link to refPage - A page linking to disambPage Returns False if the user pressed q to completely quit the program. Otherwise, returns True. """ # TODO: break this function up into subroutines! include = False unlink_counter = 0 new_targets = [] try: text = refPage.get() ignoreReason = self.checkContents(text) if ignoreReason: pywikibot.output('\n\nSkipping %s because it contains %s.\n\n' % (refPage.title(), ignoreReason)) else: include = True except pywikibot.IsRedirectPage: pywikibot.output(u'%s is a redirect to %s' % (refPage.title(), disambPage.title())) if disambPage.isRedirectPage(): target = self.alternatives[0] if pywikibot.input_yn(u'Do you want to make redirect %s point ' 'to %s?' % (refPage.title(), target), default=False, automatic_quit=False): redir_text = '#%s [[%s]]' \ % (self.mysite.redirect(), target) try: refPage.put_async(redir_text, summary=self.comment) except pywikibot.PageNotSaved as error: pywikibot.output(u'Page not saved: %s' % error.args) else: choice = pywikibot.input_choice( u'Do you want to work on pages linking to %s?' % refPage.title(), [('yes', 'y'), ('no', 'n'), ('change redirect', 'c')], 'n', automatic_quit=False) if choice == 'y': gen = ReferringPageGeneratorWithIgnore(refPage, self.primary) preloadingGen = pagegenerators.PreloadingGenerator(gen) for refPage2 in preloadingGen: # run until the user selected 'quit' if not self.treat(refPage2, refPage): break elif choice == 'c': text = refPage.get(get_redirect=True) include = "redirect" except pywikibot.NoPage: pywikibot.output( u'Page [[%s]] does not seem to exist?! Skipping.' % refPage.title()) include = False if include in (True, "redirect"): # make a backup of the original text so we can show the changes later original_text = text n = 0 curpos = 0 dn = False edited = False # This loop will run until we have finished the current page while True: m = self.linkR.search(text, pos=curpos) if not m: if n == 0: pywikibot.output(u"No changes necessary in %s" % refPage.title()) return True else: # stop loop and save page break # Make sure that next time around we will not find this same hit. curpos = m.start() + 1 try: foundlink = pywikibot.Link(m.group('title'), disambPage.site) foundlink.parse() except pywikibot.Error: continue # ignore interwiki links if foundlink.site != disambPage.site: continue # Check whether the link found is to disambPage. try: if foundlink.canonical_title() != disambPage.title(): continue except pywikibot.Error: # must be a broken link pywikibot.log(u"Invalid link [[%s]] in page [[%s]]" % (m.group('title'), refPage.title())) continue n += 1 # how many bytes should be displayed around the current link context = 60 # check if there's a dn-template here already if (self.dnSkip and self.dn_template_str and self.dn_template_str[:-2] in text[m.end():m.end() + len(self.dn_template_str) + 8]): continue # This loop will run while the user doesn't choose an option # that will actually change the page while True: self.current_page = refPage if not self.always: # at the beginning of the link, start red color. # at the end of the link, reset the color to default pywikibot.output( text[max(0, m.start() - context):m.start()] + '\03{lightred}' + text[m.start():m.end()] + '\03{default}' + text[m.end():m.end() + context]) options = ['#', 'r#', '[s]kip link', '[e]dit page', '[n]ext page', '[u]nlink', '[q]uit'] if self.dn_template_str: options.append(u'[t]ag template %s' % self.dn_template_str) options.append('[m]ore context') if not edited: options.append('show [d]isambiguation page') options += ['[l]ist', '[a]dd new'] if edited: options += ['save in this form [x]'] options = concat_options('Option', 72, options) choice = pywikibot.input(options) else: choice = self.always if choice in ['a', 'A']: newAlternative = pywikibot.input(u'New alternative:') self.alternatives.append(newAlternative) self.listAlternatives() elif choice in ['e', 'E']: editor = editarticle.TextEditor() newText = editor.edit(text, jumpIndex=m.start(), highlight=disambPage.title()) # if user didn't press Cancel if newText and newText != text: text = newText break elif choice in ['d', 'D']: editor = editarticle.TextEditor() if disambPage.isRedirectPage(): disambredir = disambPage.getRedirectTarget() editor.edit( disambredir.get(), jumpIndex=m.start(), highlight=disambredir.title()) else: editor.edit( disambPage.get(), jumpIndex=m.start(), highlight=disambPage.title()) elif choice in ['l', 'L']: self.listAlternatives() elif choice in ['m', 'M']: # show more text around the link we're working on context *= 2 else: break if choice in ['e', 'E']: # user has edited the page and then pressed 'OK' edited = True curpos = 0 continue elif choice in ['n', 'N']: # skip this page if self.primary: # If run with the -primary argument, skip this # occurrence next time. self.primaryIgnoreManager.ignore(refPage) return True elif choice in ['q', 'Q']: # quit the program self.quit() elif choice in ['s', 'S']: # Next link on this page n -= 1 continue elif choice in ['x', 'X'] and edited: # Save the page as is break # The link looks like this: # [[page_title|link_text]]trailing_chars page_title = m.group('title') link_text = m.group('label') if not link_text: # or like this: [[page_title]]trailing_chars link_text = page_title if m.group('section') is None: section = '' else: section = m.group('section') trailing_chars = m.group('linktrail') if trailing_chars: link_text += trailing_chars # '?', '/' for old choice if choice in ['t', 'T', '?', '/'] and self.dn_template_str: # small chunk of text to search search_text = text[m.end():m.end() + context] # figure out where the link (and sentance) ends, put note # there end_of_word_match = re.search(r'\s', search_text) if end_of_word_match: position_split = end_of_word_match.start(0) else: position_split = 0 # insert dab needed template text = (text[:m.end() + position_split] + self.dn_template_str + text[m.end() + position_split:]) dn = True continue elif choice in ['u', 'U']: # unlink - we remove the section if there's any text = text[:m.start()] + link_text + text[m.end():] unlink_counter += 1 continue else: if len(choice) > 0 and choice[0] == 'r': # we want to throw away the original link text replaceit = link_text == page_title choice = choice[1:] elif include == "redirect": replaceit = True else: replaceit = False try: choice = int(choice) except ValueError: pywikibot.output(u"Unknown option") # step back to ask the user again what to do with the # current link curpos -= 1 continue if choice >= len(self.alternatives) or choice < 0: pywikibot.output( u"Choice out of range. Please select a number " u"between 0 and %i." % (len(self.alternatives) - 1)) # show list of possible choices self.listAlternatives() # step back to ask the user again what to do with the # current link curpos -= 1 continue new_page_title = self.alternatives[choice] repPl = pywikibot.Page(pywikibot.Link(new_page_title, disambPage.site)) if (new_page_title[0].isupper() or link_text[0].isupper()): new_page_title = repPl.title() else: new_page_title = repPl.title() new_page_title = first_lower(new_page_title) if new_page_title not in new_targets: new_targets.append(new_page_title) if replaceit and trailing_chars: newlink = "[[%s%s]]%s" % (new_page_title, section, trailing_chars) elif replaceit or (new_page_title == link_text and not section): newlink = "[[%s]]" % new_page_title # check if we can create a link with trailing characters # instead of a pipelink elif ( (len(new_page_title) <= len(link_text)) and (firstcap(link_text[:len(new_page_title)]) == firstcap(new_page_title)) and (re.sub(self.trailR, '', link_text[len(new_page_title):]) == '') and (not section) ): newlink = "[[%s]]%s" \ % (link_text[:len(new_page_title)], link_text[len(new_page_title):]) else: newlink = "[[%s%s|%s]]" \ % (new_page_title, section, link_text) text = text[:m.start()] + newlink + text[m.end():] continue pywikibot.output(text[max(0, m.start() - 30):m.end() + 30]) if text == original_text: pywikibot.output(u'\nNo changes have been made:\n') else: pywikibot.output(u'\nThe following changes have been made:\n') pywikibot.showDiff(original_text, text) pywikibot.output(u'') # save the page self.setSummaryMessage(disambPage, new_targets, unlink_counter, dn) try: refPage.put_async(text, summary=self.comment) except pywikibot.LockedPage: pywikibot.output(u'Page not saved: page is locked') except pywikibot.PageNotSaved as error: pywikibot.output(u'Page not saved: %s' % error.args) return True
def main(*args): """ Process command line arguments and generate user-config. If args is an empty list, sys.argv is used. @param args: command line arguments @type args: list of unicode """ global base_dir # set the config family and mylang values to an invalid state so that # the script can detect that the command line arguments -family & -lang # were used and and handle_args has updated these config values, # and 'force' mode can be activated below. (config.family, config.mylang) = ('wikipedia', None) local_args = pywikibot.handle_args(args) if local_args: pywikibot.output('Unknown arguments: %s' % ' '.join(local_args)) return False if config.mylang is not None: force = True pywikibot.output(u'Automatically generating user-config.py') else: force = False # Force default site of en.wikipedia (config.family, config.mylang) = ('wikipedia', 'en') username = config.usernames[config.family].get(config.mylang) args = (config.family, config.mylang, username) while not force or config.verbose_output: pywikibot.output(u'\nYour default user directory is "%s"' % base_dir) if pywikibot.input_yn("Do you want to use that directory?", default=True, automatic_quit=False, force=force): break else: new_base = change_base_dir() if new_base: base_dir = new_base break copied_config = False copied_fixes = False while not force or config.verbose_output: if os.path.exists(os.path.join(base_dir, "user-config.py")): break if pywikibot.input_yn( "Do you want to copy user files from an existing Pywikibot " "installation?", default=False, force=force, automatic_quit=False): oldpath = pywikibot.input("Path to existing user-config.py?") if not os.path.exists(oldpath): pywikibot.error("Not a valid path") continue if os.path.isfile(oldpath): # User probably typed /user-config.py at the end, so strip it oldpath = os.path.dirname(oldpath) if not os.path.isfile(os.path.join(oldpath, "user-config.py")): pywikibot.error("No user_config.py found in that directory") continue shutil.copyfile(os.path.join(oldpath, "user-config.py"), os.path.join(base_dir, "user-config.py")) copied_config = True if os.path.isfile(os.path.join(oldpath, "user-fixes.py")): shutil.copyfile(os.path.join(oldpath, "user-fixes.py"), os.path.join(base_dir, "user-fixes.py")) copied_fixes = True else: break if not os.path.isfile(os.path.join(base_dir, "user-config.py")): if ((force and not config.verbose_output) or pywikibot.input_yn( 'Create user-config.py file? Required for ' 'running bots.', default=True, automatic_quit=False, force=force)): create_user_config(args, force=force) elif not copied_config: pywikibot.output("user-config.py already exists in the directory") if not os.path.isfile(os.path.join(base_dir, "user-fixes.py")): if ((force and not config.verbose_output) or pywikibot.input_yn( 'Create user-fixes.py file? Optional and ' 'for advanced users.', force=force, default=False, automatic_quit=False)): create_user_fixes() elif not copied_fixes: pywikibot.output("user-fixes.py already exists in the directory")
def __init__(self, generator, templates, **kwargs): """ Constructor. @param generator: the pages to work on @type generator: iterable @param templates: a dictionary which maps old template names to their replacements. If remove or subst is True, it maps the names of the templates that should be removed/resolved to None. @type templates: dict """ self.availableOptions.update({ 'subst': False, 'remove': False, 'summary': None, 'addedCat': None, }) Bot.__init__(self, generator=generator, **kwargs) self.templates = templates # get edit summary message if it's empty if not self.getOption('summary'): comma = self.site.mediawiki_message('comma-separator') params = {'list': comma.join(self.templates.keys()), 'num': len(self.templates)} site = self.site if self.getOption('remove'): self.options['summary'] = i18n.twtranslate( site, 'template-removing', params) elif self.getOption('subst'): self.options['summary'] = i18n.twtranslate( site, 'template-substituting', params) else: self.options['summary'] = i18n.twtranslate( site, 'template-changing', params) # regular expression to find the original template. # {{vfd}} does the same thing as {{Vfd}}, so both will be found. # The old syntax, {{msg:vfd}}, will also be found. # The group 'parameters' will either match the parameters, or an # empty string if there are none. replacements = [] exceptions = {} builder = textlib._MultiTemplateMatchBuilder(site) for old, new in self.templates.items(): templateRegex = builder.pattern(old) if self.getOption('subst') and self.getOption('remove'): replacements.append((templateRegex, r'{{{{subst:{0!s}\g<parameters>}}}}'.format(new))) exceptions['inside-tags'] = ['ref', 'gallery'] elif self.getOption('subst'): replacements.append((templateRegex, r'{{{{subst:{0!s}\g<parameters>}}}}'.format(old))) exceptions['inside-tags'] = ['ref', 'gallery'] elif self.getOption('remove'): replacements.append((templateRegex, '')) else: template = pywikibot.Page(self.site, new, ns=10) if not template.exists(): pywikibot.warning(u'Template "{0!s}" does not exist.'.format(new)) if not pywikibot.input_yn('Do you want to proceed anyway?', default=False, automatic_quit=False): continue replacements.append((templateRegex, r'{{{{{0!s}\g<parameters>}}}}'.format(new))) super(TemplateRobot, self).__init__( generator, replacements, exceptions, always=self.getOption('always'), addedCat=self.getOption('addedCat'), summary=self.getOption('summary'))
def get_site_and_lang(default_family='wikipedia', default_lang='en', default_username=None, force=False): """ Ask the user for the family, language and username. @param default_family: The default family which should be chosen. @type default_family: None or str @param default_lang: The default language which should be chosen, if the family supports this language. @type default_lang: None or str @param default_username: The default username which should be chosen. @type default_username: None or str @return: The family, language and username @rtype: tuple of three str """ known_families = sorted(pywikibot.config2.family_files.keys()) if default_family not in known_families: default_family = None fam = pywikibot.bot.input_list_choice( u"Select family of sites we are working on, " u"just enter the number or name", known_families, force=force, default=default_family) fam = pywikibot.family.Family.load(fam) if hasattr(fam, "langs"): if hasattr(fam, "languages_by_size"): by_size = [ code for code in fam.languages_by_size if code in fam.langs.keys() ] else: by_size = [] known_langs = by_size + sorted( set(fam.langs.keys()).difference(by_size)) else: known_langs = [] if len(known_langs) == 0: pywikibot.output('There were no known languages found in {0}.'.format( fam.name)) default_lang = None elif len(known_langs) == 1: pywikibot.output('The only known language: {0}'.format(known_langs[0])) default_lang = known_langs[0] else: pywikibot.output("This is the list of known languages:") pywikibot.output(u", ".join(known_langs)) if default_lang not in known_langs: if default_lang != 'en' and 'en' in known_langs: default_lang = 'en' else: default_lang = None message = "The language code of the site we're working on" mylang = None while not mylang: mylang = pywikibot.input(message, default=default_lang, force=force) if known_langs and mylang and mylang not in known_langs: if not pywikibot.input_yn("The language code {0} is not in the " "list of known languages. Do you want " "to continue?".format(mylang), default=False, automatic_quit=False): mylang = None message = u"Username on {0}:{1}".format(mylang, fam.name) username = pywikibot.input(message, default=default_username, force=force) # Escape ''s if username: username = username.replace("'", "\\'") return fam.name, mylang, username
def treat(self, refPage, disambPage): """Treat a page. @param disambPage: the disambiguation page or redirect we don't want anything to link to @type disambPage: pywikibot.Page @param refPage: a page linking to disambPage @type refPage: pywikibot.Page @return: False if the user pressed q to completely quit the program, True otherwise @rtype: bool """ # TODO: break this function up into subroutines! self.current_page = refPage include = False unlink_counter = 0 new_targets = [] try: text = refPage.get() ignoreReason = self.checkContents(text) if ignoreReason: pywikibot.output('\n\nSkipping %s because it contains %s.\n\n' % (refPage.title(), ignoreReason)) else: include = True except pywikibot.IsRedirectPage: pywikibot.output(u'%s is a redirect to %s' % (refPage.title(), disambPage.title())) if disambPage.isRedirectPage(): target = self.alternatives[0] if pywikibot.input_yn(u'Do you want to make redirect %s point ' 'to %s?' % (refPage.title(), target), default=False, automatic_quit=False): redir_text = '#%s [[%s]]' \ % (self.mysite.redirect(), target) try: refPage.put_async(redir_text, summary=self.comment) except pywikibot.PageNotSaved as error: pywikibot.output(u'Page not saved: %s' % error.args) else: choice = pywikibot.input_choice( u'Do you want to work on pages linking to %s?' % refPage.title(), [('yes', 'y'), ('no', 'n'), ('change redirect', 'c')], 'n', automatic_quit=False) if choice == 'y': gen = ReferringPageGeneratorWithIgnore( refPage, self.primary, main_only=self.main_only ) preloadingGen = pagegenerators.PreloadingGenerator(gen) for refPage2 in preloadingGen: # run until the user selected 'quit' if not self.treat(refPage2, refPage): break elif choice == 'c': text = refPage.get(get_redirect=True) include = "redirect" except pywikibot.NoPage: pywikibot.output( u'Page [[%s]] does not seem to exist?! Skipping.' % refPage.title()) include = False if include in (True, "redirect"): # make a backup of the original text so we can show the changes later original_text = text n = 0 curpos = 0 dn = False edited = False # This loop will run until we have finished the current page while True: m = self.linkR.search(text, pos=curpos) if not m: if n == 0: pywikibot.output(u"No changes necessary in %s" % refPage.title()) return True else: # stop loop and save page break # Make sure that next time around we will not find this same hit. curpos = m.start() + 1 try: foundlink = pywikibot.Link(m.group('title'), disambPage.site) foundlink.parse() except pywikibot.Error: continue # ignore interwiki links if foundlink.site != disambPage.site: continue # Check whether the link found is to disambPage. try: if foundlink.canonical_title() != disambPage.title(): continue except pywikibot.Error: # must be a broken link pywikibot.log(u"Invalid link [[%s]] in page [[%s]]" % (m.group('title'), refPage.title())) continue n += 1 # how many bytes should be displayed around the current link context = 60 # check if there's a dn-template here already if (self.dnSkip and self.dn_template_str and self.dn_template_str[:-2] in text[m.end():m.end() + len(self.dn_template_str) + 8]): continue edit = EditOption('edit page', 'e', text, m.start(), disambPage.title()) context_option = HighlightContextOption( 'more context', 'm', text, 60, start=m.start(), end=m.end()) context_option.before_question = True options = [ListOption(self.alternatives, ''), ListOption(self.alternatives, 'r'), StandardOption('skip link', 's'), edit, StandardOption('next page', 'n'), StandardOption('unlink', 'u')] if self.dn_template_str: # '?', '/' for old choice options += [AliasOption('tag template %s' % self.dn_template_str, ['t', '?', '/'])] options += [context_option] if not edited: options += [ShowPageOption('show disambiguation page', 'd', m.start(), disambPage)] options += [ OutputProxyOption('list', 'l', SequenceOutputter(self.alternatives)), AddAlternativeOption('add new', 'a', SequenceOutputter(self.alternatives))] if edited: options += [StandardOption('save in this form', 'x')] # TODO: Output context on each question answer = pywikibot.input_choice('Option', options, default=self.always, force=bool(self.always)) if answer == 'x': assert edited, 'invalid option before editing' break elif answer == 's': n -= 1 # TODO what's this for? continue elif answer == 'e': text = edit.new_text edited = True curpos = 0 continue elif answer == 'n': # skip this page if self.primary: # If run with the -primary argument, skip this # occurrence next time. self.primaryIgnoreManager.ignore(refPage) return True # The link looks like this: # [[page_title|link_text]]trailing_chars page_title = m.group('title') link_text = m.group('label') if not link_text: # or like this: [[page_title]]trailing_chars link_text = page_title if m.group('section') is None: section = '' else: section = m.group('section') trailing_chars = m.group('linktrail') if trailing_chars: link_text += trailing_chars if answer == 't': assert self.dn_template_str # small chunk of text to search search_text = text[m.end():m.end() + context] # figure out where the link (and sentance) ends, put note # there end_of_word_match = re.search(r'\s', search_text) if end_of_word_match: position_split = end_of_word_match.start(0) else: position_split = 0 # insert dab needed template text = (text[:m.end() + position_split] + self.dn_template_str + text[m.end() + position_split:]) dn = True continue elif answer == 'u': # unlink - we remove the section if there's any text = text[:m.start()] + link_text + text[m.end():] unlink_counter += 1 continue else: # Check that no option from above was missed assert isinstance(answer, tuple), 'only tuple answer left.' assert answer[0] in ['r', ''], 'only valid tuple answers.' if answer[0] == 'r': # we want to throw away the original link text replaceit = link_text == page_title elif include == "redirect": replaceit = True else: replaceit = False new_page_title = answer[1] repPl = pywikibot.Page(pywikibot.Link(new_page_title, disambPage.site)) if (new_page_title[0].isupper() or link_text[0].isupper()): new_page_title = repPl.title() else: new_page_title = repPl.title() new_page_title = first_lower(new_page_title) if new_page_title not in new_targets: new_targets.append(new_page_title) if replaceit and trailing_chars: newlink = "[[%s%s]]%s" % (new_page_title, section, trailing_chars) elif replaceit or (new_page_title == link_text and not section): newlink = "[[%s]]" % new_page_title # check if we can create a link with trailing characters # instead of a pipelink elif ( (len(new_page_title) <= len(link_text)) and (firstcap(link_text[:len(new_page_title)]) == firstcap(new_page_title)) and (re.sub(self.trailR, '', link_text[len(new_page_title):]) == '') and (not section) ): newlink = "[[%s]]%s" \ % (link_text[:len(new_page_title)], link_text[len(new_page_title):]) else: newlink = "[[%s%s|%s]]" \ % (new_page_title, section, link_text) text = text[:m.start()] + newlink + text[m.end():] continue pywikibot.output(text[max(0, m.start() - 30):m.end() + 30]) if text == original_text: pywikibot.output(u'\nNo changes have been made:\n') else: pywikibot.output(u'\nThe following changes have been made:\n') pywikibot.showDiff(original_text, text) pywikibot.output(u'') # save the page self.setSummaryMessage(disambPage, new_targets, unlink_counter, dn) try: refPage.put_async(text, summary=self.comment) except pywikibot.LockedPage: pywikibot.output(u'Page not saved: page is locked') except pywikibot.PageNotSaved as error: pywikibot.output(u'Page not saved: %s' % error.args) return True
def process_filename(self, file_url): """Return base filename portion of file_url.""" # Isolate the pure name filename = file_url # Filename may be either a URL or a local file path if '://' in filename: # extract the path portion of the URL filename = urlparse(filename).path filename = os.path.basename(filename) if self.use_filename: filename = self.use_filename if self.filename_prefix: filename = self.filename_prefix + filename if not self.keep_filename: pywikibot.output( '\nThe filename on the target wiki will default to: {}\n'. format(filename)) assert not self.opt.always newfn = pywikibot.input( 'Enter a better name, or press enter to accept:') if newfn != '': filename = newfn # FIXME: these 2 belong somewhere else, presumably in family # forbidden characters are handled by pywikibot/page.py forbidden = ':*?/\\' # to be extended try: allowed_formats = self.target_site.siteinfo.get('fileextensions', get_default=False) except KeyError: allowed_formats = [] else: allowed_formats = [item['ext'] for item in allowed_formats] # ask until it's valid first_check = True while True: if not first_check: if self.opt.always: filename = None else: filename = pywikibot.input('Enter a better name, or press ' 'enter to skip the file:') if not filename: return None first_check = False ext = os.path.splitext(filename)[1].lower().strip('.') # are any chars in forbidden also in filename? invalid = set(forbidden) & set(filename) if invalid: c = ''.join(invalid) pywikibot.output( 'Invalid character(s): {}. Please try again'.format(c)) continue if allowed_formats and ext not in allowed_formats: if self.opt.always: pywikibot.output('File format is not one of ' '[{}]'.format(' '.join(allowed_formats))) continue if not pywikibot.input_yn( 'File format is not one of [{}], but {!r}. Continue?'. format(' '.join(allowed_formats), ext), default=False): continue potential_file_page = pywikibot.FilePage(self.target_site, filename) if potential_file_page.exists(): overwrite = self._handle_warning('exists') if overwrite is False: pywikibot.output( 'File exists and you asked to abort. Skipping.') return None if potential_file_page.has_permission(): if overwrite is None: overwrite = not pywikibot.input_yn( 'File with name {} already exists. ' 'Would you like to change the name? ' '(Otherwise file will be overwritten.)'.format( filename), default=True, automatic_quit=False) if not overwrite: continue break pywikibot.output('File with name {} already exists and ' 'cannot be overwritten.'.format(filename)) continue with suppress(NoPageError): if potential_file_page.file_is_shared(): pywikibot.output( 'File with name {} already exists in shared ' 'repository and cannot be overwritten.'.format( filename)) continue break # A proper description for the submission. # Empty descriptions are not accepted. if self.description: pywikibot.output('The suggested description is:\n{}'.format( self.description)) while not self.description or self.verify_description: if not self.description: pywikibot.output( color_format( '{lightred}It is not possible to upload a file ' 'without a description.{default}')) assert not self.opt.always # if no description, ask if user want to add one or quit, # and loop until one is filled. # if self.verify_description, ask if user want to change it # or continue. if self.description: question = 'Do you want to change this description?' else: question = 'No description was given. Add one?' if pywikibot.input_yn(question, default=not self.description, automatic_quit=self.description): from pywikibot import editor as editarticle editor = editarticle.TextEditor() try: new_description = editor.edit(self.description) except ImportError: raise except Exception as e: pywikibot.error(e) continue # if user saved / didn't press Cancel if new_description: self.description = new_description elif not self.description: raise QuitKeyboardInterrupt self.verify_description = False return filename
def __init__(self, generator, templates, **kwargs): """ Constructor. @param generator: the pages to work on @type generator: iterable @param templates: a dictionary which maps old template names to their replacements. If remove or subst is True, it maps the names of the templates that should be removed/resolved to None. @type templates: dict """ self.availableOptions.update({ 'subst': False, 'remove': False, 'summary': None, 'addedCat': None, }) Bot.__init__(self, generator=generator, **kwargs) self.templates = templates # get edit summary message if it's empty if not self.getOption('summary'): comma = self.site.mediawiki_message('comma-separator') params = {'list': comma.join(self.templates.keys()), 'num': len(self.templates)} if self.getOption('remove'): self.options['summary'] = i18n.twtranslate( self.site, 'template-removing', params) elif self.getOption('subst'): self.options['summary'] = i18n.twtranslate( self.site, 'template-substituting', params) else: self.options['summary'] = i18n.twtranslate( self.site, 'template-changing', params) replacements = [] exceptions = {} builder = textlib._MultiTemplateMatchBuilder(self.site) for old, new in self.templates.items(): templateRegex = builder.pattern(old) if self.getOption('subst') and self.getOption('remove'): replacements.append((templateRegex, r'{{subst:%s\g<parameters>}}' % new)) exceptions['inside-tags'] = ['ref', 'gallery', 'poem', 'pagelist', ] elif self.getOption('subst'): replacements.append((templateRegex, r'{{subst:%s\g<parameters>}}' % old)) exceptions['inside-tags'] = ['ref', 'gallery', 'poem', 'pagelist', ] elif self.getOption('remove'): separate_line_regex = re.compile( r'^[*#:]* *{0} *\n'.format(templateRegex.pattern), re.DOTALL | re.MULTILINE) replacements.append((separate_line_regex, '')) spaced_regex = re.compile( r' +{0} +'.format(templateRegex.pattern), re.DOTALL) replacements.append((spaced_regex, ' ')) replacements.append((templateRegex, '')) else: template = pywikibot.Page(self.site, new, ns=10) if not template.exists(): pywikibot.warning(u'Template "%s" does not exist.' % new) if not pywikibot.input_yn('Do you want to proceed anyway?', default=False, automatic_quit=False): continue replacements.append((templateRegex, r'{{%s\g<parameters>}}' % new)) super(TemplateRobot, self).__init__( generator, replacements, exceptions, always=self.getOption('always'), addedCat=self.getOption('addedCat'), summary=self.getOption('summary'))
def main(*args): """ Process command line arguments and generate user-config. If args is an empty list, sys.argv is used. @param args: command line arguments @type args: list of unicode """ global base_dir # set the config family and mylang values to an invalid state so that # the script can detect that the command line arguments -family & -lang # were used and and handle_args has updated these config values, # and 'force' mode can be activated below. (config.family, config.mylang) = ('wikipedia', None) local_args = pywikibot.handle_args(args) if local_args: pywikibot.output('Unknown arguments: %s' % ' '.join(local_args)) return False if config.mylang is not None: force = True pywikibot.output(u'Automatically generating user-config.py') else: force = False # Force default site of en.wikipedia (config.family, config.mylang) = ('wikipedia', 'en') username = config.usernames[config.family].get(config.mylang) args = (config.family, config.mylang, username) while not force or config.verbose_output: pywikibot.output(u'\nYour default user directory is "%s"' % base_dir) if pywikibot.input_yn("Do you want to use that directory?", default=True, automatic_quit=False, force=force): break else: new_base = change_base_dir() if new_base: base_dir = new_base break copied_config = False copied_fixes = False while not force or config.verbose_output: if os.path.exists(os.path.join(base_dir, "user-config.py")): break if pywikibot.input_yn( "Do you want to copy user files from an existing Pywikibot " "installation?", default=False, force=force, automatic_quit=False): oldpath = pywikibot.input("Path to existing user-config.py?") if not os.path.exists(oldpath): pywikibot.error("Not a valid path") continue if os.path.isfile(oldpath): # User probably typed /user-config.py at the end, so strip it oldpath = os.path.dirname(oldpath) if not os.path.isfile(os.path.join(oldpath, "user-config.py")): pywikibot.error("No user_config.py found in that directory") continue shutil.copyfile(os.path.join(oldpath, "user-config.py"), os.path.join(base_dir, "user-config.py")) copied_config = True if os.path.isfile(os.path.join(oldpath, "user-fixes.py")): shutil.copyfile(os.path.join(oldpath, "user-fixes.py"), os.path.join(base_dir, "user-fixes.py")) copied_fixes = True else: break if not os.path.isfile(os.path.join(base_dir, "user-config.py")): if ((force and not config.verbose_output) or pywikibot.input_yn('Create user-config.py file? Required for ' 'running bots.', default=True, automatic_quit=False, force=force)): create_user_config(args, force=force) elif not copied_config: pywikibot.output("user-config.py already exists in the directory") if not os.path.isfile(os.path.join(base_dir, "user-fixes.py")): if ((force and not config.verbose_output) or pywikibot.input_yn('Create user-fixes.py file? Optional and ' 'for advanced users.', force=force, default=False, automatic_quit=False)): create_user_fixes() elif not copied_fixes: pywikibot.output("user-fixes.py already exists in the directory")
def treat_disamb_only(self, refPage, disambPage): """Resolve the links to disambPage but don't look for its redirects. @param disambPage: the disambiguation page or redirect we don't want anything to link to @type disambPage: pywikibot.Page @param refPage: a page linking to disambPage @type refPage: pywikibot.Page @return: "nextpage" if the user enters "n" to skip this page, "nochange" if the page needs no change, and "done" if the page is processed successfully @rtype: str """ # TODO: break this function up into subroutines! self.current_page = refPage include = False unlink_counter = 0 new_targets = [] try: text = refPage.get() ignoreReason = self.checkContents(text) if ignoreReason: pywikibot.output( '\n\nSkipping %s because it contains %s.\n\n' % (refPage.title(), ignoreReason)) else: include = True except pywikibot.IsRedirectPage: pywikibot.output(u'%s is a redirect to %s' % (refPage.title(), disambPage.title())) if disambPage.isRedirectPage(): target = self.alternatives[0] if pywikibot.input_yn(u'Do you want to make redirect %s point ' 'to %s?' % (refPage.title(), target), default=False, automatic_quit=False): redir_text = '#%s [[%s]]' \ % (self.mysite.redirect(), target) try: refPage.put(redir_text, summary=self.comment, asynchronous=True) except pywikibot.PageNotSaved as error: pywikibot.output(u'Page not saved: %s' % error.args) else: choice = pywikibot.input_choice( u'Do you want to work on pages linking to %s?' % refPage.title(), [('yes', 'y'), ('no', 'n'), ('change redirect', 'c')], 'n', automatic_quit=False) if choice == 'y': gen = ReferringPageGeneratorWithIgnore( refPage, self.primary, main_only=self.main_only) preloadingGen = pagegenerators.PreloadingGenerator(gen) for refPage2 in preloadingGen: # run until the user selected 'quit' self.treat(refPage2, refPage) elif choice == 'c': text = refPage.get(get_redirect=True) include = "redirect" except pywikibot.NoPage: pywikibot.output( u'Page [[%s]] does not seem to exist?! Skipping.' % refPage.title()) include = False if include in (True, "redirect"): # save the original text so we can show the changes later original_text = text n = 0 curpos = 0 dn = False edited = False # This loop will run until we have finished the current page while True: m = self.linkR.search(text, pos=curpos) if not m: if n == 0: # No changes necessary for this disambiguation title. return 'nochange' else: # stop loop and save page break # Ensure that next time around we will not find this same hit. curpos = m.start() + 1 try: foundlink = pywikibot.Link(m.group('title'), disambPage.site) foundlink.parse() except pywikibot.Error: continue # ignore interwiki links if foundlink.site != disambPage.site: continue # Check whether the link found is to disambPage. try: if foundlink.canonical_title() != disambPage.title(): continue except pywikibot.Error: # must be a broken link pywikibot.log(u"Invalid link [[%s]] in page [[%s]]" % (m.group('title'), refPage.title())) continue n += 1 # how many bytes should be displayed around the current link context = 60 # check if there's a dn-template here already if (self.dnSkip and self.dn_template_str and self.dn_template_str[:-2] in text[m.end():m.end() + len(self.dn_template_str) + 8]): continue edit = EditOption('edit page', 'e', text, m.start(), disambPage.title()) context_option = HighlightContextOption('more context', 'm', text, 60, start=m.start(), end=m.end()) context_option.before_question = True options = [ ListOption(self.alternatives, ''), ListOption(self.alternatives, 'r'), StandardOption('skip link', 's'), edit, StandardOption('next page', 'n'), StandardOption('unlink', 'u') ] if self.dn_template_str: # '?', '/' for old choice options += [ AliasOption('tag template %s' % self.dn_template_str, ['t', '?', '/']) ] options += [context_option] if not edited: options += [ ShowPageOption('show disambiguation page', 'd', m.start(), disambPage) ] options += [ OutputProxyOption('list', 'l', SequenceOutputter(self.alternatives)), AddAlternativeOption('add new', 'a', SequenceOutputter(self.alternatives)) ] if edited: options += [StandardOption('save in this form', 'x')] # TODO: Output context on each question answer = pywikibot.input_choice('Option', options, default=self.always, force=bool(self.always)) if answer == 'x': assert edited, 'invalid option before editing' break elif answer == 's': n -= 1 # TODO what's this for? continue elif answer == 'e': text = edit.new_text edited = True curpos = 0 continue elif answer == 'n': # skip this page if self.primary: # If run with the -primary argument, skip this # occurrence next time. self.primaryIgnoreManager.ignore(refPage) return 'nextpage' # The link looks like this: # [[page_title|link_text]]trailing_chars page_title = m.group('title') link_text = m.group('label') if not link_text: # or like this: [[page_title]]trailing_chars link_text = page_title if m.group('section') is None: section = '' else: section = m.group('section') trailing_chars = m.group('linktrail') if trailing_chars: link_text += trailing_chars if answer == 't': assert self.dn_template_str # small chunk of text to search search_text = text[m.end():m.end() + context] # figure out where the link (and sentance) ends, put note # there end_of_word_match = re.search(r'\s', search_text) if end_of_word_match: position_split = end_of_word_match.start(0) else: position_split = 0 # insert dab needed template text = (text[:m.end() + position_split] + self.dn_template_str + text[m.end() + position_split:]) dn = True continue elif answer == 'u': # unlink - we remove the section if there's any text = text[:m.start()] + link_text + text[m.end():] unlink_counter += 1 continue else: # Check that no option from above was missed assert isinstance(answer, tuple), 'only tuple answer left.' assert answer[0] in ['r', ''], 'only valid tuple answers.' if answer[0] == 'r': # we want to throw away the original link text replaceit = link_text == page_title elif include == "redirect": replaceit = True else: replaceit = False new_page_title = answer[1] repPl = pywikibot.Page( pywikibot.Link(new_page_title, disambPage.site)) if (new_page_title[0].isupper() or link_text[0].isupper()): new_page_title = repPl.title() else: new_page_title = repPl.title() new_page_title = first_lower(new_page_title) if new_page_title not in new_targets: new_targets.append(new_page_title) if replaceit and trailing_chars: newlink = "[[%s%s]]%s" % (new_page_title, section, trailing_chars) elif replaceit or (new_page_title == link_text and not section): newlink = "[[%s]]" % new_page_title # check if we can create a link with trailing characters # instead of a pipelink elif ((len(new_page_title) <= len(link_text)) and (firstcap(link_text[:len(new_page_title)]) == firstcap(new_page_title)) and (re.sub(self.trailR, '', link_text[len(new_page_title):]) == '') and (not section)): newlink = "[[%s]]%s" \ % (link_text[:len(new_page_title)], link_text[len(new_page_title):]) else: newlink = "[[%s%s|%s]]" \ % (new_page_title, section, link_text) text = text[:m.start()] + newlink + text[m.end():] continue # Todo: This line is unreachable (T155337) pywikibot.output(text[max(0, m.start() - 30):m.end() + 30]) if text == original_text: pywikibot.output(u'\nNo changes have been made:\n') else: pywikibot.output(u'\nThe following changes have been made:\n') pywikibot.showDiff(original_text, text) pywikibot.output(u'') # save the page self.setSummaryMessage(disambPage, new_targets, unlink_counter, dn) try: refPage.put(text, summary=self.comment, asynchronous=True) except pywikibot.LockedPage: pywikibot.output(u'Page not saved: page is locked') except pywikibot.PageNotSaved as error: pywikibot.output(u'Page not saved: %s' % error.args) return 'done'
def get_site_and_lang(default_family='wikipedia', default_lang='en', default_username=None, force=False): """ Ask the user for the family, language and username. @param default_family: The default family which should be chosen. @type default_family: None or str @param default_lang: The default language which should be chosen, if the family supports this language. @type default_lang: None or str @param default_username: The default username which should be chosen. @type default_username: None or str @return: The family, language and username @rtype: tuple of three str """ known_families = sorted(pywikibot.config2.family_files.keys()) if default_family not in known_families: default_family = None fam = pywikibot.bot.input_list_choice( u"Select family of sites we are working on, " u"just enter the number or name", known_families, force=force, default=default_family) fam = pywikibot.family.Family.load(fam) if hasattr(fam, "langs"): if hasattr(fam, "languages_by_size"): by_size = [code for code in fam.languages_by_size if code in fam.langs.keys()] else: by_size = [] known_langs = by_size + sorted( set(fam.langs.keys()).difference(by_size)) else: known_langs = [] if len(known_langs) == 0: pywikibot.output('There were no known languages found in {0}.'.format(fam.name)) default_lang = None elif len(known_langs) == 1: pywikibot.output('The only known language: {0}'.format(known_langs[0])) default_lang = known_langs[0] else: pywikibot.output("This is the list of known languages:") pywikibot.output(u", ".join(known_langs)) if default_lang not in known_langs: if default_lang != 'en' and 'en' in known_langs: default_lang = 'en' else: default_lang = None message = "The language code of the site we're working on" mylang = None while not mylang: mylang = pywikibot.input(message, default=default_lang, force=force) if known_langs and mylang and mylang not in known_langs: if not pywikibot.input_yn("The language code {0} is not in the " "list of known languages. Do you want " "to continue?".format(mylang), default=False, automatic_quit=False): mylang = None message = u"Username on {0}:{1}".format(mylang, fam.name) username = pywikibot.input(message, default=default_username, force=force) # Escape ''s if username: username = username.replace("'", "\\'") return fam.name, mylang, username
def upload_image(self, debug=False): """Upload the image at self.url to the target wiki. Return the filename that was used to upload the image. If the upload fails, ask the user whether to try again or not. If the user chooses not to retry, return null. """ filename = self.process_filename() site = self.targetSite imagepage = pywikibot.FilePage(site, filename) # normalizes filename imagepage.text = self.description pywikibot.output(u'Uploading file to %s via API....' % site) try: apiIgnoreWarnings = False if self.ignoreWarning is True: apiIgnoreWarnings = True if self.uploadByUrl: site.upload(imagepage, source_url=self.url, ignore_warnings=apiIgnoreWarnings) else: if "://" in self.url: temp = self.read_file_content() else: temp = self.url site.upload(imagepage, source_filename=temp, ignore_warnings=apiIgnoreWarnings, chunk_size=self.chunk_size) except pywikibot.data.api.UploadWarning as warn: pywikibot.output( u'We got a warning message: {0} - {1}'.format(warn.code, warn.message)) if self.abort_on_warn(warn.code): answer = False elif self.ignore_on_warn(warn.code): answer = True else: answer = pywikibot.input_yn(u"Do you want to ignore?", default=False, automatic_quit=False) if answer: self.ignoreWarning = True self.keepFilename = True return self.upload_image(debug) else: pywikibot.output(u"Upload aborted.") return except pywikibot.data.api.APIError as error: if error.code == u'uploaddisabled': pywikibot.error("Upload error: Local file uploads are disabled on %s." % site) else: pywikibot.error("Upload error: ", exc_info=True) except Exception: pywikibot.error("Upload error: ", exc_info=True) else: # No warning, upload complete. pywikibot.output(u"Upload successful.") return filename # data['filename']
def main(*args): """ Process command line arguments and invoke bot. If args is an empty list, sys.argv is used. @param args: command line arguments @type args: list of unicode """ # the option that's always selected when the bot wonders what to do with # a link. If it's None, the user is prompted (default behaviour). always = None alternatives = [] getAlternatives = True dnSkip = False generator = None primary = False main_only = False # For sorting the linked pages, case can be ignored minimum = 0 local_args = pywikibot.handle_args(args) generator_factory = pagegenerators.GeneratorFactory( positional_arg_name='page') for arg in local_args: if arg.startswith('-primary:'): primary = True getAlternatives = False alternatives.append(arg[9:]) elif arg == '-primary': primary = True elif arg.startswith('-always:'): always = arg[8:] elif arg.startswith('-pos:'): if arg[5] != ':': mysite = pywikibot.Site() page = pywikibot.Page(pywikibot.Link(arg[5:], mysite)) if page.exists(): alternatives.append(page.title()) else: if pywikibot.input_yn( u'Possibility %s does not actually exist. Use it ' 'anyway?' % page.title(), default=False, automatic_quit=False): alternatives.append(page.title()) else: alternatives.append(arg[5:]) elif arg == '-just': getAlternatives = False elif arg == '-dnskip': dnSkip = True elif arg == '-main': main_only = True elif arg.startswith('-min:'): minimum = int(arg[5:]) elif arg.startswith('-start'): try: generator = pagegenerators.CategorizedPageGenerator( pywikibot.Site().disambcategory(), start=arg[7:], namespaces=[0]) except pywikibot.NoPage: pywikibot.output("Disambiguation category for your wiki is not known.") raise else: generator_factory.handleArg(arg) site = pywikibot.Site() generator = generator_factory.getCombinedGenerator(generator) if not generator: pywikibot.bot.suggest_help(missing_generator=True) return False site.login() bot = DisambiguationRobot(always, alternatives, getAlternatives, dnSkip, generator, primary, main_only, minimum=minimum) bot.run()
def main(*args): """ Process command line arguments and invoke bot. If args is an empty list, sys.argv is used. @param args: command line arguments @type args: list of unicode """ gen = None notitle = False fmt = '1' outputlang = None page_get = False base_dir = None encoding = config.textfile_encoding # Process global args and prepare generator args parser local_args = pywikibot.handle_args(args) genFactory = GeneratorFactory() for arg in local_args: if arg == '-notitle': notitle = True elif arg.startswith('-format:'): fmt = arg[len('-format:'):] fmt = fmt.replace(u'\\03{{', u'\03{{') elif arg.startswith('-outputlang:'): outputlang = arg[len('-outputlang:'):] elif arg == '-get': page_get = True elif arg.startswith('-save'): base_dir = arg.partition(':')[2] or '.' elif arg.startswith('-encode:'): encoding = arg.partition(':')[2] else: genFactory.handleArg(arg) if base_dir: base_dir = os.path.expanduser(base_dir) if not os.path.isabs(base_dir): base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir)) if not os.path.exists(base_dir): pywikibot.output(u'Directory "%s" does not exist.' % base_dir) choice = pywikibot.input_yn( u'Do you want to create it ("No" to continue without saving)?') if choice: os.makedirs(base_dir, mode=0o744) else: base_dir = None elif not os.path.isdir(base_dir): # base_dir is a file. pywikibot.warning(u'Not a directory: "%s"\n' u'Skipping saving ...' % base_dir) base_dir = None gen = genFactory.getCombinedGenerator() if gen: i = 0 for i, page in enumerate(gen, start=1): if not notitle: page_fmt = Formatter(page, outputlang) pywikibot.stdout(page_fmt.output(num=i, fmt=fmt)) if page_get: try: pywikibot.output(page.text, toStdout=True) except pywikibot.Error as err: pywikibot.output(err) if base_dir: filename = os.path.join(base_dir, page.title(as_filename=True)) pywikibot.output(u'Saving %s to %s' % (page.title(), filename)) with open(filename, mode='wb') as f: f.write(page.text.encode(encoding)) pywikibot.output(u"%i page(s) found" % i) else: pywikibot.showHelp()
def main(*args): """ Process command line arguments and invoke bot. If args is an empty list, sys.argv is used. @param args: command line arguments @type args: str """ gen = None notitle = False fmt = '1' outputlang = None page_get = False base_dir = None encoding = config.textfile_encoding page_target = None overwrite = False summary = 'listpages-save-list' # Process global args and prepare generator args parser local_args = pywikibot.handle_args(args) gen_factory = GeneratorFactory() for arg in local_args: option, sep, value = arg.partition(':') if option == '-notitle': notitle = True elif option == '-format': fmt = value.replace('\\03{{', '\03{{') if not fmt.strip(): notitle = True elif option == '-outputlang:': outputlang = value elif option == '-get': page_get = True elif option == '-save': base_dir = value or '.' elif option == '-encode': encoding = value elif option == '-put': page_target = value elif option == '-overwrite': overwrite = True elif option == '-summary': summary = value else: gen_factory.handleArg(arg) if base_dir: base_dir = os.path.expanduser(base_dir) if not os.path.isabs(base_dir): base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir)) if not os.path.exists(base_dir): pywikibot.output( 'Directory "{0}" does not exist.'.format(base_dir)) choice = pywikibot.input_yn( 'Do you want to create it ("No" to continue without saving)?') if choice: os.makedirs(base_dir, mode=0o744) else: base_dir = None elif not os.path.isdir(base_dir): # base_dir is a file. pywikibot.warning('Not a directory: "{0}"\n' 'Skipping saving ...'.format(base_dir)) base_dir = None if page_target: site = pywikibot.Site() page_target = pywikibot.Page(site, page_target) if not overwrite and page_target.exists(): pywikibot.bot.suggest_help( additional_text='Page {0} already exists.\n' 'You can use the -overwrite argument to ' 'replace the content of this page.'.format( page_target.title(as_link=True))) return False if re.match('^[a-z_-]+$', summary): summary = i18n.twtranslate(site, summary) gen = gen_factory.getCombinedGenerator() if gen: i = 0 output_list = [] for i, page in enumerate(gen, start=1): if not notitle: page_fmt = Formatter(page, outputlang) output_list += [page_fmt.output(num=i, fmt=fmt)] pywikibot.stdout(output_list[-1]) if page_get: try: pywikibot.stdout(page.text) except pywikibot.Error as err: pywikibot.output(err) if base_dir: filename = os.path.join(base_dir, page.title(as_filename=True)) pywikibot.output('Saving {0} to {1}'.format( page.title(), filename)) with open(filename, mode='wb') as f: f.write(page.text.encode(encoding)) pywikibot.output('{0} page(s) found'.format(i)) if page_target: page_target.text = '\n'.join(output_list) page_target.save(summary=summary) return True else: pywikibot.bot.suggest_help(missing_generator=True) return False
def run_bot(give_url, image_url, desc): """Run the bot.""" url = give_url image_url = '' if url == '': if image_url: url = pywikibot.input(u"What URL range should I check " u"(use $ for the part that is changeable)") else: url = pywikibot.input(u"From what URL should I get the images?") if image_url: minimum = 1 maximum = 99 answer = pywikibot.input( u"What is the first number to check (default: 1)") if answer: minimum = int(answer) answer = pywikibot.input( u"What is the last number to check (default: 99)") if answer: maximum = int(answer) if not desc: basicdesc = pywikibot.input( u"What text should be added at the end of " u"the description of each image from this url?") else: basicdesc = desc if image_url: ilinks = [] i = minimum while i <= maximum: ilinks += [url.replace("$", str(i))] i += 1 else: ilinks = get_imagelinks(url) for image in ilinks: if pywikibot.input_yn('Include image %s?' % image, default=False, automatic_quit=False): desc = pywikibot.input(u"Give the description of this image:") categories = [] while True: cat = pywikibot.input(u"Specify a category (or press enter to " u"end adding categories)") if not cat.strip(): break if ":" in cat: categories.append(u"[[%s]]" % cat) else: categories.append(u"[[%s:%s]]" % (mysite.namespace(14), cat)) desc += "\r\n\r\n" + basicdesc + "\r\n\r\n" + \ "\r\n".join(categories) uploadBot = UploadRobot(image, description=desc) uploadBot.run() elif answer == 's': break
def main(*args): """ Process command line arguments and invoke bot. If args is an empty list, sys.argv is used. @param args: command line arguments @type args: list of unicode """ # the option that's always selected when the bot wonders what to do with # a link. If it's None, the user is prompted (default behaviour). always = None alternatives = [] getAlternatives = True dnSkip = False generator = None primary = False main_only = False # For sorting the linked pages, case can be ignored minimum = 0 local_args = pywikibot.handle_args(args) generator_factory = pagegenerators.GeneratorFactory( positional_arg_name='page') for arg in local_args: if arg.startswith('-primary:'): primary = True getAlternatives = False alternatives.append(arg[9:]) elif arg == '-primary': primary = True elif arg.startswith('-always:'): always = arg[8:] elif arg.startswith('-pos:'): if arg[5] != ':': mysite = pywikibot.Site() page = pywikibot.Page(pywikibot.Link(arg[5:], mysite)) if page.exists(): alternatives.append(page.title()) else: if pywikibot.input_yn( u'Possibility %s does not actually exist. Use it ' 'anyway?' % page.title(), default=False, automatic_quit=False): alternatives.append(page.title()) else: alternatives.append(arg[5:]) elif arg == '-just': getAlternatives = False elif arg == '-dnskip': dnSkip = True elif arg == '-main': main_only = True elif arg.startswith('-min:'): minimum = int(arg[5:]) elif arg.startswith('-start'): try: generator = pagegenerators.CategorizedPageGenerator( pywikibot.Site().disambcategory(), start=arg[7:], namespaces=[0]) except pywikibot.NoPage: pywikibot.output( 'Disambiguation category for your wiki is not known.') raise else: generator_factory.handleArg(arg) site = pywikibot.Site() generator = generator_factory.getCombinedGenerator(generator) if not generator: pywikibot.bot.suggest_help(missing_generator=True) return False site.login() bot = DisambiguationRobot(always, alternatives, getAlternatives, dnSkip, generator, primary, main_only, minimum=minimum) bot.run()
def __init__(self, generator, templates, **kwargs): """ Constructor. @param generator: the pages to work on @type generator: iterable @param templates: a dictionary which maps old template names to their replacements. If remove or subst is True, it maps the names of the templates that should be removed/resolved to None. @type templates: dict """ self.availableOptions.update({ 'subst': False, 'remove': False, 'summary': None, 'addedCat': None, }) Bot.__init__(self, generator=generator, **kwargs) self.templates = templates # get edit summary message if it's empty if not self.getOption('summary'): comma = self.site.mediawiki_message('comma-separator') params = {'list': comma.join(self.templates.keys()), 'num': len(self.templates)} site = self.site if self.getOption('remove'): self.options['summary'] = i18n.twntranslate( site, 'template-removing', params) elif self.getOption('subst'): self.options['summary'] = i18n.twntranslate( site, 'template-substituting', params) else: self.options['summary'] = i18n.twntranslate( site, 'template-changing', params) # regular expression to find the original template. # {{vfd}} does the same thing as {{Vfd}}, so both will be found. # The old syntax, {{msg:vfd}}, will also be found. # The group 'parameters' will either match the parameters, or an # empty string if there are none. replacements = [] exceptions = {} namespace = self.site.namespaces[10] for old, new in self.templates.items(): if namespace.case == 'first-letter': pattern = '[' + \ re.escape(old[0].upper()) + \ re.escape(old[0].lower()) + \ ']' + re.escape(old[1:]) else: pattern = re.escape(old) pattern = re.sub(r'_|\\ ', r'[_ ]', pattern) templateRegex = re.compile(r'\{\{ *(' + ':|'.join(namespace) + r':|[mM][sS][gG]:)?' + pattern + r'(?P<parameters>\s*\|.+?|) *}}', re.DOTALL) if self.getOption('subst') and self.getOption('remove'): replacements.append((templateRegex, r'{{subst:%s\g<parameters>}}' % new)) exceptions['inside-tags'] = ['ref', 'gallery'] elif self.getOption('subst'): replacements.append((templateRegex, r'{{subst:%s\g<parameters>}}' % old)) exceptions['inside-tags'] = ['ref', 'gallery'] elif self.getOption('remove'): replacements.append((templateRegex, '')) else: template = pywikibot.Page(self.site, new, ns=10) if not template.exists(): pywikibot.warning(u'Template "%s" does not exist.' % new) if not pywikibot.input_yn('Do you want to proceed anyway?', default=False, automatic_quit=False): continue replacements.append((templateRegex, r'{{%s\g<parameters>}}' % new)) super(TemplateRobot, self).__init__( generator, replacements, exceptions, always=self.getOption('always'), addedCat=self.getOption('addedCat'), summary=self.getOption('summary'))
def treat(self, page): """It loads the given page, does some changes, and saves it.""" choice = False try: # page: title, date, username, comment, loginfo, rcid, token username = page['user'] # when the feed isnt from the API, it used to contain # '(not yet written)' or '(page does not exist)' when it was # a redlink rcid = page['rcid'] title = page['title'] if not rcid: raise Exception('rcid not present') # check whether we have wrapped around to higher rcids # which indicates a new RC feed is being processed if rcid > self.last_rcid: # refresh the whitelist self.load_whitelist() self.repeat_start_ts = time.time() if pywikibot.config.verbose_output or self.getOption('ask'): pywikibot.output(u'User %s has created or modified page %s' % (username, title)) if self.getOption('autopatroluserns') and (page['ns'] == 2 or page['ns'] == 3): # simple rule to whitelist any user editing their own userspace if title.partition(':')[2].split('/')[0].startswith(username): if pywikibot.config.verbose_output: pywikibot.output(u'%s is whitelisted to modify %s' % (username, title)) choice = True if not choice and username in self.whitelist: if self.in_list(self.whitelist[username], title): if pywikibot.config.verbose_output: pywikibot.output(u'%s is whitelisted to modify %s' % (username, title)) choice = True if self.getOption('ask'): choice = pywikibot.input_yn( 'Do you want to mark page as patrolled?') # Patrol the page if choice: # list() iterates over patrol() which returns a generator list(self.site.patrol(rcid)) self.patrol_counter = self.patrol_counter + 1 pywikibot.output(u'Patrolled %s (rcid %d) by user %s' % (title, rcid, username)) else: if pywikibot.config.verbose_output: pywikibot.output(u'Skipped') if rcid > self.highest_rcid: self.highest_rcid = rcid self.last_rcid = rcid self.rc_item_counter = self.rc_item_counter + 1 except pywikibot.NoPage: pywikibot.output(u'Page %s does not exist; skipping.' % title(asLink=True)) except pywikibot.IsRedirectPage: pywikibot.output(u'Page %s is a redirect; skipping.' % title(asLink=True))
def run(self): """Run the bot.""" commons = self.commons comment = self.summary for page in self.generator: self.current_page = page try: localImagePage = pywikibot.FilePage(self.site, page.title()) if localImagePage.fileIsShared(): pywikibot.output(u'File is already on Commons.') continue sha1 = localImagePage.latest_file_info.sha1 filenameOnCommons = self.findFilenameOnCommons(localImagePage) if not filenameOnCommons: pywikibot.output(u'NowCommons template not found.') continue commonsImagePage = pywikibot.FilePage( commons, 'Image:%s' % filenameOnCommons) if (localImagePage.title(withNamespace=False) != commonsImagePage.title(withNamespace=False)): usingPages = list(localImagePage.usingPages()) if usingPages and usingPages != [localImagePage]: pywikibot.output( color_format( '"{lightred}{0}{default}" is still used in {1} pages.', localImagePage.title(withNamespace=False), len(usingPages))) if self.getOption('replace') is True: pywikibot.output( color_format( 'Replacing "{lightred}{0}{default}" by ' '"{lightgreen}{1}{default}\".', localImagePage.title(withNamespace=False), commonsImagePage.title( withNamespace=False))) bot = ImageBot( pg.FileLinksGenerator(localImagePage), localImagePage.title(withNamespace=False), commonsImagePage.title(withNamespace=False), '', self.getOption('replacealways'), self.getOption('replaceloose')) bot.run() # If the image is used with the urlname the # previous function won't work is_used = bool( list( pywikibot.FilePage( self.site, page.title()).usingPages(total=1))) if is_used and self.getOption('replaceloose'): bot = ImageBot( pg.FileLinksGenerator(localImagePage), localImagePage.title(withNamespace=False, asUrl=True), commonsImagePage.title( withNamespace=False), '', self.getOption('replacealways'), self.getOption('replaceloose')) bot.run() # refresh because we want the updated list usingPages = len( list( pywikibot.FilePage( self.site, page.title()).usingPages())) else: pywikibot.output(u'Please change them manually.') continue else: pywikibot.output( color_format( 'No page is using "{lightgreen}{0}{default}" ' 'anymore.', localImagePage.title(withNamespace=False))) commonsText = commonsImagePage.get() if self.getOption('replaceonly') is False: if sha1 == commonsImagePage.latest_file_info.sha1: pywikibot.output( u'The image is identical to the one on Commons.') if len(localImagePage.getFileVersionHistory()) > 1: pywikibot.output( 'This image has a version history. Please ' 'delete it manually after making sure that the ' 'old versions are not worth keeping.') continue if self.getOption('always') is False: format_str = color_format( '\n\n>>>> Description on {lightpurple}%s' '{default} <<<<\n') pywikibot.output(format_str % page.title()) pywikibot.output(localImagePage.get()) pywikibot.output(format_str % commonsImagePage.title()) pywikibot.output(commonsText) if pywikibot.input_yn( u'Does the description on Commons contain ' 'all required source and license\n' 'information?', default=False, automatic_quit=False): localImagePage.delete( '%s [[:commons:Image:%s]]' % (comment, filenameOnCommons), prompt=False) else: localImagePage.delete( comment + ' [[:commons:Image:%s]]' % filenameOnCommons, prompt=False) else: pywikibot.output( u'The image is not identical to the one on Commons.' ) except (pywikibot.NoPage, pywikibot.IsRedirectPage) as e: pywikibot.output(u'%s' % e[0]) continue else: self._treat_counter += 1 if not self._treat_counter: pywikibot.output('No transcluded files found for %s.' % self.ncTemplates()[0]) self.exit()
def add_template(self, source, dest, task, fromsite): """Place or remove the Link_GA/FA template on/from a page.""" def compile_link(site, templates): """Compile one link template list.""" findtemplate = '(%s)' % '|'.join(templates) return re.compile( r"\{\{%s\|%s\}\}" % (findtemplate.replace(u' ', u'[ _]'), site.code), re.IGNORECASE) tosite = dest.site add_tl, remove_tl = self.getTemplateList(tosite.code, task) re_link_add = compile_link(fromsite, add_tl) re_link_remove = compile_link(fromsite, remove_tl) text = dest.text m1 = add_tl and re_link_add.search(text) m2 = remove_tl and re_link_remove.search(text) changed = False interactive = self.getOption('interactive') if add_tl: if m1: pywikibot.output(u"(already added)") else: # insert just before interwiki if (not interactive or pywikibot.input_yn(u'Connecting %s -> %s. Proceed?' % (source.title(), dest.title()), default=False, automatic_quit=False)): if self.getOption('side'): # Placing {{Link FA|xx}} right next to # corresponding interwiki text = (text[:m1.end()] + u" {{%s|%s}}" % (add_tl[0], fromsite.code) + text[m1.end():]) else: # Moving {{Link FA|xx}} to top of interwikis iw = textlib.getLanguageLinks(text, tosite) text = textlib.removeLanguageLinks(text, tosite) text += u"%s{{%s|%s}}%s" % (config.LS, add_tl[0], fromsite.code, config.LS) text = textlib.replaceLanguageLinks(text, iw, tosite) changed = True if remove_tl: if m2: if (changed or # Don't force the user to say "Y" twice not interactive or pywikibot.input_yn(u'Connecting %s -> %s. Proceed?' % (source.title(), dest.title()), default=False, automatic_quit=False)): text = re.sub(re_link_remove, '', text) changed = True elif task == 'former': pywikibot.output(u"(already removed)") if changed: comment = i18n.twtranslate(tosite, 'featured-' + task, {'page': source}) try: dest.put(text, comment) self._save_counter += 1 except pywikibot.LockedPage: pywikibot.output(u'Page %s is locked!' % dest.title()) except pywikibot.PageNotSaved: pywikibot.output(u"Page not saved")
def process_filename(self, file_url=None): """Return base filename portion of file_url.""" if not file_url: file_url = self.url pywikibot.warning("file_url is not given. " "Set to self.url by default.") always = self.getOption('always') # Isolate the pure name filename = file_url # Filename may be either a URL or a local file path if "://" in filename: # extract the path portion of the URL filename = urlparse(filename).path filename = os.path.basename(filename) if self.useFilename: filename = self.useFilename if not self.keepFilename: pywikibot.output( u"The filename on the target wiki will default to: %s" % filename) assert not always newfn = pywikibot.input( u'Enter a better name, or press enter to accept:') if newfn != "": filename = newfn # FIXME: these 2 belong somewhere else, presumably in family # forbidden characters are handled by pywikibot/page.py forbidden = ':*?/\\' # to be extended try: allowed_formats = self.targetSite.siteinfo.get( 'fileextensions', get_default=False) except KeyError: allowed_formats = [] else: allowed_formats = [item['ext'] for item in allowed_formats] # ask until it's valid first_check = True while True: if not first_check: if always: filename = None else: filename = pywikibot.input('Enter a better name, or press ' 'enter to skip the file:') if not filename: return None first_check = False ext = os.path.splitext(filename)[1].lower().strip('.') # are any chars in forbidden also in filename? invalid = set(forbidden) & set(filename) if invalid: c = "".join(invalid) pywikibot.output( 'Invalid character(s): %s. Please try again' % c) continue if allowed_formats and ext not in allowed_formats: if always: pywikibot.output('File format is not one of ' '[{0}]'.format(' '.join(allowed_formats))) continue elif not pywikibot.input_yn( u"File format is not one of [%s], but %s. Continue?" % (u' '.join(allowed_formats), ext), default=False, automatic_quit=False): continue potential_file_page = pywikibot.FilePage(self.targetSite, filename) if potential_file_page.exists(): overwrite = self._handle_warning('exists') if overwrite is False: pywikibot.output( 'File exists and you asked to abort. Skipping.') return None if potential_file_page.canBeEdited(): if overwrite is None: overwrite = not pywikibot.input_yn( "File with name %s already exists. " "Would you like to change the name? " "(Otherwise file will be overwritten.)" % filename, default=True, automatic_quit=False) if not overwrite: continue else: break else: pywikibot.output(u"File with name %s already exists and " "cannot be overwritten." % filename) continue else: try: if potential_file_page.fileIsShared(): pywikibot.output( 'File with name %s already exists in shared ' 'repository and cannot be overwritten.' % filename) continue else: break except pywikibot.NoPage: break # A proper description for the submission. # Empty descriptions are not accepted. if self.description: pywikibot.output('The suggested description is:\n%s' % self.description) while not self.description or self.verifyDescription: if not self.description: pywikibot.output(color_format( '{lightred}It is not possible to upload a file ' 'without a description.{default}')) assert not always # if no description, ask if user want to add one or quit, # and loop until one is filled. # if self.verifyDescription, ask if user want to change it # or continue. if self.description: question = 'Do you want to change this description?' else: question = 'No description was given. Add one?' if pywikibot.input_yn(question, default=not self.description, automatic_quit=self.description): from pywikibot import editor as editarticle editor = editarticle.TextEditor() try: newDescription = editor.edit(self.description) except ImportError: raise except Exception as e: pywikibot.error(e) continue # if user saved / didn't press Cancel if newDescription: self.description = newDescription elif not self.description: raise QuitKeyboardInterrupt self.verifyDescription = False return filename
def __init__(self, generator, templates: dict, **kwargs) -> None: """ Initializer. :param generator: the pages to work on :type generator: iterable :param templates: a dictionary which maps old template names to their replacements. If remove or subst is True, it maps the names of the templates that should be removed/resolved to None. """ SingleSiteBot.__init__(self, **kwargs) self.templates = templates # get edit summary message if it's empty if not self.opt.summary: comma = self.site.mediawiki_message('comma-separator') params = {'list': comma.join(self.templates.keys()), 'num': len(self.templates)} if self.opt.remove: tw_key = 'template-removing' elif self.opt.subst: tw_key = 'template-substituting' else: tw_key = 'template-changing' self.opt.summary = i18n.twtranslate(self.site, tw_key, params) replacements = [] exceptions = {} builder = textlib.MultiTemplateMatchBuilder(self.site) for old, new in self.templates.items(): template_regex = builder.pattern(old) if self.opt.subst and self.opt.remove: replacements.append((template_regex, r'{{subst:%s\g<parameters>}}' % new)) exceptions['inside-tags'] = ['ref', 'gallery', 'poem', 'pagelist', ] elif self.opt.subst: replacements.append( (template_regex, r'{{%s:%s\g<parameters>}}' % (self.opt.subst, old))) exceptions['inside-tags'] = ['ref', 'gallery', 'poem', 'pagelist', ] elif self.opt.remove: separate_line_regex = re.compile( r'^[*#:]* *{} *\n'.format(template_regex.pattern), re.DOTALL | re.MULTILINE) replacements.append((separate_line_regex, '')) spaced_regex = re.compile( r' +{} +'.format(template_regex.pattern), re.DOTALL) replacements.append((spaced_regex, ' ')) replacements.append((template_regex, '')) else: template = pywikibot.Page(self.site, new, ns=10) if not template.exists(): pywikibot.warning('Template "{}" does not exist.' .format(new)) if not pywikibot.input_yn('Do you want to proceed anyway?', default=False, automatic_quit=False): continue replacements.append((template_regex, r'{{%s\g<parameters>}}' % new)) super().__init__( generator, replacements, exceptions, always=self.opt.always, addcat=self.opt.addcat, summary=self.opt.summary)
def get_site_and_lang(default_family='wikipedia', default_lang='en', default_username=None): """ Ask the user for the family, language and username. @param default_family: The default family which should be chosen. @type default_family: None or str @param default_lang: The default language which should be chosen, if the family supports this language. @type default_lang: None or str @param default_username: The default username which should be chosen. @type default_username: None or str @return: The family, language and username @rtype: tuple of three str """ known_families = sorted(pywikibot.config2.family_files.keys()) if default_family not in known_families: default_family = None fam = listchoice(known_families, u"Select family of sites we are working on, " u"just enter the number or name", default=default_family) fam = pywikibot.family.Family.load(fam) if hasattr(fam, "langs"): if hasattr(fam, "languages_by_size"): key = {lang: i for i, lang in enumerate(fam.languages_by_size)}.get by_size = sorted(set(fam.langs.keys()).intersection( fam.languages_by_size), key=key) else: by_size = [] known_langs = by_size + sorted( set(fam.langs.keys()).difference(by_size)) else: known_langs = [] if len(known_langs) == 0: pywikibot.output('There were no known languages found in {0}.'.format( fam.name)) default_lang = None elif len(known_langs) == 1: pywikibot.output('The only known language: {0}'.format(known_langs[0])) default_lang = known_langs[0] else: pywikibot.output("This is the list of known languages:") pywikibot.output(u", ".join(known_langs)) if default_lang not in known_langs: if default_lang != 'en' and 'en' in known_langs: default_lang = 'en' else: default_lang = None message = "The language code of the site we're working on" if default_lang: message += " (default: '{0}')".format(default_lang) message += ":" mylang = None while not mylang: mylang = pywikibot.input(message) or default_lang if known_langs and mylang and mylang not in known_langs: if not pywikibot.input_yn("The language code {0} is not in the " "list of known languages. Do you want " "to continue?".format(mylang), default=False, automatic_quit=False): mylang = None username = None message = u"Username on {0}:{1}".format(mylang, fam.name) if default_username: message += " (default: '{0}')".format(default_username) message += ":" while not username: username = pywikibot.input(message) or default_username if not username: pywikibot.error('The username may not be empty.') if sys.version_info == 2: username = username.decode(console_encoding) # Escape ''s username = username.replace("'", "\\'") return fam.name, mylang, username