def weblinksIn(text, withoutBracketed=False, onlyBracketed=False): text = textlib.removeDisabledParts(text) # MediaWiki parses templates before parsing external links. Thus, there # might be a | or a } directly after a URL which does not belong to # the URL itself. # First, remove the curly braces of inner templates: nestedTemplateR = re.compile(r'{{([^}]*?){{(.*?)}}(.*?)}}') while nestedTemplateR.search(text): text = nestedTemplateR.sub(r'{{\1 \2 \3}}', text) # Then blow up the templates with spaces so that the | and }} will not # be regarded as part of the link:. templateWithParamsR = re.compile(r'{{([^}]*?[^ ])\|([^ ][^}]*?)}}', re.DOTALL) while templateWithParamsR.search(text): text = templateWithParamsR.sub(r'{{ \1 | \2 }}', text) # Add <blank> at the end of a template # URL as last param of multiline template would not be correct text = text.replace('}}', ' }}') # Remove HTML comments in URLs as well as URLs in HTML comments. # Also remove text inside nowiki links etc. text = textlib.removeDisabledParts(text) linkR = textlib.compileLinkR(withoutBracketed, onlyBracketed) for m in linkR.finditer(text): if m.group('url'): yield m.group('url') else: yield m.group('urlb')
def weblinksIn(text, withoutBracketed=False, onlyBracketed=False): text = textlib.removeDisabledParts(text) # MediaWiki parses templates before parsing external links. Thus, there # might be a | or a } directly after a URL which does not belong to # the URL itself. # First, remove the curly braces of inner templates: nestedTemplateR = re.compile(r'{{([^}]*?){{(.*?)}}(.*?)}}') while nestedTemplateR.search(text): text = nestedTemplateR.sub(r'{{\1 \2 \3}}', text) # Then blow up the templates with spaces so that the | and }} will not # be regarded as part of the link:. templateWithParamsR = re.compile(r'{{([^}]*?[^ ])\|([^ ][^}]*?)}}', re.DOTALL) while templateWithParamsR.search(text): text = templateWithParamsR.sub(r'{{ \1 | \2 }}', text) # Add <blank> at the end of a template # URL as last param of multiline template would not be correct text = text.replace('}}', ' }}') # Remove HTML comments in URLs as well as URLs in HTML comments. # Also remove text inside nowiki links etc. text = textlib.removeDisabledParts(text) linkR = textlib.compileLinkR(withoutBracketed, onlyBracketed) for m in linkR.finditer(text): if m.group('url'): yield m.group('url') else: yield m.group('urlb')
def loadTypos(self): pywikibot.output('Loading typo rules') self.typoRules = [] if self.typos_page_name is None: self.typos_page_name = 'Wikipedie:WPCleaner/Typo' typos_page = pywikibot.Page(self.site, self.typos_page_name) if not typos_page.exists(): # todo: feedback return text = textlib.removeDisabledParts( typos_page.text, include=['nowiki'], site=self.site) load_all = self.load_all is True for template, fielddict in textlib.extract_templates_and_params( text, remove_disabled_parts=False, strip=False): if template.lower() == 'typo': try: rule = TypoRule.newFromParameters(fielddict, self.site) except IncompleteTypoRuleException as exc: pywikibot.warning(exc.message) # pwb.exception? except InvalidExpressionException as exc: if 'fixed-width' not in exc.message: pywikibot.warning('Invalid %s %s: %s' % ( exc.aspect, fielddict['1'], exc.message)) else: rule.id = self.top_id # fixme: cvar or ivar? self.top_id += 1 if load_all or not rule.needs_decision(): self.typoRules.append(rule) pywikibot.output('%d typo rules loaded' % len(self.typoRules)) return self.typoRules
def find_discussion(self, category: pywikibot.Category) -> 'CfdPage': """ Return the relevant discussion. @param category: The category being discussed """ if self.section(): return self text = removeDisabledParts(self.text, tags=EXCEPTIONS, site=self.site) wikicode = mwparserfromhell.parse(text, skip_style_tags=True) for section in wikicode.get_sections(levels=[4]): heading = section.filter_headings()[0] section_title = str(heading.title).strip() discussion = self.__class__( self.site, '{}#{}'.format(self.title(), section_title)) if category.title() == section_title: return discussion # Split approximately into close, nom, and others. parts = str(section).split('(UTC)') if len(parts) < 3: continue # Parse the nom for category links. nom = mwparserfromhell.parse(parts[1], skip_style_tags=True) for node in nom.ifilter(): page = self._cat_from_node(node) if page and category == page: return discussion return self
def find_discussion(self, category): """Find the section with the relevant discussion.""" if self.section(): return self.title(as_link=True) text = removeDisabledParts(self.text, site=self.site) wikicode = mwparserfromhell.parse(text, skip_style_tags=True) for section in wikicode.get_sections(levels=[4]): heading = section.filter(forcetype=Heading)[0] section_title = str(heading.title).strip() discussion = '[[{}#{}]]'.format(self.title(), section_title) if category.title() == section_title: return discussion # Split approximately into close, nom, and others parts = str(section).split('(UTC)') if len(parts) < 3: continue # Parse the nom for links for wikilink in pywikibot.link_regex.finditer(parts[1]): title = wikilink.group('title').strip().split('#')[0] if not title: continue title = pywikibot.Page(self.site, title).title() if category.title() == title: return discussion return self.title(as_link=True)
def __iter__(self): from pywikibot import xmlreader dump = xmlreader.XmlDump(self.xmlFilename) for entry in dump.parse(): text = textlib.removeDisabledParts(entry.text) if self.refR.search(text) and not self.referencesR.search(text): yield pywikibot.Page(pywikibot.Site(), entry.title)
def replace_gallery_files( self, wikicode: mwparserfromhell.wikicode.Wikicode) -> None: """ Replace files in <gallery>. :param wikicode: Parsed wikitext """ for tag in wikicode.ifilter_tags(): if tag.tag.lower() != "gallery": continue lines = str(tag.contents).splitlines() for i, line in enumerate(lines): title, sep, caption = removeDisabledParts(line).partition("|") if not title: continue try: current_icon = BSiconPage(self.current_page.site, title) current_icon.title() except (pywikibot.exceptions.Error, ValueError): continue new_icon = self.opt.bsicons_map.get(current_icon, None) if new_icon: lines[i] = f"{new_icon.title()}{sep}{caption}" self.current_page.replacements.add( Replacement(current_icon, new_icon)) if self.current_page.replacements: tag.contents = "\n".join(lines) + "\n"
def lacksReferences(self, text): """Check whether or not the page is lacking a references tag.""" oldTextCleaned = textlib.removeDisabledParts(text) if self.referencesR.search(oldTextCleaned) or \ self.referencesTagR.search(oldTextCleaned): if self.getOption('verbose'): pywikibot.output( u'No changes necessary: references tag found.') return False elif self.referencesTemplates: templateR = u'{{(' + u'|'.join(self.referencesTemplates) + ')' if re.search(templateR, oldTextCleaned, re.IGNORECASE | re.UNICODE): if self.getOption('verbose'): pywikibot.output( u'No changes necessary: references template found.') return False if not self.refR.search(oldTextCleaned): if self.getOption('verbose'): pywikibot.output(u'No changes necessary: no ref tags found.') return False else: if self.getOption('verbose'): pywikibot.output(u'Found ref without references.') return True
def treat_page(self): """Process one page.""" self.check_disabled() target = self.current_page.getCategoryRedirectTarget() seen = {self.current_page, target} while target.isCategoryRedirect(): target = target.getCategoryRedirectTarget() if target in seen: pywikibot.error( 'Skipping {} due to possible circular redirect at {}.'. format(self.current_page, target)) return seen.add(target) wikicode = mwparserfromhell.parse(self.current_page.text, skip_style_tags=True) for tpl in wikicode.ifilter_templates(): try: template = pywikibot.Page( self.site, removeDisabledParts(str(tpl.name), site=self.site), ns=10, ) template.title() except pywikibot.InvalidTitle: continue if template in self.templates: tpl.add('1', target.title(with_ns=False)) break self.put_current(str(wikicode), summary=self.getOption('summary'))
def lacksReferences(self, text) -> bool: """Check whether or not the page is lacking a references tag.""" oldTextCleaned = textlib.removeDisabledParts(text) if self.referencesR.search(oldTextCleaned) \ or self.referencesTagR.search(oldTextCleaned): if self.opt.verbose: pywikibot.output('No changes necessary: references tag found.') return False if self.referencesTemplates: templateR = '{{(' + '|'.join(self.referencesTemplates) + ')' if re.search(templateR, oldTextCleaned, re.IGNORECASE): if self.opt.verbose: pywikibot.output( 'No changes necessary: references template found.') return False if not self.refR.search(oldTextCleaned): if self.opt.verbose: pywikibot.output('No changes necessary: no ref tags found.') return False if self.opt.verbose: pywikibot.output('Found ref without references.') return True
def get_action(self, category: pywikibot.Category) -> str: """ Return the discussion action. @param category: The category being discussed """ if not self.section(): return '' text = removeDisabledParts(self.text, tags=EXCEPTIONS, site=self.site) wikicode = mwparserfromhell.parse(text, skip_style_tags=True) for section in wikicode.get_sections(levels=[4]): heading = section.filter_headings()[0] if str(heading.title).strip() == self.section(): break else: section = None # Trick pylint. return '' # Parse the discussion for category links and action. for line in str(section).splitlines(): found = False line_wc = mwparserfromhell.parse(line, skip_style_tags=True) for node in line_wc.ifilter(): page = self._cat_from_node(node) if page and category == page: found = True break matches = re.findall(r"'''Propose (.+?)'''", line) if found and matches: return matches[0] return ''
def treat(self, page): # get all linkedPages # check for disambigs linksR = re.compile('\[\[(?P<short>[^\]]*)\]\] *\|\| *\[\[(?P<long>[^\]]*)\]\]') res = [] counter = 0 if self.opt.test: pywikibot.output('Treat(%s)' % page.title(as_link=True)) for p in linksR.finditer(textlib.removeDisabledParts(page.text)): counter += 1 longn = p.group('long') shortn = p.group('short') if self.opt.testlinks: pywikibot.output('[%s][#%i] S:%s L:%s' % ( datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), counter, shortn, longn)) rpl = pywikibot.Page(pywikibot.Site(), longn) rplcount = len(list(rpl.getReferences(namespaces=0))) if self.opt.testlinks: pywikibot.output('L:%s #%i In %s checking:%s - referenced by %i' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), counter, page.title(as_link=True), rpl.title(as_link=True), rplcount)) rps = pywikibot.Page(pywikibot.Site(), shortn) rpscount = len(list(rps.getReferences(namespaces=0))) if self.opt.testlinks: pywikibot.output('S:%s #%i In %s checking:%s - referenced by %i' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), counter, page.title(as_link=True), rps.title(as_link=True), rpscount)) res.append({"long": longn, "refl": rplcount, "short": shortn, "refs": rpscount}) print(res) return res
def __iter__(self): import xmlreader dump = xmlreader.XmlDump(self.xmlFilename) for entry in dump.parse(): text = textlib.removeDisabledParts(entry.text) if self.refR.search(text) and not self.referencesR.search(text): yield pywikibot.Page(pywikibot.Site(), entry.title)
def getWordCount(self, text): text = textlib.removeDisabledParts(text) text = textlib.removeHTMLParts(text) text = textlib.removeLanguageLinks(text) text = textlib.removeCategoryLinks(text) word_list = re.findall(r"[\w']+", text) return len(word_list)
def checkUserPage(self,page): text = page.text if self.catsPresent(textlib.removeDisabledParts(text)): text = textlib.replaceExcept(text, ur'\[\[kategoria', '[[:Kategoria', ['comment','pre','nowiki'], caseInsensitive=True) #text = re.sub('\[\[kategoria', '[[:Kategoria', text, flags=re.I) pywikibot.output(u'Kategorie usunięte') page.text = text #page.save(summary=u'Bot usuwa stronę użytkownika z kategorii', apply_cosmetic_changes=False) else: pywikibot.output(u'Strona użytkownika OK') return
def get_pages_with_descriptions(self, text): tags = {'category', 'comment', 'file', 'header', 'hyperlink', 'interwiki', 'nowiki', 'pre', 'ref', 'source', 'timeline', 'template'} text = textlib.removeDisabledParts(text, tags, site=self.site) data = {} for match in self.regex.finditer(text): title, desc = match.groups() page = pywikibot.Page(self.site, title) data[page] = self.parse_description(desc) return data
def check(text, languages): tags = ['comments', 'nowiki', 'pre', 'source'] text = textlib.removeDisabledParts(text, tags) interwikiR = re.compile(r'\[\[([a-zA-Z\-]+)\s?:([^\[\]\n]*)\]\]') for lang, pagetitle in interwikiR.findall(text): lang = lang.lower() # Check if it really is in fact an interwiki link to a known # language, or if it's e.g. a category tag or an internal link if lang in languages: return True return False
def treat(self, page): match = self.current_rule.find.search(page.text) if not match: return text = textlib.removeDisabledParts(page.text, TypoRule.exceptions, site=self.site) match = self.current_rule.find.search(text) if match: text = self.pattern.format(page.title(as_link=True), match.group(0)) pywikibot.stdout(text) self.data.append(text)
def processArticle(page): text = page.get() text = textlib.removeDisabledParts(text) # pywikibot.output(u'Working on "%s"' % title) global codeRegexp global templateRegexp result = re.findall(codeRegexp, text) template = re.findall(templateRegexp, text) if len(result) > 0 and len(template) == 0: msg = u"* [[%s]]: " % page.title() for res in result: msg += str(res) log(msg) pywikibot.output(msg)
def lacksReferences(self, text): """Check whether or not the page is lacking a references tag.""" oldTextCleaned = textlib.removeDisabledParts(text) if self.referencesR.search(oldTextCleaned) or \ self.referencesTagR.search(oldTextCleaned): return False elif self.referencesTemplates: templateR = u'{{(' + u'|'.join(self.referencesTemplates) + ')' if re.search(templateR, oldTextCleaned, re.IGNORECASE | re.UNICODE): return False if not self.refR.search(oldTextCleaned): return False else: return True
def treat_property_and_talk(self, prop, page): self.current_talk_page = page # todo: skip sandbox properties # todo: removeDisabledParts now? code = mwparserfromhell.parse(page.text, skip_style_tags=True) for template in code.ifilter_templates(): if not template.name.matches(self.template_metadata): continue params = OrderedDict() for param in template.params: params[str(param.name).strip()] = str(param.value).strip() break else: pywikibot.output('Template "{}" not found'.format( self.template_metadata)) return keys = set(self.func_dict.keys()) & set(params.keys()) # formatter URL must go before example if {'formatter URL', 'example'} <= keys: keys.remove('formatter URL') keys = ['formatter URL'] + list(keys) clear_params = [] for key in keys: param = textlib.removeDisabledParts(params[key]) if param == '-': continue if param != '': pywikibot.output('Found param "{}"'.format(key)) try: remove = self.func_dict[key](param) except pywikibot.data.api.APIError as exc: remove = False if remove: clear_params.append(key) if self.getOption('importonly'): return for par in clear_params: template.remove(par, keep_field=True) for par in set(params.keys()) & set(self.obsolete_params): template.remove(par) self.current_page = self.current_talk_page self.put_current(str(code), show_diff=True, summary='removing migrated/obsolete parameters')
def treat_property_and_talk(self, prop, page): self.current_talk_page = page # todo: skip sandbox properties # todo: removeDisabledParts now? code = mwparserfromhell.parse(page.text, skip_style_tags=True) for template in code.ifilter_templates(): if not template.name.matches(self.template_metadata): continue params = OrderedDict() for param in template.params: params[str(param.name).strip()] = str(param.value).strip() break else: pywikibot.output('Template "{}" not found'.format( self.template_metadata)) return keys = set(self.func_dict.keys()) & set(params.keys()) # formatter URL must go before example if {'formatter URL', 'example'} <= keys: keys.remove('formatter URL') keys = ['formatter URL'] + list(keys) clear_params = [] for key in keys: param = textlib.removeDisabledParts(params[key]) if param == '-': continue if param != '': pywikibot.output('Found param "{}"'.format(key)) try: remove = self.func_dict[key](param) except pywikibot.data.api.APIError as exc: remove = False if remove: clear_params.append(key) if self.getOption('importonly'): return for par in clear_params: template.remove(par, keep_field=True) for par in set(params.keys()) & set(self.obsolete_params): template.remove(par) self.current_page = self.current_talk_page self.put_current(str(code), show_diff=True, summary='removing migrated/obsolete parameters')
def parse_page(page): """Parse a CFD working page.""" text = removeDisabledParts(page.text, site=page.site) wikicode = mwparserfromhell.parse(text, skip_style_tags=True) for section in wikicode.get_sections(flat=True, include_lead=False): heading = section.filter(forcetype=Heading)[0] section_title = str(heading.title).lower() print(section_title) if 'move' in section_title: mode = 'move' edit_summary = 'Moving {old_cat} to {new_cats} per {cfd}' elif 'empty' in section_title: mode = 'empty' edit_summary = 'Removing {old_cat} per {cfd}' else: continue parse_section(section, page.site, mode, edit_summary)
def parse(self) -> None: """Parse the page.""" text = removeDisabledParts(self.text, tags=EXCEPTIONS, site=self.site) wikicode = mwparserfromhell.parse(text, skip_style_tags=True) for section in wikicode.get_sections(flat=True, include_lead=False): heading = section.filter_headings()[0] section_title = str(heading.title).lower() for mode in self.MODES: if mode in section_title: self.mode = mode break else: continue try: self._parse_section(str(section)) except (ValueError, pywikibot.Error): pywikibot.exception(tb=True) self._check_run()
def sectionList(self,page): sections = [] sectionR = re.compile(ur'(?im)^=+(?P<section>[^<]*?)(<ref.*?)?=+$') text = page.text # expand templates etext = page.expand_text() etext = textlib.removeDisabledParts(etext) #if self.getOption('test'): # pywikibot.output(etext) for s in sectionR.finditer(etext): if self.getOption('test'): pywikibot.output(u'>>>%s<<<' % s.group('section').strip()) sections.append(s.group('section').strip()) return(sections)
def parse_description(self, text): desc = textlib.removeDisabledParts(text, [ 'comment', 'file', 'nowiki', 'template', self.FORMATTING_REGEX, self.REF_REGEX ]) desc = LINK_REGEX.sub(self.handle_link, desc) desc = desc.replace(' ', ' ').strip() desc = re.sub(r' *\([^)]+\)$', '', desc) desc = desc.partition(';')[0] desc = re.sub(r'^.*\) [-–] +', '', desc) desc = re.sub(r'^\([^)]+\) +', '', desc) while ' ' * 2 in desc: desc = desc.replace(' ' * 2, ' ') if re.search('[^IVX]\.$', desc) or desc.endswith(tuple(',:')): desc = desc[:-1].rstrip() if desc.startswith(('a ', 'an ')): desc = desc.partition(' ')[2] return desc
def get_result(self) -> str: """Return the discussion result.""" if not self.section(): return '' text = removeDisabledParts(self.text, tags=EXCEPTIONS, site=self.site) wikicode = mwparserfromhell.parse(text, skip_style_tags=True) for section in wikicode.get_sections(levels=[4]): heading = section.filter_headings()[0] if str(heading.title).strip() == self.section(): break else: section = None # Trick pylint. return '' for line in str(section).splitlines(): matches = re.findall( r"''The result of the discussion was:''\s+'''(.+?)'''", line) if matches: return matches[0] return ''
def replace_file_links(self, text: str) -> str: """ Return text with file links replaced. :param text: Article text """ assert self.site_config is not None for match in self.site_config.file_regex.finditer( removeDisabledParts(text)): try: current_icon = BSiconPage(self.current_page.site, match.group("filename")) current_icon.title() except (pywikibot.exceptions.Error, ValueError): continue new_icon = self.opt.bsicons_map.get(current_icon, None) if new_icon: text = text.replace(match.group("filename"), new_icon.title(with_ns=False)) self.current_page.replacements.add( Replacement(current_icon, new_icon)) return text
def treat(self, page): #get all linkedPages # check for disambigs linkR = re.compile( ur'\[\[(?P<title>.*?)(#(?P<section>.*?))?(\|(?P<label>.*?))?\]\]') counter = 0 reqcounter = 0 checkedpages = [] for p in linkR.finditer(textlib.removeDisabledParts(page.text)): counter += 1 t = p.group('title') if t in checkedpages or t == '': continue try: rp = pywikibot.Page(pywikibot.Site(), t) if not rp.namespace() == 0: continue if self.getOption('testlinks'): pywikibot.output( u'%s #%i (%i) In %s checking:%s' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), counter, reqcounter, page.title(asLink=True), rp.title(asLink=True))) if not rp.exists(): reqcounter += 1 self.addResult(page.title(), rp.title()) checkedpages.append(t) except KeyboardInterrupt: pywikibot.output( 'PICKLING at %s' % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) with open('masti/reqlinks.dat', 'wb') as f: pickle.dump(self.results, f, protocol=config.pickle_protocol) return ('STOP') except: continue return (reqcounter)
def lacksReferences(self, text): """Check whether or not the page is lacking a references tag.""" oldTextCleaned = textlib.removeDisabledParts(text) if self.referencesR.search(oldTextCleaned) or \ self.referencesTagR.search(oldTextCleaned): if self.getOption('verbose'): pywikibot.output(u'No changes necessary: references tag found.') return False elif self.referencesTemplates: templateR = u'{{(' + u'|'.join(self.referencesTemplates) + ')' if re.search(templateR, oldTextCleaned, re.IGNORECASE | re.UNICODE): if self.getOption('verbose'): pywikibot.output( u'No changes necessary: references template found.') return False if not self.refR.search(oldTextCleaned): if self.getOption('verbose'): pywikibot.output(u'No changes necessary: no ref tags found.') return False else: if self.getOption('verbose'): pywikibot.output(u'Found ref without references.') return True
def replace_template_files( self, wikicode: mwparserfromhell.wikicode.Wikicode) -> None: """ Replace files in templates. :param wikicode: Parsed wikitext """ assert self.site_config is not None for tpl in wikicode.ifilter_templates(): try: template = Page( self.current_page.site, removeDisabledParts(str(tpl.name)), ns=self.current_page.site.namespaces.TEMPLATE, ) template.title() except (pywikibot.exceptions.Error, ValueError): continue if template in self.site_config.routemap_templates: self._replace_routemap_files(tpl) elif template in self.site_config.railway_track_templates: self._replace_rt_template_files(tpl) else: self._replace_bs_template_files(tpl, template)
def treat_page(self) -> None: """Process one page.""" self.check_disabled() try: errors = self.validate_svg() except (AssertionError, RequestException, RuntimeError): pywikibot.exception() return if errors: n_errors = len(errors) new_tpl = Template('Invalid SVG') new_tpl.add('1', n_errors) summary = 'W3C invalid SVG: {} error{}'.format( n_errors, 's' if n_errors > 1 else '') else: new_tpl = Template('Valid SVG') summary = 'W3C valid SVG' wikicode = mwparserfromhell.parse(self.current_page.text, skip_style_tags=True) for tpl in wikicode.ifilter_templates(): try: template = pywikibot.Page( self.site, removeDisabledParts(str(tpl.name), site=self.site).strip(), ns=10, ) template.title() except pywikibot.InvalidTitle: continue if template in self.templates: wikicode.replace(tpl, new_tpl) break else: wikicode.insert(0, '\n') wikicode.insert(0, new_tpl) self.put_current(str(wikicode), summary=summary, minor=not errors)
def treat(self, page): """ Returns page title if param 'text' not in page """ if self.getOption('wikipedia'): resultR = re.compile( ur'(?i)(?P<result>https?://(?P<lang>[^\.]*?)\.(?P<project>wikipedia)\.org/wiki/[^\s\|<\]\}]*)' ) else: resultR = re.compile( ur'(?i)(?P<result>https?://(?P<lang>[^\.]*?)\.(?P<project>wikipedia|wikisource|wiktionary|wikivoyage)\.org/wiki/[^\s\|<\]\}]*)' ) # allowed filtypes: svg, png, jpeg, tiff, gif, xcf imageR = re.compile(ur'(?i).*\.(svg|png|jpeg|jpg|tiff|tif|gif|xcf)$') source = textlib.removeDisabledParts(page.text) #return all found results resultslist = [] found = False for r in re.finditer(resultR, source): if self.getOption('test'): pywikibot.output(u'R:%s' % r.group('result')) img = imageR.search(r.group('result')) if not img: resultslist.append({ 'link': r.group('result'), 'lang': r.group('lang'), 'project': r.group('project') }) found = True if found: return ({'page': page.title(), 'links': resultslist}) else: return (None)
def find_and_replace(self, text, init): new_params = [] old_params = [] unknown_params = [] removed_params = [] changed = False for template, fielddict in textlib.extract_templates_and_params( text, remove_disabled_parts=False, strip=False): if self.normalize(template) not in (self.template, self.new_template): continue changed = self.normalize(template) != self.new_template start_match = re.search(r'\{\{\s*((%s)\s*:\s*)?%s\s*' % ( '|'.join(self.site.namespaces[10]), re.escape(template)), text) if not start_match: if not init: pywikibot.error("Couldn't find the template") return text, 0 start = start_match.start() if len(fielddict) > 0: end = text.index('|', start) else: end = text.index('}}', start) unnamed = {} for name, value in chain(fielddict.items(), IterUnnamed(unnamed)): end += len('|%s=%s' % (name, value)) name = name.strip() value = (value .replace('\n<!-- Zastaralé parametry -->', '') .replace('\n<!-- Neznámé parametry -->', '') .strip()) try: new_name = self.handle_param(name) except OldParamException: if textlib.removeDisabledParts(value, ['comments']).strip(): old_params.append( (name, value) ) except RemoveParamException: changed = True if textlib.removeDisabledParts(value, ['comments']).strip(): removed_params.append( (name, value) ) except UnknownParamException: if textlib.removeDisabledParts(value, ['comments']).strip(): unknown_params.append( (name, value) ) except AssertionError: pywikibot.error('Couldn\'t handle parameter "%s"' % name) return text, 0 except UnnamedParamException: unnamed[value] = '' else: new_params.append( (new_name, value) ) if new_name != name: changed = True end += len('}}') while text[start:end].count('{{') < text[start:end].count('}}'): end = text[:end].rindex('}}') + len('}}') if text[start:end].count('{{') > text[start:end].count('}}'): ballance = 1 end = start while ballance > 0: next_close = text.index('}}', end) ballance += text[end:next_close].count('{{') - 1 end = next_close + len('}}') if not text[start:end].endswith('}}'): # elif? end = text[:end].rindex('}}') + len('}}') if (end < start or not text[start:end].endswith('}}') or text[start:end].count('{{') != text[start:end].count('}}')): pywikibot.error("Couldn't parse the template") return text, 0 break else: pywikibot.error("Couldn't parse the template") return text, 0 if not changed: pywikibot.output('No parameters changed') return text, 0 while end < len(text) and text[end].isspace(): # todo: also before end += 1 lines = [] nested = 0 for line in text[start:end].splitlines(): if nested == 1 and re.match(' *\|', line): lines.append(line) nested += line.count('{{') - line.count('}}') space_before = '' if len(lines) > 0 and choice(lines).startswith(' '): space_before = ' ' self.handle_params(new_params, old_params, removed_params, unknown_params) self.deduplicate(new_params) new_params.sort(key=self.key_for_sort) new_template = '{{%s' % self.new_template if len(new_params) > 0: new_template += '\n' for param, value in new_params: new_template += '%s| %s = %s\n' % (space_before, param, value) if len(old_params) > 0: new_template += '<!-- Zastaralé parametry -->\n' for param, value in old_params: new_template += '%s| %s = %s\n' % (space_before, param, value) if len(unknown_params) > 0: new_template += '<!-- Neznámé parametry -->\n' for param, value in unknown_params: new_template += '%s| %s = %s\n' % (space_before, param, value) new_template += '}}\n' return text[:start] + new_template + text[end:], end
def run(self): """Run the Bot.""" try: deadLinks = codecs.open(listof404pages, 'r', 'latin_1').read() except IOError: raise NotImplementedError( '404-links.txt is required for reflinks.py\n' 'You need to download\n' 'http://www.twoevils.org/files/wikipedia/404-links.txt.gz\n' 'and to unzip it in the same directory') editedpages = 0 for page in self.generator: try: # Load the page's text from the wiki new_text = page.get() if not page.canBeEdited(): pywikibot.output(u"You can't edit page %s" % page.title(asLink=True)) continue except pywikibot.NoPage: pywikibot.output(u'Page %s not found' % page.title(asLink=True)) continue except pywikibot.IsRedirectPage: pywikibot.output(u'Page %s is a redirect' % page.title(asLink=True)) continue # for each link to change for match in linksInRef.finditer( textlib.removeDisabledParts(page.get())): link = match.group(u'url') # debugging purpose # print link if u'jstor.org' in link: # TODO: Clean URL blacklist continue ref = RefLink(link, match.group('name')) f = None try: f = comms.http.fetch( ref.url, use_fake_user_agent=self._use_fake_user_agent) # Try to get Content-Type from server contentType = f.response_headers.get('content-type') if contentType and not self.MIME.search(contentType): if ref.link.lower().endswith('.pdf') and \ not self.getOption('ignorepdf'): # If file has a PDF suffix self.getPDFTitle(ref, f) else: pywikibot.output(color_format( '{lightyellow}WARNING{default} : ' 'media : {0} ', ref.link)) if ref.title: if not re.match( u'(?i) *microsoft (word|excel|visio)', ref.title): ref.transform(ispdf=True) repl = ref.refTitle() else: pywikibot.output(color_format( '{lightyellow}WARNING{default} : ' 'PDF title blacklisted : {0} ', ref.title)) repl = ref.refLink() else: repl = ref.refLink() new_text = new_text.replace(match.group(), repl) continue # Get the real url where we end (http redirects !) redir = f.data.url if redir != ref.link and \ domain.findall(redir) == domain.findall(link): if soft404.search(redir) and \ not soft404.search(ref.link): pywikibot.output(color_format( '{lightyellow}WARNING{default} : ' 'Redirect 404 : {0} ', ref.link)) continue if dirIndex.match(redir) and \ not dirIndex.match(ref.link): pywikibot.output(color_format( u'{lightyellow}WARNING{default} : ' u'Redirect to root : {0} ', ref.link)) continue if f.status != requests.codes.ok: pywikibot.output(u'HTTP error (%s) for %s on %s' % (f.status, ref.url, page.title(asLink=True)), toStdout=True) # 410 Gone, indicates that the resource has been purposely # removed if f.status == 410 or \ (f.status == 404 and (u'\t%s\t' % ref.url in deadLinks)): repl = ref.refDead() new_text = new_text.replace(match.group(), repl) continue linkedpagetext = f.raw except UnicodeError: # example : http://www.adminet.com/jo/20010615¦/ECOC0100037D.html # in [[fr:Cyanure]] pywikibot.output(color_format( '{lightred}Bad link{default} : {0} in {1}', ref.url, page.title(asLink=True))) continue except (URLError, socket.error, IOError, httplib.error) as e: pywikibot.output(u'Can\'t retrieve page %s : %s' % (ref.url, e)) continue # remove <script>/<style>/comments/CDATA tags linkedpagetext = self.NON_HTML.sub(b'', linkedpagetext) meta_content = self.META_CONTENT.search(linkedpagetext) enc = [] s = None if contentType: # use charset from http header s = self.CHARSET.search(contentType) if meta_content: tag = meta_content.group() # Prefer the contentType from the HTTP header : if not contentType: contentType = tag if not s: # use charset from html s = self.CHARSET.search(str(tag)) if s: tmp = s.group('enc').strip("\"' ").lower() naked = re.sub(r'[ _\-]', '', tmp) # Convert to python correct encoding names if naked == "gb2312": enc.append("gbk") elif naked == "shiftjis": enc.append("shift jis 2004") enc.append("cp932") elif naked == "xeucjp": enc.append("euc-jp") else: enc.append(tmp) else: pywikibot.output(u'No charset found for %s' % ref.link) if not contentType: pywikibot.output(u'No content-type found for %s' % ref.link) continue elif not self.MIME.search(contentType): pywikibot.output(color_format( '{lightyellow}WARNING{default} : media : {0} ', ref.link)) repl = ref.refLink() new_text = new_text.replace(match.group(), repl) continue # Ugly hacks to try to survive when both server and page # return no encoding. # Uses most used encodings for each national suffix if u'.ru' in ref.link or u'.su' in ref.link: # see http://www.sci.aha.ru/ATL/ra13a.htm : no server # encoding, no page encoding enc = enc + ['koi8-r', 'windows-1251'] elif u'.jp' in ref.link: enc.append("shift jis 2004") enc.append("cp932") elif u'.kr' in ref.link: enc.append("euc-kr") enc.append("cp949") elif u'.zh' in ref.link: enc.append("gbk") if 'utf-8' not in enc: enc.append('utf-8') try: u = linkedpagetext.decode(enc[0]) # Bug T69410 except (UnicodeDecodeError, LookupError) as e: pywikibot.output(u'%s : Decoding error - %s' % (ref.link, e)) continue # Retrieves the first non empty string inside <title> tags for m in self.TITLE.finditer(u): t = m.group() if t: ref.title = t ref.transform() if ref.title: break if not ref.title: repl = ref.refLink() new_text = new_text.replace(match.group(), repl) pywikibot.output(u'%s : No title found...' % ref.link) continue # XXX Ugly hack if u'é' in ref.title: repl = ref.refLink() new_text = new_text.replace(match.group(), repl) pywikibot.output(u'%s : Hybrid encoding...' % ref.link) continue if self.titleBlackList.match(ref.title): repl = ref.refLink() new_text = new_text.replace(match.group(), repl) pywikibot.output(color_format( '{lightred}WARNING{default} {0} : ' 'Blacklisted title ({1})', ref.link, ref.title)) continue # Truncate long titles. 175 is arbitrary if len(ref.title) > 175: ref.title = ref.title[:175] + "..." repl = ref.refTitle() new_text = new_text.replace(match.group(), repl) # Add <references/> when needed, but ignore templates ! if page.namespace != 10: if self.norefbot.lacksReferences(new_text): new_text = self.norefbot.addReferences(new_text) new_text = self.deduplicator.process(new_text) old_text = page.text self.userPut(page, old_text, new_text, summary=self.msg, ignore_save_related_errors=True, ignore_server_errors=True) if new_text == old_text: continue else: editedpages += 1 if self.getOption('limit') and editedpages >= self.getOption('limit'): pywikibot.output('Edited %s pages, stopping.' % self.getOption('limit')) return if self.site_stop_page and editedpages % 20 == 0: self.stop_page = pywikibot.Page(self.site, self.site_stop_page) if self.stop_page.exists(): pywikibot.output(color_format( '{lightgreen}Checking stop page...{default}')) actual_rev = self.stop_page.latest_revision_id if actual_rev != self.stop_page_rev_id: pywikibot.output( '%s has been edited : Someone wants us to stop.' % self.stop_page.title(asLink=True)) return
def add_text(page, addText, summary=None, regexSkip=None, regexSkipUrl=None, always=False, up=False, putText=True, oldTextGiven=None, reorderEnabled=True, create=False): """ Add text to a page. @rtype: tuple of (text, newtext, always) """ site = page.site if not summary: summary = i18n.twtranslate(site, 'add_text-adding', {'adding': addText[:200]}) # When a page is tagged as "really well written" it has a star in the # interwiki links. This is a list of all the templates used (in regex # format) to make the stars appear. errorCount = 0 if putText: pywikibot.output(u'Loading %s...' % page.title()) if oldTextGiven is None: try: text = page.get() except pywikibot.NoPage: if create: pywikibot.output(u"%s doesn't exist, creating it!" % page.title()) text = u'' else: pywikibot.output(u"%s doesn't exist, skip!" % page.title()) return (False, False, always) except pywikibot.IsRedirectPage: pywikibot.output(u"%s is a redirect, skip!" % page.title()) return (False, False, always) else: text = oldTextGiven # Understand if the bot has to skip the page or not # In this way you can use both -except and -excepturl if regexSkipUrl is not None: url = page.full_url() result = re.findall(regexSkipUrl, site.getUrl(url)) if result != []: pywikibot.output('Exception! regex (or word) used with -exceptUrl ' 'is in the page. Skip!\n' 'Match was: %s' % result) return (False, False, always) if regexSkip is not None: result = re.findall(regexSkip, text) if result != []: pywikibot.output('Exception! regex (or word) used with -except ' 'is in the page. Skip!\n' 'Match was: %s' % result) return (False, False, always) # If not up, text put below if not up: newtext = text # Translating the \\n into binary \n addText = addText.replace('\\n', config.line_separator) if (reorderEnabled): # Getting the categories categoriesInside = textlib.getCategoryLinks(newtext, site) # Deleting the categories newtext = textlib.removeCategoryLinks(newtext, site) # Getting the interwiki interwikiInside = textlib.getLanguageLinks(newtext, site) # Removing the interwiki newtext = textlib.removeLanguageLinks(newtext, site) # Adding the text newtext += u"%s%s" % (config.line_separator, addText) # Reputting the categories newtext = textlib.replaceCategoryLinks(newtext, categoriesInside, site, True) # Dealing the stars' issue allstars = [] starstext = textlib.removeDisabledParts(text) for star in starsList: regex = re.compile( '(\{\{(?:template:|)%s\|.*?\}\}[\s]*)' % star, re.I) found = regex.findall(starstext) if found != []: newtext = regex.sub('', newtext) allstars += found if allstars != []: newtext = newtext.strip() + config.line_separator * 2 allstars.sort() for element in allstars: newtext += '%s%s' % (element.strip(), config.LS) # Adding the interwiki newtext = textlib.replaceLanguageLinks(newtext, interwikiInside, site) else: newtext += u"%s%s" % (config.line_separator, addText) else: newtext = addText + config.line_separator + text if putText and text != newtext: pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<" % page.title()) pywikibot.showDiff(text, newtext) # Let's put the changes. while True: # If someone load it as module, maybe it's not so useful to put the # text in the page if putText: if not always: choice = pywikibot.input_choice( u'Do you want to accept these changes?', [('Yes', 'y'), ('No', 'n'), ('All', 'a'), ('open in Browser', 'b')], 'n', automatic_quit=False) if choice == 'a': always = True elif choice == 'n': return (False, False, always) elif choice == 'b': pywikibot.bot.open_webbrowser(page) if always or choice == 'y': try: if always: page.put(newtext, summary, minorEdit=page.namespace() != 3) else: page.put_async(newtext, summary, minorEdit=page.namespace() != 3) except pywikibot.EditConflict: pywikibot.output(u'Edit conflict! skip!') return (False, False, always) except pywikibot.ServerError: errorCount += 1 if errorCount < config.max_retries: pywikibot.output(u'Server Error! Wait..') time.sleep(config.retry_wait) continue else: raise pywikibot.ServerError(u'Fifth Server Error!') except pywikibot.SpamfilterError as e: pywikibot.output( u'Cannot change %s because of blacklist entry %s' % (page.title(), e.url)) return (False, False, always) except pywikibot.LockedPage: pywikibot.output(u'Skipping %s (locked page)' % page.title()) return (False, False, always) except pywikibot.PageNotSaved as error: pywikibot.output(u'Error putting page: %s' % error.args) return (False, False, always) else: # Break only if the errors are one after the other... errorCount = 0 return (True, True, always) else: return (text, newtext, always)
def processRE(param, rx): cleaned_text = textlib.removeDisabledParts(unicode(param.value.strip())) relist = re.findall(rx, cleaned_text) return relist
def remove_cats_and_comments(self, text): """Remove categories, comments and trailing spaces from wikitext.""" text = textlib.removeCategoryLinks(text, site=self.site) text = textlib.removeDisabledParts(text, tags=['comments']) return text.strip()
def run(self): """Run the Bot.""" try: deadLinks = codecs.open(listof404pages, 'r', 'latin_1').read() except IOError: pywikibot.output( 'You need to download ' 'http://www.twoevils.org/files/wikipedia/404-links.txt.gz ' 'and to ungzip it in the same directory') raise socket.setdefaulttimeout(30) editedpages = 0 for page in self.generator: try: # Load the page's text from the wiki new_text = page.get() if not page.canBeEdited(): pywikibot.output(u"You can't edit page %s" % page.title(asLink=True)) continue except pywikibot.NoPage: pywikibot.output(u'Page %s not found' % page.title(asLink=True)) continue except pywikibot.IsRedirectPage: pywikibot.output(u'Page %s is a redirect' % page.title(asLink=True)) continue # for each link to change for match in linksInRef.finditer( textlib.removeDisabledParts(page.get())): link = match.group(u'url') # debugging purpose # print link if u'jstor.org' in link: # TODO: Clean URL blacklist continue ref = RefLink(link, match.group('name')) f = None try: socket.setdefaulttimeout(20) try: f = urlopen(ref.url.decode("utf8")) except UnicodeError: ref.url = quote(ref.url.encode("utf8"), "://") f = urlopen(ref.url) # Try to get Content-Type from server headers = f.info() contentType = headers.getheader('Content-Type') if contentType and not self.MIME.search(contentType): if ref.link.lower().endswith('.pdf') and \ not self.getOption('ignorepdf'): # If file has a PDF suffix self.getPDFTitle(ref, f) else: pywikibot.output( u'\03{lightyellow}WARNING\03{default} : ' u'media : %s ' % ref.link) if ref.title: if not re.match( u'(?i) *microsoft (word|excel|visio)', ref.title): ref.transform(ispdf=True) repl = ref.refTitle() else: pywikibot.output( u'\03{lightyellow}WARNING\03{default} : ' u'PDF title blacklisted : %s ' % ref.title) repl = ref.refLink() else: repl = ref.refLink() new_text = new_text.replace(match.group(), repl) continue # Get the real url where we end (http redirects !) redir = f.geturl() if redir != ref.link and \ domain.findall(redir) == domain.findall(link): if soft404.search(redir) and \ not soft404.search(ref.link): pywikibot.output( u'\03{lightyellow}WARNING\03{default} : ' u'Redirect 404 : %s ' % ref.link) continue if dirIndex.match(redir) and \ not dirIndex.match(ref.link): pywikibot.output( u'\03{lightyellow}WARNING\03{default} : ' u'Redirect to root : %s ' % ref.link) continue # uncompress if necessary if headers.get('Content-Encoding') in ('gzip', 'x-gzip'): # XXX: small issue here: the whole page is downloaded # through f.read(). It might fetch big files/pages. # However, truncating an encoded gzipped stream is not # an option, or unzipping will fail. compressed = io.BytesIO(f.read()) f = gzip.GzipFile(fileobj=compressed) # Read the first 1,000,000 bytes (0.95 MB) linkedpagetext = f.read(1000000) socket.setdefaulttimeout(None) except UnicodeError: # example : http://www.adminet.com/jo/20010615¦/ECOC0100037D.html # in [[fr:Cyanure]] pywikibot.output( u'\03{lightred}Bad link\03{default} : %s in %s' % (ref.url, page.title(asLink=True))) continue except HTTPError as e: pywikibot.output(u'HTTP error (%s) for %s on %s' % (e.code, ref.url, page.title(asLink=True)), toStdout=True) # 410 Gone, indicates that the resource has been purposely # removed if e.code == 410 or \ (e.code == 404 and (u'\t%s\t' % ref.url in deadLinks)): repl = ref.refDead() new_text = new_text.replace(match.group(), repl) continue except (URLError, socket.error, IOError, httplib.error) as e: pywikibot.output(u'Can\'t retrieve page %s : %s' % (ref.url, e)) continue except ValueError: # Known bug of httplib, google for : # "httplib raises ValueError reading chunked content" continue finally: if f: f.close() # remove <script>/<style>/comments/CDATA tags linkedpagetext = self.NON_HTML.sub('', linkedpagetext) meta_content = self.META_CONTENT.search(linkedpagetext) enc = [] s = None if contentType: # use charset from http header s = self.CHARSET.search(contentType) if meta_content: tag = meta_content.group() # Prefer the contentType from the HTTP header : if not contentType: contentType = tag if not s: # use charset from html s = self.CHARSET.search(tag) if s: tmp = s.group('enc').strip("\"' ").lower() naked = re.sub(r'[ _\-]', '', tmp) # Convert to python correct encoding names if naked == "gb2312": enc.append("gbk") elif naked == "shiftjis": enc.append("shift jis 2004") enc.append("cp932") elif naked == "xeucjp": enc.append("euc-jp") else: enc.append(tmp) else: pywikibot.output(u'No charset found for %s' % ref.link) if not contentType: pywikibot.output(u'No content-type found for %s' % ref.link) continue elif not self.MIME.search(contentType): pywikibot.output( u'\03{lightyellow}WARNING\03{default} : media : %s ' % ref.link) repl = ref.refLink() new_text = new_text.replace(match.group(), repl) continue # Ugly hacks to try to survive when both server and page # return no encoding. # Uses most used encodings for each national suffix if u'.ru' in ref.link or u'.su' in ref.link: # see http://www.sci.aha.ru/ATL/ra13a.htm : no server # encoding, no page encoding enc = enc + ['koi8-r', 'windows-1251'] elif u'.jp' in ref.link: enc.append("shift jis 2004") enc.append("cp932") elif u'.kr' in ref.link: enc.append("euc-kr") enc.append("cp949") elif u'.zh' in ref.link: enc.append("gbk") if 'utf-8' not in enc: enc.append('utf-8') try: u = linkedpagetext.decode(enc[0]) # Bug 67410 except (UnicodeDecodeError, LookupError) as e: pywikibot.output(u'%s : Decoding error - %s' % (ref.link, e)) continue # Retrieves the first non empty string inside <title> tags for m in self.TITLE.finditer(u): t = m.group() if t: ref.title = t ref.transform() if ref.title: break if not ref.title: repl = ref.refLink() new_text = new_text.replace(match.group(), repl) pywikibot.output(u'%s : No title found...' % ref.link) continue # XXX Ugly hack if u'é' in ref.title: repl = ref.refLink() new_text = new_text.replace(match.group(), repl) pywikibot.output(u'%s : Hybrid encoding...' % ref.link) continue if self.titleBlackList.match(ref.title): repl = ref.refLink() new_text = new_text.replace(match.group(), repl) pywikibot.output(u'\03{lightred}WARNING\03{default} %s : ' u'Blacklisted title (%s)' % (ref.link, ref.title)) continue # Truncate long titles. 175 is arbitrary if len(ref.title) > 175: ref.title = ref.title[:175] + "..." repl = ref.refTitle() new_text = new_text.replace(match.group(), repl) # Add <references/> when needed, but ignore templates ! if page.namespace != 10: if self.norefbot.lacksReferences(new_text): new_text = self.norefbot.addReferences(new_text) new_text = self.deduplicator.process(new_text) self.userPut(page, page.text, new_text, comment=self.msg, ignore_save_related_errors=True, ignore_server_errors=True) if new_text == page.text: continue else: editedpages += 1 if self.getOption('limit') and editedpages >= self.getOption('limit'): pywikibot.output('Edited %s pages, stopping.' % self.getOption('limit')) return if editedpages % 20 == 0: pywikibot.output( '\03{lightgreen}Checking stop page...\03{default}') actualRev = self.stopPage.latest_revision_id if actualRev != self.stopPageRevId: pywikibot.output( u'[[%s]] has been edited : Someone wants us to stop.' % self.stopPage) return
def run(self): """Run the Bot.""" try: deadLinks = codecs.open(listof404pages, 'r', 'latin_1').read() except IOError: pywikibot.output( 'You need to download ' 'http://www.twoevils.org/files/wikipedia/404-links.txt.gz ' 'and to ungzip it in the same directory') raise editedpages = 0 for page in self.generator: try: # Load the page's text from the wiki new_text = page.get() if not page.canBeEdited(): pywikibot.output(u"You can't edit page %s" % page.title(asLink=True)) continue except pywikibot.NoPage: pywikibot.output(u'Page %s not found' % page.title(asLink=True)) continue except pywikibot.IsRedirectPage: pywikibot.output(u'Page %s is a redirect' % page.title(asLink=True)) continue # for each link to change for match in linksInRef.finditer( textlib.removeDisabledParts(page.get())): link = match.group(u'url') # debugging purpose # print link if u'jstor.org' in link: # TODO: Clean URL blacklist continue ref = RefLink(link, match.group('name')) f = None try: f = comms.http.fetch( ref.url, use_fake_user_agent=self._use_fake_user_agent) # Try to get Content-Type from server contentType = f.response_headers.get('content-type') if contentType and not self.MIME.search(contentType): if ref.link.lower().endswith('.pdf') and \ not self.getOption('ignorepdf'): # If file has a PDF suffix self.getPDFTitle(ref, f) else: pywikibot.output(color_format( '{lightyellow}WARNING{default} : ' 'media : {0} ', ref.link)) if ref.title: if not re.match( u'(?i) *microsoft (word|excel|visio)', ref.title): ref.transform(ispdf=True) repl = ref.refTitle() else: pywikibot.output(color_format( '{lightyellow}WARNING{default} : ' 'PDF title blacklisted : {0} ', ref.title)) repl = ref.refLink() else: repl = ref.refLink() new_text = new_text.replace(match.group(), repl) continue # Get the real url where we end (http redirects !) redir = f.data.url if redir != ref.link and \ domain.findall(redir) == domain.findall(link): if soft404.search(redir) and \ not soft404.search(ref.link): pywikibot.output(color_format( '{lightyellow}WARNING{default} : ' 'Redirect 404 : {0} ', ref.link)) continue if dirIndex.match(redir) and \ not dirIndex.match(ref.link): pywikibot.output(color_format( u'{lightyellow}WARNING{default} : ' u'Redirect to root : {0} ', ref.link)) continue if f.status != requests.codes.ok: pywikibot.output(u'HTTP error (%s) for %s on %s' % (f.status, ref.url, page.title(asLink=True)), toStdout=True) # 410 Gone, indicates that the resource has been purposely # removed if f.status == 410 or \ (f.status == 404 and (u'\t%s\t' % ref.url in deadLinks)): repl = ref.refDead() new_text = new_text.replace(match.group(), repl) continue linkedpagetext = f.content except UnicodeError: # example : http://www.adminet.com/jo/20010615¦/ECOC0100037D.html # in [[fr:Cyanure]] pywikibot.output(color_format( '{lightred}Bad link{default} : {0} in {1}', ref.url, page.title(asLink=True))) continue except (URLError, socket.error, IOError, httplib.error) as e: pywikibot.output(u'Can\'t retrieve page %s : %s' % (ref.url, e)) continue # remove <script>/<style>/comments/CDATA tags linkedpagetext = self.NON_HTML.sub(b'', linkedpagetext) meta_content = self.META_CONTENT.search(linkedpagetext) enc = [] s = None if contentType: # use charset from http header s = self.CHARSET.search(contentType) if meta_content: tag = meta_content.group() # Prefer the contentType from the HTTP header : if not contentType: contentType = tag if not s: # use charset from html s = self.CHARSET.search(str(tag)) if s: tmp = s.group('enc').strip("\"' ").lower() naked = re.sub(r'[ _\-]', '', tmp) # Convert to python correct encoding names if naked == "gb2312": enc.append("gbk") elif naked == "shiftjis": enc.append("shift jis 2004") enc.append("cp932") elif naked == "xeucjp": enc.append("euc-jp") else: enc.append(tmp) else: pywikibot.output(u'No charset found for %s' % ref.link) if not contentType: pywikibot.output(u'No content-type found for %s' % ref.link) continue elif not self.MIME.search(contentType): pywikibot.output(color_format( '{lightyellow}WARNING{default} : media : {0} ', ref.link)) repl = ref.refLink() new_text = new_text.replace(match.group(), repl) continue # Ugly hacks to try to survive when both server and page # return no encoding. # Uses most used encodings for each national suffix if u'.ru' in ref.link or u'.su' in ref.link: # see http://www.sci.aha.ru/ATL/ra13a.htm : no server # encoding, no page encoding enc = enc + ['koi8-r', 'windows-1251'] elif u'.jp' in ref.link: enc.append("shift jis 2004") enc.append("cp932") elif u'.kr' in ref.link: enc.append("euc-kr") enc.append("cp949") elif u'.zh' in ref.link: enc.append("gbk") if 'utf-8' not in enc: enc.append('utf-8') try: u = linkedpagetext.decode(enc[0]) # Bug T69410 except (UnicodeDecodeError, LookupError) as e: pywikibot.output(u'%s : Decoding error - %s' % (ref.link, e)) continue # Retrieves the first non empty string inside <title> tags for m in self.TITLE.finditer(u): t = m.group() if t: ref.title = t ref.transform() if ref.title: break if not ref.title: repl = ref.refLink() new_text = new_text.replace(match.group(), repl) pywikibot.output(u'%s : No title found...' % ref.link) continue # XXX Ugly hack if u'é' in ref.title: repl = ref.refLink() new_text = new_text.replace(match.group(), repl) pywikibot.output(u'%s : Hybrid encoding...' % ref.link) continue if self.titleBlackList.match(ref.title): repl = ref.refLink() new_text = new_text.replace(match.group(), repl) pywikibot.output(color_format( '{lightred}WARNING{default} {0} : ' 'Blacklisted title ({1})', ref.link, ref.title)) continue # Truncate long titles. 175 is arbitrary if len(ref.title) > 175: ref.title = ref.title[:175] + "..." repl = ref.refTitle() new_text = new_text.replace(match.group(), repl) # Add <references/> when needed, but ignore templates ! if page.namespace != 10: if self.norefbot.lacksReferences(new_text): new_text = self.norefbot.addReferences(new_text) new_text = self.deduplicator.process(new_text) self.userPut(page, page.text, new_text, summary=self.msg, ignore_save_related_errors=True, ignore_server_errors=True) if new_text == page.text: continue else: editedpages += 1 if self.getOption('limit') and editedpages >= self.getOption('limit'): pywikibot.output('Edited %s pages, stopping.' % self.getOption('limit')) return if editedpages % 20 == 0: pywikibot.output(color_format( '{lightgreen}Checking stop page...{default}')) actualRev = self.stopPage.latest_revision_id if actualRev != self.stopPageRevId: pywikibot.output( u'[[%s]] has been edited : Someone wants us to stop.' % self.stopPage) return
def _match_xml_page_text(text): """Match page text.""" text = textlib.removeDisabledParts(text) return _ref_regex.search(text) and not _references_regex.search(text)
def linkedImages(page): """Return a list of Pages that this Page links to. Only returns pages from "normal" internal links. Category links are omitted unless prefixed with ":". Image links are omitted when parameter withImageLinks is False. Embedded templates are omitted (but links within them are returned). All interwiki and external links are omitted. @param thistxt: the wikitext of the page @return: a list of Page objects. """ Rlink = re.compile(r'\[\[(?P<title>[^\]\|\[]*)(\|[^\]]*)?\]\]') result = [] try: thistxt = textlib.removeLanguageLinks(page.get(get_redirect=True), page.site) except pywikibot.NoPage: raise except pywikibot.IsRedirectPage: raise except pywikibot.SectionError: return [] thistxt = textlib.removeCategoryLinks(thistxt, page.site) # remove HTML comments, pre, nowiki, and includeonly sections # from text before processing thistxt = textlib.removeDisabledParts(thistxt) # resolve {{ns:-1}} or {{ns:Help}} # thistxt = page.site.resolvemagicwords(thistxt) for match in Rlink.finditer(thistxt): try: #print(match.group(0)) title = match.group('title') title = title.replace("_", " ").strip(" ") # print title if title == "": # empty link - problem in the page continue # convert relative link to absolute link if title.startswith(".."): parts = self.title().split('/') parts.pop() title = '/'.join(parts) + title[2:] elif title.startswith("/"): title = '%s/%s' % (page.title(), title[1:]) if title.startswith("#"): # this is an internal section link continue if not page.site.isInterwikiLink(title): page2 = pywikibot.Page(page.site, title) try: hash(str(page2)) except Exception: pywikibot.output("Page %s contains invalid link to [[%s]]." % (page.title(), title)) continue if not page2.isImage(): continue if page2.title(withSection=False) and page2 not in result: result.append(page2) except pywikibot.NoUsername: continue except: raise return result
def remove_cats_and_comments(self, text): """Remove categories, comments and trailing spaces from wikitext.""" text = textlib.removeCategoryLinks(text, site=self.site) text = textlib.removeDisabledParts(text, tags=['comments']) return text.strip()
def _match_xml_page_text(text): """Match page text.""" text = textlib.removeDisabledParts(text) return _ref_regex.search(text) and not _references_regex.search(text)
def add_text(page, addText, summary=None, regexSkip=None, regexSkipUrl=None, always=False, up=False, putText=True, oldTextGiven=None, reorderEnabled=True, create=False): """ Add text to a page. @rtype: tuple of (text, newtext, always) """ site = page.site if not summary: summary = i18n.twtranslate(site, 'add_text-adding', {'adding': addText[:200]}) # When a page is tagged as "really well written" it has a star in the # interwiki links. This is a list of all the templates used (in regex # format) to make the stars appear. errorCount = 0 if putText: pywikibot.output(u'Loading %s...' % page.title()) if oldTextGiven is None: try: text = page.get() except pywikibot.NoPage: if create: pywikibot.output(u"%s doesn't exist, creating it!" % page.title()) text = u'' else: pywikibot.output(u"%s doesn't exist, skip!" % page.title()) return (False, False, always) except pywikibot.IsRedirectPage: pywikibot.output(u"%s is a redirect, skip!" % page.title()) return (False, False, always) else: text = oldTextGiven # Understand if the bot has to skip the page or not # In this way you can use both -except and -excepturl if regexSkipUrl is not None: url = page.full_url() result = re.findall(regexSkipUrl, site.getUrl(url)) if result != []: pywikibot.output( 'Exception! regex (or word) used with -exceptUrl ' 'is in the page. Skip!\n' 'Match was: %s' % result) return (False, False, always) if regexSkip is not None: result = re.findall(regexSkip, text) if result != []: pywikibot.output( 'Exception! regex (or word) used with -except ' 'is in the page. Skip!\n' 'Match was: %s' % result) return (False, False, always) # If not up, text put below if not up: newtext = text # Translating the \\n into binary \n addText = addText.replace('\\n', config.line_separator) if (reorderEnabled): # Getting the categories categoriesInside = textlib.getCategoryLinks(newtext, site) # Deleting the categories newtext = textlib.removeCategoryLinks(newtext, site) # Getting the interwiki interwikiInside = textlib.getLanguageLinks(newtext, site) # Removing the interwiki newtext = textlib.removeLanguageLinks(newtext, site) # Adding the text newtext += u"%s%s" % (config.line_separator, addText) # Reputting the categories newtext = textlib.replaceCategoryLinks(newtext, categoriesInside, site, True) # Dealing the stars' issue allstars = [] starstext = textlib.removeDisabledParts(text) for star in starsList: regex = re.compile('(\{\{(?:template:|)%s\|.*?\}\}[\s]*)' % star, re.I) found = regex.findall(starstext) if found != []: newtext = regex.sub('', newtext) allstars += found if allstars != []: newtext = newtext.strip() + config.line_separator * 2 allstars.sort() for element in allstars: newtext += '%s%s' % (element.strip(), config.LS) # Adding the interwiki newtext = textlib.replaceLanguageLinks(newtext, interwikiInside, site) else: newtext += u"%s%s" % (config.line_separator, addText) else: newtext = addText + config.line_separator + text if putText and text != newtext: pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<" % page.title()) pywikibot.showDiff(text, newtext) # Let's put the changes. while True: # If someone load it as module, maybe it's not so useful to put the # text in the page if putText: if not always: choice = pywikibot.input_choice( u'Do you want to accept these changes?', [('Yes', 'y'), ('No', 'n'), ('All', 'a'), ('open in Browser', 'b')], 'n', automatic_quit=False) if choice == 'a': always = True elif choice == 'n': return (False, False, always) elif choice == 'b': pywikibot.bot.open_webbrowser(page) if always or choice == 'y': try: if always: page.put(newtext, summary, minorEdit=page.namespace() != 3) else: page.put_async(newtext, summary, minorEdit=page.namespace() != 3) except pywikibot.EditConflict: pywikibot.output(u'Edit conflict! skip!') return (False, False, always) except pywikibot.ServerError: errorCount += 1 if errorCount < config.max_retries: pywikibot.output(u'Server Error! Wait..') time.sleep(config.retry_wait) continue else: raise pywikibot.ServerError(u'Fifth Server Error!') except pywikibot.SpamfilterError as e: pywikibot.output( u'Cannot change %s because of blacklist entry %s' % (page.title(), e.url)) return (False, False, always) except pywikibot.LockedPage: pywikibot.output(u'Skipping %s (locked page)' % page.title()) return (False, False, always) except pywikibot.PageNotSaved as error: pywikibot.output(u'Error putting page: %s' % error.args) return (False, False, always) else: # Break only if the errors are one after the other... errorCount = 0 return (True, True, always) else: return (text, newtext, always)
def standardizePageFooter(self, text): """ Standardize page footer. Makes sure that interwiki links, categories and star templates are put to the correct position and into the right order. This combines the old instances standardizeInterwiki and standardizeCategories The page footer has the following section in that sequence: 1. categories 2. ## TODO: template beyond categories ## 3. additional information depending on local site policy 4. stars templates for featured and good articles 5. interwiki links """ starsList = [ u'bueno', u'bom interwiki', u'cyswllt[ _]erthygl[ _]ddethol', u'dolen[ _]ed', u'destacado', u'destaca[tu]', u'enllaç[ _]ad', u'enllaz[ _]ad', u'leam[ _]vdc', u'legătură[ _]a[bcf]', u'liamm[ _]pub', u'lien[ _]adq', u'lien[ _]ba', u'liên[ _]kết[ _]bài[ _]chất[ _]lượng[ _]tốt', u'liên[ _]kết[ _]chọn[ _]lọc', u'ligam[ _]adq', u'ligazón[ _]a[bd]', u'ligoelstara', u'ligoleginda', u'link[ _][afgu]a', u'link[ _]adq', u'link[ _]f[lm]', u'link[ _]km', u'link[ _]sm', u'linkfa', u'na[ _]lotura', u'nasc[ _]ar', u'tengill[ _][úg]g', u'ua', u'yüm yg', u'רא', u'وصلة مقالة جيدة', u'وصلة مقالة مختارة', ] categories = None interwikiLinks = None allstars = [] # The PyWikipediaBot is no longer allowed to touch categories on the # German Wikipedia. See # https://de.wikipedia.org/wiki/Hilfe_Diskussion:Personendaten/Archiv/1#Position_der_Personendaten_am_.22Artikelende.22 # ignoring nn-wiki of cause of the comment line above iw section if not self.template and '{{Personendaten' not in text and \ '{{SORTIERUNG' not in text and '{{DEFAULTSORT' not in text and \ self.site.code not in ('et', 'it', 'bg', 'ru'): categories = textlib.getCategoryLinks(text, site=self.site) if not self.talkpage: # and pywikibot.calledModuleName() <> 'interwiki': subpage = False if self.template: loc = None try: tmpl, loc = moved_links[self.site.code] del tmpl except KeyError: pass if loc is not None and loc in self.title: subpage = True interwikiLinks = textlib.getLanguageLinks( text, insite=self.site, template_subpage=subpage) # Removing the interwiki text = textlib.removeLanguageLinks(text, site=self.site) # Removing the stars' issue starstext = textlib.removeDisabledParts(text) for star in starsList: regex = re.compile(r'(\{\{(?:template:|)%s\|.*?\}\}[\s]*)' % star, re.I) found = regex.findall(starstext) if found != []: text = regex.sub('', text) allstars += found # Adding categories if categories: # TODO: Sorting categories in alphabetic order. # e.g. using categories.sort() # TODO: Taking main cats to top # for name in categories: # if re.search(u"(.+?)\|(.{,1}?)",name.title()) or name.title()==name.title().split(":")[0]+title: # categories.remove(name) # categories.insert(0, name) text = textlib.replaceCategoryLinks(text, categories, site=self.site) # Adding stars templates if allstars: text = text.strip() + self.site.family.interwiki_text_separator allstars.sort() for element in allstars: text += '%s%s' % (element.strip(), config.line_separator) pywikibot.log(u'%s' % element.strip()) # Adding the interwiki if interwikiLinks: text = textlib.replaceLanguageLinks(text, interwikiLinks, site=self.site, template=self.template, template_subpage=subpage) return text
def find_and_replace(self, text, init): new_params = [] old_params = [] unknown_params = [] removed_params = [] changed = False for template, fielddict in textlib.extract_templates_and_params( text, remove_disabled_parts=False, strip=False): if self.normalize(template) not in (self.template, self.new_template): continue changed = self.normalize(template) != self.new_template start_match = re.search(r'\{\{\s*((%s)\s*:\s*)?%s\s*' % ( '|'.join(self.site.namespaces[10]), re.escape(template)), text) if not start_match: if not init: pywikibot.error("Couldn't find the template") return text, 0 start = start_match.start() if len(fielddict) > 0: end = text.index('|', start) else: end = text.index('}}', start) unnamed = {} for name, value in chain(fielddict.items(), IterUnnamed(unnamed)): end += len('|%s=%s' % (name, value)) name = name.strip() value = (value .replace('\n<!-- Zastaralé parametry -->', '') .replace('\n<!-- Neznámé parametry -->', '') .strip()) try: new_name = self.handle_param(name) except OldParamException: if textlib.removeDisabledParts(value, ['comments']).strip(): old_params.append( (name, value) ) except RemoveParamException: changed = True if textlib.removeDisabledParts(value, ['comments']).strip(): removed_params.append( (name, value) ) except UnknownParamException: if textlib.removeDisabledParts(value, ['comments']).strip(): unknown_params.append( (name, value) ) except AssertionError: pywikibot.error('Couldn\'t handle parameter "%s"' % name) return text, 0 except UnnamedParamException: unnamed[value] = '' else: new_params.append( (new_name, value) ) if new_name != name: changed = True end += len('}}') while text[start:end].count('{{') < text[start:end].count('}}'): end = text[:end].rindex('}}') + len('}}') if text[start:end].count('{{') > text[start:end].count('}}'): ballance = 1 end = start while ballance > 0: next_close = text.index('}}', end) ballance += text[end:next_close].count('{{') - 1 end = next_close + len('}}') if not text[start:end].endswith('}}'): # elif? end = text[:end].rindex('}}') + len('}}') if (end < start or not text[start:end].endswith('}}') or text[start:end].count('{{') != text[start:end].count('}}')): pywikibot.error("Couldn't parse the template") return text, 0 break else: pywikibot.error("Couldn't parse the template") return text, 0 if not changed: pywikibot.output('No parameters changed') return text, 0 while end < len(text) and text[end].isspace(): # todo: also before end += 1 lines = [] nested = 0 for line in text[start:end].splitlines(): if nested == 1 and re.match(' *\|', line): lines.append(line) nested += line.count('{{') - line.count('}}') space_before = '' if len(lines) > 0 and choice(lines).startswith(' '): space_before = ' ' self.handle_params(new_params, old_params, removed_params, unknown_params) self.deduplicate(new_params) new_params.sort(key=self.key_for_sort) new_template = '{{%s' % self.new_template if len(new_params) > 0: new_template += '\n' for param, value in new_params: new_template += '%s| %s = %s\n' % (space_before, param, value) if len(old_params) > 0: new_template += '<!-- Zastaralé parametry -->\n' for param, value in old_params: new_template += '%s| %s = %s\n' % (space_before, param, value) if len(unknown_params) > 0: new_template += '<!-- Neznámé parametry -->\n' for param, value in unknown_params: new_template += '%s| %s = %s\n' % (space_before, param, value) new_template += '}}\n' return text[:start] + new_template + text[end:], end