def main(): xmlFilename = None always = False ignorepdf = False limit = None namespaces = [] generator = None summary = None # Process global args and prepare generator args parser local_args = pywikibot.handleArgs() genFactory = pagegenerators.GeneratorFactory() for arg in local_args: if arg.startswith('-namespace:'): try: namespaces.append(int(arg[11:])) except ValueError: namespaces.append(arg[11:]) elif arg.startswith('-summary:'): summary = arg[9:] elif arg == '-always': always = True elif arg == '-ignorepdf': ignorepdf = True elif arg.startswith('-limit:'): limit = int(arg[7:]) elif arg.startswith('-xmlstart'): if len(arg) == 9: xmlStart = pywikibot.input( u'Please enter the dumped article to start with:') else: xmlStart = arg[10:] elif arg.startswith('-xml'): if len(arg) == 4: xmlFilename = pywikibot.input( u'Please enter the XML dump\'s filename:') else: xmlFilename = arg[5:] else: genFactory.handleArg(arg) if xmlFilename: try: xmlStart except NameError: xmlStart = None generator = XmlDumpPageGenerator(xmlFilename, xmlStart, namespaces) if not generator: generator = genFactory.getCombinedGenerator() if not generator: # syntax error, show help text from the top of this file pywikibot.showHelp() return generator = pagegenerators.PreloadingGenerator(generator, step=50) generator = pagegenerators.RedirectFilterPageGenerator(generator) bot = ReferencesRobot(generator, always, limit, ignorepdf, summary) bot.run()
def main(*args): """ Process command line arguments and invoke bot. If args is an empty list, sys.argv is used. @param args: command line arguments @type args: list of unicode """ xmlFilename = None xmlStart = None options = {} generator = None # Process global args and prepare generator args parser local_args = pywikibot.handle_args(args) genFactory = pagegenerators.GeneratorFactory() for arg in local_args: if arg.startswith('-summary:'): options['summary'] = arg[9:] elif arg == '-always': options['always'] = True elif arg == '-ignorepdf': options['ignorepdf'] = True elif arg == '-repair': options['repair'] = True elif arg.startswith('-limit:'): options['limit'] = int(arg[7:]) elif arg.startswith('-xmlstart'): if len(arg) == 9: xmlStart = pywikibot.input( u'Please enter the dumped article to start with:') else: xmlStart = arg[10:] elif arg.startswith('-xml'): if len(arg) == 4: xmlFilename = pywikibot.input( u'Please enter the XML dump\'s filename:') else: xmlFilename = arg[5:] else: genFactory.handleArg(arg) if xmlFilename: generator = XmlDumpPageGenerator(xmlFilename, xmlStart, genFactory.namespaces) if not generator: generator = genFactory.getCombinedGenerator() if not generator: pywikibot.bot.suggest_help(missing_generator=True) return False if not genFactory.nopreload: generator = pagegenerators.PreloadingGenerator(generator) generator = pagegenerators.RedirectFilterPageGenerator(generator) bot = ReferencesRobot(generator, **options) bot.run() return True
def main(*args): """ Process command line arguments and invoke bot. If args is an empty list, sys.argv is used. @param args: command line arguments @type args: str """ xml_filename = None xml_start = None options = {} generator = None # Process global args and prepare generator args parser local_args = pywikibot.handle_args(args) gen_factory = pagegenerators.GeneratorFactory() for arg in local_args: opt, _, value = arg.partition(':') if opt in ('-summary', '-limit'): options[opt[1:]] = value elif opt in ('-always', '-ignorepdf'): options[opt[1:]] = True elif opt == '-xmlstart': xml_start = value or pywikibot.input( 'Please enter the dumped article to start with:') elif opt == '-xml': xml_filename = value or pywikibot.input( "Please enter the XML dump's filename:") else: gen_factory.handle_arg(arg) if xml_filename: generator = XmlDumpPageGenerator(xml_filename, xml_start, gen_factory.namespaces) if not generator: generator = gen_factory.getCombinedGenerator() if not generator: pywikibot.bot.suggest_help(missing_generator=True) return if not gen_factory.nopreload: generator = pagegenerators.PreloadingGenerator(generator) generator = pagegenerators.RedirectFilterPageGenerator(generator) bot = ReferencesRobot(generator=generator, **options) bot.run()
def main(*args): """ Process command line arguments and invoke bot. If args is an empty list, sys.argv is used. @param args: command line arguments @type args: unicode """ gen = None xmlFilename = None HTTPignore = [] if isinstance(memento_client, ImportError): warn('memento_client not imported: {0}'.format(memento_client), ImportWarning) # Process global args and prepare generator args parser local_args = pywikibot.handle_args(args) genFactory = pagegenerators.GeneratorFactory() for arg in local_args: if arg == '-talk': config.report_dead_links_on_talk = True elif arg == '-notalk': config.report_dead_links_on_talk = False elif arg == '-repeat': gen = RepeatPageGenerator() elif arg.startswith('-ignore:'): HTTPignore.append(int(arg[8:])) elif arg.startswith('-day:'): config.weblink_dead_days = int(arg[5:]) elif arg.startswith('-xmlstart'): if len(arg) == 9: xmlStart = pywikibot.input( 'Please enter the dumped article to start with:') else: xmlStart = arg[10:] elif arg.startswith('-xml'): if len(arg) == 4: xmlFilename = i18n.input('pywikibot-enter-xml-filename') else: xmlFilename = arg[5:] else: genFactory.handleArg(arg) if xmlFilename: try: xmlStart except NameError: xmlStart = None gen = XmlDumpPageGenerator(xmlFilename, xmlStart, genFactory.namespaces) if not gen: gen = genFactory.getCombinedGenerator() if gen: if not genFactory.nopreload: # fetch at least 240 pages simultaneously from the wiki, but more # if a high thread number is set. pageNumber = max(240, config.max_external_links * 2) gen = pagegenerators.PreloadingGenerator(gen, groupsize=pageNumber) gen = pagegenerators.RedirectFilterPageGenerator(gen) bot = WeblinkCheckerRobot(gen, HTTPignore, config.weblink_dead_days) try: bot.run() finally: waitTime = 0 # Don't wait longer than 30 seconds for threads to finish. while countLinkCheckThreads() > 0 and waitTime < 30: try: pywikibot.output('Waiting for remaining {0} threads to ' 'finish, please wait...'.format( countLinkCheckThreads())) # wait 1 second time.sleep(1) waitTime += 1 except KeyboardInterrupt: pywikibot.output('Interrupted.') break if countLinkCheckThreads() > 0: pywikibot.output( 'Remaining {0} threads will be killed.'.format( countLinkCheckThreads())) # Threads will die automatically because they are daemonic. if bot.history.reportThread: bot.history.reportThread.shutdown() # wait until the report thread is shut down; the user can # interrupt it by pressing CTRL-C. try: while bot.history.reportThread.isAlive(): time.sleep(0.1) except KeyboardInterrupt: pywikibot.output('Report thread interrupted.') bot.history.reportThread.kill() pywikibot.output('Saving history...') bot.history.save() return True else: pywikibot.bot.suggest_help(missing_generator=True) return False
def main(): gen = None singlePageTitle = [] xmlFilename = None # Which namespaces should be processed? # default to [] which means all namespaces will be processed namespaces = [] HTTPignore = [] day = 7 # Process global args and prepare generator args parser local_args = pywikibot.handleArgs() genFactory = pagegenerators.GeneratorFactory() for arg in local_args: if arg == '-talk': config.report_dead_links_on_talk = True elif arg == '-notalk': config.report_dead_links_on_talk = False elif arg.startswith('-namespace:'): try: namespaces.append(int(arg[11:])) except ValueError: namespaces.append(arg[11:]) elif arg == '-repeat': gen = RepeatPageGenerator() elif arg.startswith('-ignore:'): HTTPignore.append(int(arg[8:])) elif arg.startswith('-day:'): day = int(arg[5:]) elif arg.startswith('-xmlstart'): if len(arg) == 9: xmlStart = pywikibot.input( u'Please enter the dumped article to start with:') else: xmlStart = arg[10:] elif arg.startswith('-xml'): if len(arg) == 4: xmlFilename = i18n.input('pywikibot-enter-xml-filename') else: xmlFilename = arg[5:] else: if not genFactory.handleArg(arg): singlePageTitle.append(arg) if singlePageTitle: singlePageTitle = ' '.join(singlePageTitle) page = pywikibot.Page(pywikibot.Site(), singlePageTitle) gen = iter([page]) if xmlFilename: try: xmlStart except NameError: xmlStart = None gen = XmlDumpPageGenerator(xmlFilename, xmlStart, namespaces) if not gen: gen = genFactory.getCombinedGenerator() if gen: if namespaces != []: gen = pagegenerators.NamespaceFilterPageGenerator(gen, namespaces) # fetch at least 240 pages simultaneously from the wiki, but more if # a high thread number is set. pageNumber = max(240, config.max_external_links * 2) gen = pagegenerators.PreloadingGenerator(gen, step=pageNumber) gen = pagegenerators.RedirectFilterPageGenerator(gen) bot = WeblinkCheckerRobot(gen, HTTPignore, day) try: bot.run() finally: waitTime = 0 # Don't wait longer than 30 seconds for threads to finish. while countLinkCheckThreads() > 0 and waitTime < 30: try: pywikibot.output(u"Waiting for remaining %i threads to " u"finish, please wait..." % countLinkCheckThreads()) # wait 1 second time.sleep(1) waitTime += 1 except KeyboardInterrupt: pywikibot.output(u'Interrupted.') break if countLinkCheckThreads() > 0: pywikibot.output(u'Remaining %i threads will be killed.' % countLinkCheckThreads()) # Threads will die automatically because they are daemonic. if bot.history.reportThread: bot.history.reportThread.shutdown() # wait until the report thread is shut down; the user can # interrupt it by pressing CTRL-C. try: while bot.history.reportThread.isAlive(): time.sleep(0.1) except KeyboardInterrupt: pywikibot.output(u'Report thread interrupted.') bot.history.reportThread.kill() pywikibot.output(u'Saving history...') bot.history.save() else: pywikibot.showHelp()
def main(*args): """ Process command line arguments and invoke bot. If args is an empty list, sys.argv is used. @param args: command line arguments @type args: list of unicode """ xmlFilename = None options = {} namespaces = [] generator = None # Process global args and prepare generator args parser local_args = pywikibot.handle_args(args) genFactory = pagegenerators.GeneratorFactory() for arg in local_args: if arg.startswith('-namespace:'): try: namespaces.append(int(arg[11:])) except ValueError: namespaces.append(arg[11:]) elif arg.startswith('-summary:'): options['summary'] = arg[9:] elif arg == '-always': options['always'] = True elif arg == '-ignorepdf': options['ignorepdf'] = True elif arg.startswith('-limit:'): options['limit'] = int(arg[7:]) elif arg.startswith('-xmlstart'): if len(arg) == 9: xmlStart = pywikibot.input( u'Please enter the dumped article to start with:') else: xmlStart = arg[10:] elif arg.startswith('-xml'): if len(arg) == 4: xmlFilename = pywikibot.input( u'Please enter the XML dump\'s filename:') else: xmlFilename = arg[5:] else: genFactory.handleArg(arg) if xmlFilename: try: xmlStart except NameError: xmlStart = None generator = XmlDumpPageGenerator(xmlFilename, xmlStart, namespaces) if not generator: generator = genFactory.getCombinedGenerator() if not generator: # syntax error, show help text from the top of this file pywikibot.showHelp() return generator = pagegenerators.PreloadingGenerator(generator, step=50) generator = pagegenerators.RedirectFilterPageGenerator(generator) bot = ReferencesRobot(generator, **options) bot.run()