class file(callbacks.Commands): def plugin(self, irc): return irc.getCallback('Apt') @wrap([ getopts({ **FILTERS_OPTLIST, }), 'something', ]) @add_filters_doc def packages(self, irc, msg, args, opts, filename): """%s <filename> Lists packages that contain the given filename. %s""" opts = dict(opts) plugin = self.plugin(irc) plugin._get_cache() # open the cache filename = filename.encode() # You may want to add '^.*' at the beginning of the pattern; but # don't do that because the regexp then becomes much slower (6s for # a 500MB list, instead of 0.3s). # Given that we're matching end of lines, a line can't be matched # twice anyway because matches can't overlap. pattern = re.compile( rb'%s.*\s+(\S+)/(\S+)$' % re.escape(filename), re.MULTILINE) packages = set() # I can't find a way to do this with python-apt, so let's open and # parse the files directly rootdir = self.plugin(irc)._get_cache_dir() lists = list_content_lists(plugin, irc, msg.channel, opts, rootdir) with multiprocessing.Pool() as pool: results = pool.imap_unordered( search_file, [(pattern, filename, opener) for (filename, opener) in lists]) for result in results: packages.update(result) if packages: irc.reply(format('%L', sorted(packages))) else: irc.error(_('No package found.'))
class file(callbacks.Commands): def plugin(self, irc): return irc.getCallback('Apt') @wrap([ getopts({ 'archs': commalist('something'), }), 'something', ]) def packages(self, irc, msg, args, opts, filename): """<filename> Lists packages that contain the given filename.""" opts = dict(opts) plugin = self.plugin(irc) plugin._get_cache() # open the cache filename = filename.encode() # You may want to add '^.*' at the beginning of the pattern; but # don't do that because the regexp then becomes much slower (6s for # a 500MB list, instead of 0.3s). # Given that we're matching end of lines, a line can't be matched # anyway because matches can't overlap. line_entry_re = re.compile( rb'%s.*\s+(\S+)/(\S+)$' % re.escape(filename), re.MULTILINE) results = set() # I can't find a way to do this with python-apt, so let's open and # parse the files directly rootdir = self.plugin(irc)._get_cache_dir() lists = list_content_lists(plugin, irc, msg.channel, opts, rootdir) for (list_filename, file_opener) in lists: with file_opener(list_filename) as fd: for match in search_lines(line_entry_re, fd): results.add(match[1].decode()) if results: irc.reply(format('%L', sorted(results))) else: irc.error(_('No package found.'))
class Owner(callbacks.Plugin): """Owner-only commands for core Supybot. This is a core Supybot module that should not be removed!""" # This plugin must be first; its priority must be lowest; otherwise odd # things will happen when adding callbacks. def __init__(self, irc=None): if irc is not None: assert not irc.getCallback(self.name()) self.__parent = super(Owner, self) self.__parent.__init__(irc) # Setup command flood detection. self.commands = ircutils.FloodQueue( conf.supybot.abuse.flood.interval()) conf.supybot.abuse.flood.interval.addCallback( self.setFloodQueueTimeout) # Setup plugins and default plugins for commands. # # This needs to be done before we connect to any networks so that the # children of supybot.plugins (the actual plugins) exist and can be # loaded. for (name, s) in registry._cache.items(): if 'alwaysLoadDefault' in name or 'alwaysLoadImportant' in name: continue if name.startswith('supybot.plugins'): try: (_, _, name) = registry.split(name) except ValueError: # unpack list of wrong size. continue # This is just for the prettiness of the configuration file. # There are no plugins that are all-lowercase, so we'll at # least attempt to capitalize them. if name == name.lower(): name = name.capitalize() conf.registerPlugin(name) if name.startswith('supybot.commands.defaultPlugins'): try: (_, _, _, name) = registry.split(name) except ValueError: # unpack list of wrong size. continue registerDefaultPlugin(name, s) # Setup Irc objects, connected to networks. If world.ircs is already # populated, chances are that we're being reloaded, so don't do this. if not world.ircs: for network in conf.supybot.networks(): try: self._connect(network) except socket.error as e: self.log.error('Could not connect to %s: %s.', network, e) except Exception as e: self.log.exception('Exception connecting to %s:', network) self.log.error('Could not connect to %s: %s.', network, e) def callPrecedence(self, irc): return ([], [cb for cb in irc.callbacks if cb is not self]) def outFilter(self, irc, msg): if msg.command == 'PRIVMSG' and not world.testing: if ircutils.strEqual(msg.args[0], irc.nick): self.log.warning('Tried to send a message to myself: %r.', msg) return None return msg def reset(self): # This has to be done somewhere, I figure here is as good place as any. callbacks.IrcObjectProxy._mores.clear() self.__parent.reset() def _connect(self, network, serverPort=None, password='', ssl=False): try: group = conf.supybot.networks.get(network) group.servers()[0] except (registry.NonExistentRegistryEntry, IndexError): if serverPort is None: raise ValueError('connect requires a (server, port) ' \ 'if the network is not registered.') conf.registerNetwork(network, password, ssl) server = '%s:%s' % serverPort conf.supybot.networks.get(network).servers.append(server) assert conf.supybot.networks.get(network).servers(), \ 'No servers are set for the %s network.' % network self.log.debug('Creating new Irc for %s.', network) newIrc = irclib.Irc(network) driver = drivers.newDriver(newIrc) self._loadPlugins(newIrc) return newIrc def _loadPlugins(self, irc): self.log.debug('Loading plugins (connecting to %s).', irc.network) alwaysLoadImportant = conf.supybot.plugins.alwaysLoadImportant() important = conf.supybot.commands.defaultPlugins.importantPlugins() for (name, value) in conf.supybot.plugins.getValues(fullNames=False): if irc.getCallback(name) is None: load = value() if not load and name in important: if alwaysLoadImportant: s = '%s is configured not to be loaded, but is being '\ 'loaded anyway because ' \ 'supybot.plugins.alwaysLoadImportant is True.' self.log.warning(s, name) load = True if load: # We don't load plugins that don't start with a capital # letter. if name[0].isupper() and not irc.getCallback(name): # This is debug because each log logs its beginning. self.log.debug('Loading %s.', name) try: m = plugin.loadPluginModule(name, ignoreDeprecation=True) plugin.loadPluginClass(irc, m) except callbacks.Error as e: # This is just an error message. log.warning(str(e)) except plugins.NoSuitableDatabase as e: s = 'Failed to load %s: no suitable database(%s).' % ( name, e) log.warning(s) except ImportError as e: e = str(e) if e.endswith(name): s = 'Failed to load {0}: No plugin named {0} exists.'.format( utils.str.dqrepr(name)) elif "No module named 'config'" in e: s = ( "Failed to load %s: This plugin may be incompatible " "with your current Python version." % name) else: s = 'Failed to load %s: import error (%s).' % ( name, e) log.warning(s) except Exception as e: log.exception('Failed to load %s:', name) else: # Let's import the module so configuration is preserved. try: _ = plugin.loadPluginModule(name) except Exception as e: log.debug( 'Attempted to load %s to preserve its ' 'configuration, but load failed: %s', name, e) world.starting = False def do376(self, irc, msg): msgs = conf.supybot.networks.get(irc.network).channels.joins() if msgs: for msg in msgs: irc.queueMsg(msg) do422 = do377 = do376 def setFloodQueueTimeout(self, *args, **kwargs): self.commands.timeout = conf.supybot.abuse.flood.interval() def doBatch(self, irc, msg): if not conf.supybot.protocols.irc.experimentalExtensions(): return batch = msg.tagged('batch') # Always not-None on a BATCH message if msg.args[0].startswith('+'): # Start of a batch, we're not interested yet. return if batch.type != 'draft/multiline': # This is not a multiline batch, also not interested. return assert msg.args[0].startswith("-"), ( "BATCH's first argument should start with either - or +, but " "it is %s.") % msg.args[0] # End of multiline batch. It may be a long command. payloads = [] first_privmsg = None for message in batch.messages: if message.command != "PRIVMSG": # We're only interested in PRIVMSGs for the payloads. # (eg. exclude NOTICE) continue elif not payloads: # This is the first PRIVMSG of the batch first_privmsg = message payloads.append(message.args[1]) elif 'draft/multiline-concat' in message.server_tags: # This message is not a new line, but the continuation # of the previous one. payloads.append(message.args[1]) else: # New line; stop here. We're not processing extra lines # either as the rest of the command or as new commands. # This may change in the future. break payload = ''.join(payloads) if not payload: self.log.error( 'Got empty multiline payload. This is a bug, please ' 'report it along with logs.') return assert first_privmsg, "This shouldn't be None unless payload is empty" # Let's build a synthetic message from the various parts of the # batch, to look like the multiline batch was a single (large) # PRIVMSG: # * copy the tags and server tags of the 'BATCH +' command, # * copy the prefix and channel of any of the PRIVMSGs # inside the batch # * create a new args[1] target = first_privmsg.args[0] synthetic_msg = ircmsgs.IrcMsg( msg=batch.messages[0], # tags, server_tags, time prefix=first_privmsg.prefix, command='PRIVMSG', args=(target, payload)) self._doPrivmsgs(irc, synthetic_msg) def doPrivmsg(self, irc, msg): if 'batch' in msg.server_tags: parent_batches = irc.state.getParentBatches(msg) parent_batch_types = [batch.type for batch in parent_batches] if 'draft/multiline' in parent_batch_types \ and conf.supybot.protocols.irc.experimentalExtensions(): # We will handle the message in doBatch when the entire # batch ends. return if 'chathistory' in parent_batch_types: # Either sent automatically by the server upon join, # or triggered by a plugin (why?!) # Either way, replying to commands from the history would # look weird, because it may have been sent a while ago, # and we may have already answered to it. return self._doPrivmsgs(irc, msg) def _doPrivmsgs(self, irc, msg): """If the given message is a command, triggers Limnoria's command-dispatching for that command. Takes the same arguments as ``doPrivmsg`` would, but ``msg`` can potentially be an artificial message synthesized in doBatch from a multiline batch. Usually, a command is a single message, so ``payload=msg.params[0]`` However, when ``msg`` is part of a multiline message, the payload is the concatenation of multiple messages. See <https://ircv3.net/specs/extensions/multiline>. """ assert self is irc.callbacks[0], \ 'Owner isn\'t first callback: %r' % irc.callbacks if ircmsgs.isCtcp(msg): return s = callbacks.addressed(irc, msg) if s: ignored = ircdb.checkIgnored(msg.prefix) if ignored: self.log.info('Ignoring command from %s.', msg.prefix) return maximum = conf.supybot.abuse.flood.command.maximum() self.commands.enqueue(msg) if conf.supybot.abuse.flood.command() \ and self.commands.len(msg) > maximum \ and not ircdb.checkCapability(msg.prefix, 'trusted'): punishment = conf.supybot.abuse.flood.command.punishment() banmask = conf.supybot.protocols.irc.banmask \ .makeBanmask(msg.prefix) self.log.info( 'Ignoring %s for %s seconds due to an apparent ' 'command flood.', banmask, punishment) ircdb.ignores.add(banmask, time.time() + punishment) if conf.supybot.abuse.flood.command.notify(): irc.reply('You\'ve given me %s commands within the last ' '%i seconds; I\'m now ignoring you for %s.' % (maximum, conf.supybot.abuse.flood.interval(), utils.timeElapsed(punishment, seconds=False))) return try: tokens = callbacks.tokenize(s, channel=msg.channel, network=irc.network) self.Proxy(irc, msg, tokens) except SyntaxError as e: if conf.supybot.reply.error.detailed(): irc.error(str(e)) else: irc.replyError(msg=msg) self.log.info('Syntax error: %s', e) def logmark(self, irc, msg, args, text): """<text> Logs <text> to the global Supybot log at critical priority. Useful for marking logfiles for later searching. """ self.log.critical(text) irc.replySuccess() logmark = wrap(logmark, ['text']) def announce(self, irc, msg, args, text): """<text> Sends <text> to all channels the bot is currently on and not lobotomized in. """ u = ircdb.users.getUser(msg.prefix) template = self.registryValue('announceFormat') text = ircutils.standardSubstitute(irc, msg, template, env={ 'owner': u.name, 'text': text }) for channel in irc.state.channels: c = ircdb.channels.getChannel(channel) if not c.lobotomized: irc.queueMsg(ircmsgs.privmsg(channel, text)) irc.noReply() announce = wrap(announce, ['text']) def defaultplugin(self, irc, msg, args, optlist, command, plugin): """[--remove] <command> [<plugin>] Sets the default plugin for <command> to <plugin>. If --remove is given, removes the current default plugin for <command>. If no plugin is given, returns the current default plugin set for <command>. See also, supybot.commands.defaultPlugins.importantPlugins. """ remove = False for (option, arg) in optlist: if option == 'remove': remove = True (_, cbs) = irc.findCallbacksForArgs([command]) if remove: try: conf.supybot.commands.defaultPlugins.unregister(command) irc.replySuccess() except registry.NonExistentRegistryEntry: s = 'I don\'t have a default plugin set for that command.' irc.error(s) elif not cbs: irc.errorInvalid('command', command) elif plugin: if not plugin.isCommand(command): irc.errorInvalid('command in the %s plugin' % plugin.name(), command) registerDefaultPlugin(command, plugin.name()) irc.replySuccess() else: try: irc.reply(conf.supybot.commands.defaultPlugins.get(command)()) except registry.NonExistentRegistryEntry: s = 'I don\'t have a default plugin set for that command.' irc.error(s) defaultplugin = wrap( defaultplugin, [getopts({'remove': ''}), 'commandName', additional('plugin')]) def ircquote(self, irc, msg, args, s): """<string to be sent to the server> Sends the raw string given to the server. """ try: m = ircmsgs.IrcMsg(s) except Exception as e: irc.error(utils.exnToString(e)) else: irc.queueMsg(m) irc.noReply() ircquote = wrap(ircquote, ['text']) def quit(self, irc, msg, args, text): """[<text>] Exits the bot with the QUIT message <text>. If <text> is not given, the default quit message (supybot.plugins.Owner.quitMsg) will be used. If there is no default quitMsg set, your nick will be used. The standard substitutions ($version, $nick, etc.) are all handled appropriately. """ text = text or self.registryValue('quitMsg') or msg.nick text = ircutils.standardSubstitute(irc, msg, text) irc.noReply() m = ircmsgs.quit(text) world.upkeep() for irc in world.ircs[:]: irc.queueMsg(m) irc.die() quit = wrap(quit, [additional('text')]) def flush(self, irc, msg, args): """takes no arguments Runs all the periodic flushers in world.flushers. This includes flushing all logs and all configuration changes to disk. """ world.flush() irc.replySuccess() flush = wrap(flush) def upkeep(self, irc, msg, args, level): """[<level>] Runs the standard upkeep stuff (flushes and gc.collects()). If given a level, runs that level of upkeep (currently, the only supported level is "high", which causes the bot to flush a lot of caches as well as do normal upkeep stuff). """ L = [] if level == 'high': L.append( format('Regexp cache flushed: %n cleared.', (len(re._cache), 'regexp'))) re.purge() L.append( format('Pattern cache flushed: %n cleared.', (len(ircutils._patternCache), 'compiled pattern'))) ircutils._patternCache.clear() L.append( format('hostmaskPatternEqual cache flushed: %n cleared.', (len(ircutils._hostmaskPatternEqualCache), 'result'))) ircutils._hostmaskPatternEqualCache.clear() L.append( format( 'ircdb username cache flushed: %n cleared.', (len(ircdb.users._nameCache), 'username to id mapping'))) ircdb.users._nameCache.clear() L.append( format('ircdb hostmask cache flushed: %n cleared.', (len( ircdb.users._hostmaskCache), 'hostmask to id mapping'))) ircdb.users._hostmaskCache.clear() L.append( format('linecache line cache flushed: %n cleared.', (len(linecache.cache), 'line'))) linecache.clearcache() if minisix.PY2: sys.exc_clear() collected = world.upkeep() if gc.garbage: L.append('Garbage! %r.' % gc.garbage) if collected is not None: # Some time between 5.2 and 7.1, Pypy (3?) started returning None # when gc.collect() is called. L.append(format('%n collected.', (collected, 'object'))) if L: irc.reply(' '.join(L)) else: irc.replySuccess() upkeep = wrap(upkeep, [additional(('literal', ['high']))]) def load(self, irc, msg, args, optlist, name): """[--deprecated] <plugin> Loads the plugin <plugin> from any of the directories in conf.supybot.directories.plugins; usually this includes the main installed directory and 'plugins' in the current directory. --deprecated is necessary if you wish to load deprecated plugins. """ ignoreDeprecation = False for (option, argument) in optlist: if option == 'deprecated': ignoreDeprecation = True if name.endswith('.py'): name = name[:-3] if irc.getCallback(name): irc.error('%s is already loaded.' % name.capitalize()) return try: module = plugin.loadPluginModule(name, ignoreDeprecation) except plugin.Deprecated: irc.error('%s is deprecated. Use --deprecated ' 'to force it to load.' % name.capitalize()) return except ImportError as e: if str(e).endswith(name): irc.error('No plugin named %s exists.' % utils.str.dqrepr(name)) elif "No module named 'config'" in str(e): irc.error( 'This plugin may be incompatible with your current Python ' 'version. Try running 2to3 on it.') else: irc.error(str(e)) return cb = plugin.loadPluginClass(irc, module) name = cb.name() # Let's normalize this. conf.registerPlugin(name, True) irc.replySuccess() load = wrap(load, [getopts({'deprecated': ''}), 'something']) def reload(self, irc, msg, args, name): """<plugin> Unloads and subsequently reloads the plugin by name; use the 'list' command to see a list of the currently loaded plugins. """ if ircutils.strEqual(name, self.name()): irc.error('You can\'t reload the %s plugin.' % name) return callbacks = irc.removeCallback(name) if callbacks: module = sys.modules[callbacks[0].__module__] if hasattr(module, 'reload'): x = module.reload() try: module = plugin.loadPluginModule(name) if hasattr(module, 'reload') and 'x' in locals(): module.reload(x) if hasattr(module, 'config'): from importlib import reload reload(module.config) for callback in callbacks: callback.die() del callback gc.collect() # This makes sure the callback is collected. callback = plugin.loadPluginClass(irc, module) irc.replySuccess() except ImportError: for callback in callbacks: irc.addCallback(callback) irc.error('No plugin named %s exists.' % name) else: irc.error('There was no plugin %s.' % name) reload = wrap(reload, ['something']) def unload(self, irc, msg, args, name): """<plugin> Unloads the callback by name; use the 'list' command to see a list of the currently loaded plugins. Obviously, the Owner plugin can't be unloaded. """ if ircutils.strEqual(name, self.name()): irc.error('You can\'t unload the %s plugin.' % name) return # Let's do this so even if the plugin isn't currently loaded, it doesn't # stay attempting to load. old_callback = irc.getCallback(name) if old_callback: # Normalize the plugin case to prevent duplicate registration # entries, https://github.com/progval/Limnoria/issues/1295 name = old_callback.name() conf.registerPlugin(name, False) callbacks = irc.removeCallback(name) if callbacks: for callback in callbacks: callback.die() del callback gc.collect() irc.replySuccess() return irc.error('There was no plugin %s.' % name) unload = wrap(unload, ['something']) def defaultcapability(self, irc, msg, args, action, capability): """{add|remove} <capability> Adds or removes (according to the first argument) <capability> from the default capabilities given to users (the configuration variable supybot.capabilities stores these). """ if action == 'add': conf.supybot.capabilities().add(capability) irc.replySuccess() elif action == 'remove': try: conf.supybot.capabilities().remove(capability) irc.replySuccess() except KeyError: if ircdb.isAntiCapability(capability): irc.error('That capability wasn\'t in ' 'supybot.capabilities.') else: anticap = ircdb.makeAntiCapability(capability) conf.supybot.capabilities().add(anticap) irc.replySuccess() defaultcapability = wrap(defaultcapability, [('literal', ['add', 'remove']), 'capability']) def disable(self, irc, msg, args, plugin, command): """[<plugin>] <command> Disables the command <command> for all users (including the owners). If <plugin> is given, only disables the <command> from <plugin>. If you want to disable a command for most users but not for yourself, set a default capability of -plugin.command or -command (if you want to disable the command in all plugins). """ if command in ('enable', 'identify'): irc.error('You can\'t disable %s.' % command) return if plugin: if plugin.isCommand(command): pluginCommand = '%s.%s' % (plugin.name(), command) conf.supybot.commands.disabled().add(pluginCommand) plugin._disabled.add(command, plugin.name()) else: irc.error('%s is not a command in the %s plugin.' % (command, plugin.name())) return else: conf.supybot.commands.disabled().add(command) self._disabled.add(command) irc.replySuccess() disable = wrap(disable, [optional('plugin'), 'commandName']) def enable(self, irc, msg, args, plugin, command): """[<plugin>] <command> Enables the command <command> for all users. If <plugin> if given, only enables the <command> from <plugin>. This command is the inverse of disable. """ try: if plugin: plugin._disabled.remove(command, plugin.name()) command = '%s.%s' % (plugin.name(), command) else: self._disabled.remove(command) conf.supybot.commands.disabled().remove(command) irc.replySuccess() except KeyError: irc.error('That command wasn\'t disabled.') enable = wrap(enable, [optional('plugin'), 'commandName']) def rename(self, irc, msg, args, command_plugin, command, newName): """<plugin> <command> <new name> Renames <command> in <plugin> to the <new name>. """ if not command_plugin.isCommand(command): what = 'command in the %s plugin' % command_plugin.name() irc.errorInvalid(what, command) if hasattr(command_plugin, newName): irc.error('The %s plugin already has an attribute named %s.' % (command_plugin, newName)) return plugin.registerRename(command_plugin.name(), command, newName) plugin.renameCommand(command_plugin, command, newName) irc.replySuccess() rename = wrap(rename, ['plugin', 'commandName', 'commandName']) def unrename(self, irc, msg, args, plugin): """<plugin> Removes all renames in <plugin>. The plugin will be reloaded after this command is run. """ try: conf.supybot.commands.renames.unregister(plugin.name()) except registry.NonExistentRegistryEntry: irc.errorInvalid('plugin', plugin.name()) self.reload(irc, msg, [plugin.name()]) # This makes the replySuccess. unrename = wrap(unrename, ['plugin']) def reloadlocale(self, irc, msg, args): """takes no argument Reloads the locale of the bot.""" i18n.reloadLocales() irc.replySuccess()
class Wikifetch(callbacks.Plugin): """Grabs data from Wikipedia and other MediaWiki-powered sites.""" threaded = True def _wiki(self, irc, msg, search, baseurl): """Fetches and replies content from a MediaWiki-powered website.""" reply = '' # Different instances of MediaWiki use different URLs... This tries # to make the parser work for most sites, but still use resonable defaults # such as filling in http:// and appending /wiki to links... # Special cases: Wikia, Wikipedia, Wikimedia (i.e. Wikimedia Commons), Arch Linux Wiki if any(sitename in baseurl for sitename in ('wikia.com', 'wikipedia.org', 'wikimedia.org')): baseurl += '/wiki' elif 'wiki.archlinux.org' in baseurl: baseurl += '/index.php' if not baseurl.lower().startswith(('http://', 'https://')): baseurl = 'http://' + baseurl # first, we get the page addr = '%s/Special:Search?search=%s' % \ (baseurl, quote_plus(search)) self.log.debug('Wikifetch: using URL %s', addr) article = utils.web.getUrl(addr) if sys.version_info[0] >= 3: article = article.decode() # parse the page tree = lxml.html.document_fromstring(article) # check if it gives a "Did you mean..." redirect didyoumean = tree.xpath('//div[@class="searchdidyoumean"]/a' '[@title="Special:Search"]') if didyoumean: redirect = didyoumean[0].text_content().strip() if sys.version_info[0] < 3: if isinstance(redirect, unicode): redirect = redirect.encode('utf-8', 'replace') if isinstance(search, unicode): search = search.encode('utf-8', 'replace') if self.registryValue('showRedirects', msg.args[0]): reply += _('I didn\'t find anything for "%s". ' 'Did you mean "%s"? ') % (search, redirect) addr = "%s/%s" % (baseurl, didyoumean[0].get('href')) article = utils.web.getUrl(addr) if sys.version_info[0] >= 3: article = article.decode() tree = lxml.html.document_fromstring(article) search = redirect # check if it's a page of search results (rather than an article), and # if so, retrieve the first result searchresults = tree.xpath('//div[@class="searchresults"]/ul/li/a') if searchresults: redirect = searchresults[0].text_content().strip() if self.registryValue('showRedirects', msg.args[0]): reply += _('I didn\'t find anything for "%s", but here\'s the ' 'result for "%s": ') % (search, redirect) addr = self.registryValue('url', msg.args[0]) + \ searchresults[0].get('href') article = utils.web.getUrl(addr) if sys.version_info[0] >= 3: article = article.decode() tree = lxml.html.document_fromstring(article) search = redirect # otherwise, simply return the title and whether it redirected elif self.registryValue('showRedirects', msg.args[0]): redirect = re.search( '\(%s <a href=[^>]*>([^<]*)</a>\)' % _('Redirected from'), article) if redirect: try: redirect = tree.xpath( '//span[@class="mw-redirectedfrom"]/a')[0] redirect = redirect.text_content().strip() title = tree.xpath('//*[@class="firstHeading"]') title = title[0].text_content().strip() if sys.version_info[0] < 3: if isinstance(title, unicode): title = title.encode('utf-8', 'replace') if isinstance(redirect, unicode): redirect = redirect.encode('utf-8', 'replace') reply += '"%s" (Redirected from "%s"): ' % (title, redirect) except IndexError: pass # extract the address we got it from - most sites have the perm link # inside the page itself try: addr = tree.find(".//div[@class='printfooter']/a").attrib['href'] addr = re.sub('([&?]|(amp;)?)oldid=\d+$', '', addr) except: pass # check if it's a disambiguation page disambig = tree.xpath('//table[@id="disambigbox"]') or \ tree.xpath('//table[@id="setindexbox"]') if disambig: disambig = tree.xpath('//div[@id="bodyContent"]/div/ul/li') # Hackishly bold all <a> tags r = [] for item in disambig: for link in item.findall('a'): if link.text is not None: link.text = "%s" % link.text item = item.text_content().replace('', '\x02') # Normalize and strip whitespace, to prevent newlines and such # from corrupting the display. item = utils.str.normalizeWhitespace(item).strip() r.append(item) reply += format( _('%u is a disambiguation page. ' 'Possible results include: %L'), addr, r) # or just as bad, a page listing events in that year elif re.search( _('This article is about the year [\d]*\. ' 'For the [a-zA-Z ]* [\d]*, see'), article): reply += _('"%s" is a page full of events that happened in that ' 'year. If you were looking for information about the ' 'number itself, try searching for "%s_(number)", but ' 'don\'t expect anything useful...') % (search, search) # Catch talk pages elif 'ns-talk' in tree.find("body").attrib['class']: reply += format(_('This article appears to be a talk page: %u'), addr) else: p = tree.xpath("//div[@id='mw-content-text']/p[1]") if len(p) == 0 or 'wiki/Special:Search' in addr: if 'wikipedia:wikiproject' in addr.lower(): reply += format( _('This page appears to be a WikiProject page, ' 'but it is too complex for us to parse: %u'), addr) else: irc.error(_('Not found, or page malformed.'), Raise=True) else: p = p[0] # Replace <b> tags with IRC-style bold, this has to be # done indirectly because unescaped '\x02' is invalid in XML for b_tag in p.xpath('//b'): b_tag.text = "%s" % (b_tag.text or '') p = p.text_content() p = p.replace('', '\x02') # Get rid of newlines, etc., that can corrupt the output. p = utils.str.normalizeWhitespace(p) p = p.strip() if sys.version_info[0] < 3: if isinstance(p, unicode): p = p.encode('utf-8', 'replace') if isinstance(reply, unicode): reply = reply.encode('utf-8', 'replace') reply += format('%s %s %u', p, _('Retrieved from'), addr) reply = reply.replace('&', '&') # Remove inline citations (text[1][2][3], etc.) reply = re.sub('\[\d+\]', '', reply) return reply @internationalizeDocstring @wrap([getopts({'site': 'somethingWithoutSpaces'}), 'text']) def wiki(self, irc, msg, args, optlist, search): """[--site <site>] <search term> Returns the first paragraph of a wiki article. Optionally, a --site argument can be given to override the default (usually Wikipedia) - try using '--site lyrics.wikia.com' or '--site wiki.archlinux.org'.""" optlist = dict(optlist) baseurl = optlist.get('site') or self.registryValue('url', msg.args[0]) text = self._wiki(irc, msg, search, baseurl) irc.reply(text) @internationalizeDocstring @wrap([additional('somethingWithoutSpaces')]) def random(self, irc, msg, args, site): """[<site>] Returns the first paragraph of a random wiki article. Optionally, the --site argument can be given to override the default (usually Wikipedia).""" baseurl = site or self.registryValue('url', msg.args[0]) text = self._wiki(irc, msg, 'Special:Random', baseurl) irc.reply(text)
class package(callbacks.Commands): def plugin(self, irc): return irc.getCallback('Apt') def _get_package(self, irc, package_name): # TODO: add support for selecting the version cache = self.plugin(irc)._get_cache() pkg = cache.get(package_name) if not pkg: irc.error(_('Package not found.'), Raise=True) return pkg @wrap([ getopts({ **FILTERS_OPTLIST, 'types': commalist('somethingWithoutSpaces'), }), 'somethingWithoutSpaces', ]) @add_filters_doc def depends(self, irc, msg, args, optlist, package_name): """%s [--types <type>,<type>,...] <package> Lists dependencies of a package. <type>s are types of dependencies that will be shown. Valid types are: Breaks, Conflicts, Depends, Enhances, PreDepends, Recommends, Replaces, Suggests. %s """ opts = dict(optlist) pkg = self._get_package(irc, package_name) dep_types = opts.get('types', ['Depends', 'PreDepends']) dep_types = [dep_type for dep_type in dep_types] translations = get_dependency_reverse_translations() dep_types = [ translations.get(dep_type, dep_type) for dep_type in dep_types ] # TODO: better version selection pkg_version = filter_versions(self.plugin(irc), irc, msg.channel, opts, pkg.versions)[0] deps = pkg_version.get_dependencies(*dep_types) irc.reply( format(_('%s: %L'), pkg_version, [dep.rawstr for dep in deps])) @wrap([getopts({ **FILTERS_OPTLIST, }), 'somethingWithoutSpaces']) @add_filters_doc def description(self, irc, msg, args, optlist, package_name): """%s <package> Shows the long description of a package. %s""" opts = dict(optlist) pkg = self._get_package(irc, package_name) # TODO: better version selection pkg_version = filter_versions(self.plugin(irc), irc, msg.channel, opts, pkg.versions)[0] description = utils.str.normalizeWhitespace( pkg_version.description) irc.reply( format(_('%s %s: %s'), pkg.shortname, pkg_version.version, description)) @wrap([getopts({ **FILTERS_OPTLIST, }), 'somethingWithoutSpaces']) @add_filters_doc def info(self, irc, msg, args, optlist, package_name): """%s <package> Shows generic information about a package. %s""" opts = dict(optlist) pkg = self._get_package(irc, package_name) # TODO: better version selection pkg_version = filter_versions(self.plugin(irc), irc, msg.channel, opts, pkg.versions)[0] # source_name and priority shouldn't change too often, so I assume # it's safe to call it a "generic info" in a UI priority = get_priority_translations().get(pkg_version.priority, pkg_version.priority) generic_info = format( _('%s (source: %s) is %s and in section "%s".'), pkg.shortname, pkg_version.source_name, priority, pkg.section) version_info = format( _('Version %s package is %S and takes %S when installed.'), pkg_version.version, pkg_version.size, pkg_version.installed_size) irc.reply( format(_('%s %s Description: %s'), generic_info, version_info, pkg_version.summary)) @wrap([ getopts({ **FILTERS_OPTLIST, 'with-version': '', 'description': '', }), 'somethingWithoutSpaces' ]) @add_filters_doc def search(self, irc, msg, args, optlist, package_pattern): """%s [--with-version] [--description] <package> Shows generic information about a package. --with-version also returns matching version numbers. --description searches in package description instead of name. %s""" opts = dict(optlist) search_description = 'description' in opts cache = self.plugin(irc)._get_cache() pattern = re.compile(utils.python.glob2re(package_pattern), re.DOTALL) packages = iter(cache) if not search_description: packages = (pkg for pkg in cache if pattern.match(pkg.shortname)) # Dirty trick to check if the iterator is empty without # consuming it entirely (packages, peek_packages) = itertools.tee(packages) if not next(peek_packages, None): irc.error(_('No package found.'), Raise=True) versions = (version for pkg in packages for version in pkg.versions) if search_description: versions = (version for version in versions if pattern.match(version.description)) # filter_versions is rather slow, so we're putting this guard # before running it. versions = list(itertools.islice(versions, 10000)) if len(versions) >= 10000: irc.error(_('Too many packages match this search.'), Raise=True) versions = filter_versions(self.plugin(irc), irc, msg.channel, opts, versions) if not opts.get('with-version'): package_names = sorted( {version.package.shortname for version in versions}) irc.reply(format('%L', package_names)) return items = [] for version in versions: items.append( format('%s %s (in %L)', version.package.shortname, version.version, {origin.codename for origin in version.origins})) irc.reply(format('%L', items))
class Wikifetch(callbacks.Plugin): """Grabs data from Wikipedia and other MediaWiki-powered sites.""" threaded = True # This defines a series of suffixes this should be added after the domain name. SPECIAL_URLS = { 'wikia.com': '/wiki', 'wikipedia.org': '/wiki', 'wiki.archlinux.org': '/index.php', 'wiki.gentoo.org': '/wiki', 'mediawiki.org': '/wiki', 'wikimedia.org': '/wiki', } def _get_article_tree(self, baseurl, query, use_mw_parsing=True): """ Returns a wiki article tree given the base URL and search query. baseurl can be None, in which case, searching is skipped and the search query will be treated as a raw address. """ if baseurl is None: addr = query else: # Different instances of MediaWiki use different URLs... This tries # to make the parser work for most sites, but still use resonable defaults # such as filling in http:// and appending /wiki to links... baseurl = baseurl.lower() for match, suffix in self.SPECIAL_URLS.items(): if match in baseurl: baseurl += suffix break # Add http:// to the URL if a scheme isn't specified if not baseurl.startswith(('http://', 'https://')): baseurl = 'http://' + baseurl if use_mw_parsing: # first, we get the page addr = '%s/Special:Search?search=%s' % \ (baseurl, quote_plus(query)) else: addr = '%s/%s' % (baseurl, query) self.log.debug('Wikifetch: using URL %s', addr) try: article = utils.web.getUrl(addr, timeout=3) except utils.web.Error: self.log.exception('Failed to fetch link %s', addr) raise article = article.decode() tree = lxml.html.document_fromstring(article) return (tree, article, addr) def _wiki(self, irc, msg, search, baseurl, use_mw_parsing=True): """Fetches and replies content from a MediaWiki-powered website.""" reply = '' # First, fetch and parse the page tree, article, addr = self._get_article_tree( baseurl, search, use_mw_parsing=use_mw_parsing) # check if it gives a "Did you mean..." redirect didyoumean = tree.xpath('//div[@class="searchdidyoumean"]/a' '[@title="Special:Search"]') if didyoumean: redirect = didyoumean[0].text_content().strip() if self.registryValue('showRedirects', msg.args[0]): reply += _('I didn\'t find anything for "%s". ' 'Did you mean "%s"? ') % (search, redirect) tree, article, addr = self._get_article_tree( baseurl, didyoumean[0].get('href')) search = redirect # check if it's a page of search results (rather than an article), and # if so, retrieve the first result searchresults = tree.xpath('//div[@class="searchresults"]/ul/li/a') or \ tree.xpath('//article/ul/li/a') # Special case for Wikia (2017-01-27) self.log.debug('Wikifetch: got search results %s', searchresults) if searchresults: redirect = searchresults[0].text_content().strip() if self.registryValue('showRedirects', msg.args[0]): reply += _('I didn\'t find anything for "%s", but here\'s the ' 'result for "%s": ') % (search, redirect) # Follow the search result and fetch that article. Note: use the original # base url to prevent prefixes like "/wiki" from being added twice. self.log.debug('Wikifetch: following search result:') tree, article, addr = self._get_article_tree( None, searchresults[0].get('href')) search = redirect # otherwise, simply return the title and whether it redirected elif self.registryValue('showRedirects', msg.args[0]): redirect = re.search( r'\(%s <a href=[^>]*>([^<]*)</a>\)' % _('Redirected from'), article) if redirect: try: redirect = tree.xpath( '//span[@class="mw-redirectedfrom"]/a')[0] redirect = redirect.text_content().strip() title = tree.xpath('//*[@class="firstHeading"]') title = title[0].text_content().strip() reply += '"%s" (Redirected from "%s"): ' % (title, redirect) except IndexError: pass # extract the address we got it from - most sites have the perm link # inside the page itself try: addr = tree.find(".//link[@rel='canonical']").attrib['href'] except (ValueError, AttributeError): self.log.debug( 'Wikifetch: failed <link rel="canonical"> link extraction, skipping' ) try: addr = tree.find( ".//div[@class='printfooter']/a").attrib['href'] addr = re.sub(r'([&?]|(amp;)?)oldid=\d+$', '', addr) except (ValueError, AttributeError): self.log.debug( 'Wikifetch: failed printfooter link extraction, skipping') # If any of the above post-processing tricks fail, just ignore pass text_content = tree if use_mw_parsing: text_content = tree.xpath( "//div[@class='mw-parser-output']") or tree.xpath( "//div[@id='mw-content-text']") if text_content: text_content = text_content[0] self.log.debug('Wikifetch: Using %s as text_content', text_content) # check if it's a disambiguation page disambig = tree.xpath('//table[@id="disambigbox"]') or \ tree.xpath('//table[@id="setindexbox"]') or \ tree.xpath('//div[contains(@class, "disambig")]') # Wikia (2017-01-27) if disambig: reply += format(_('%u is a disambiguation page. '), addr) disambig = text_content.xpath('./ul/li') disambig_results = [] for item in disambig: for link in item.findall('a'): if link.text is not None: # Hackishly bold all <a> tags link.text = "%s" % link.text item = item.text_content().replace('', '\x02') # Normalize and strip whitespace, to prevent newlines and such # from corrupting the display. item = utils.str.normalizeWhitespace(item).strip() disambig_results.append(item) if disambig_results: reply += format(_('Possible results include: %s'), '; '.join(disambig_results)) # Catch talk pages elif 'ns-talk' in tree.find("body").attrib.get('class', ''): reply += format(_('This article appears to be a talk page: %u'), addr) else: # Get the first paragraph as text. paragraphs = [] for p in text_content.xpath("./p"): self.log.debug('Wikifetch: looking at paragraph %s', p.text_content()) # Skip geographic coordinates, e.g. on articles for countries if p.xpath(".//span[@class='geo-dec']"): continue # 2018-07-19: some articles have an empty p tag with this class and no content (why?) elif 'mw-empty-elt' in p.attrib.get('class', ''): continue # Skip <p> tags with no content, for obvious reasons elif not p.text_content().strip(): continue paragraphs.append(p) if (not paragraphs) or 'wiki/Special:Search' in addr: if 'wikipedia:wikiproject' in addr.lower(): reply += format( _('This page appears to be a WikiProject page, ' 'but it is too complex for us to parse: %u'), addr) else: irc.error(_('Not found, or page malformed.'), Raise=True) else: p = paragraphs[0] # Replace <b> tags with IRC-style bold, this has to be # done indirectly because unescaped '\x02' is invalid in XML for b_tag in p.xpath('//b'): b_tag.text = "%s" % (b_tag.text or '') p = p.text_content() p = p.replace('', '\x02') # Get rid of newlines, etc., that can corrupt the output. p = utils.str.normalizeWhitespace(p) p = p.strip() if not p: reply = _('<Page was too complex to parse>') reply += format('%s %s %u', p, _('Retrieved from'), addr) reply = reply.replace('&', '&') # Remove inline citations (text[1][2][3]) as well as inline notes (text[note 1]). reply = re.sub(r'\[[a-z ]*?\d+\]', '', reply) return reply @internationalizeDocstring @wrap([ getopts({ 'site': 'somethingWithoutSpaces', 'no-mw-parsing': '' }), 'text' ]) def wiki(self, irc, msg, args, optlist, search): """[--site <site>] [--no-mw-parsing] <search term> Returns the first paragraph of a wiki article. Optionally, a --site argument can be given to override the default (usually Wikipedia) - try using '--site lyrics.wikia.com' or '--site wiki.archlinux.org'. If the --no-mw-parsing option is given, MediaWiki-specific parsing is disabled. This has the following effects: 1) No attempt at searching for a relevant Wiki page is made, and an article with the same name as the search term is directly retrieved. 2) The plugin will retrieve the first <p> tag found on a page, regardless of where it's found, and print it as text. This may not work on all sites, as some use <p> for navbars and headings as well. """ optlist = dict(optlist) baseurl = optlist.get('site') or self.registryValue('url', msg.args[0]) text = self._wiki( irc, msg, search, baseurl, use_mw_parsing=not optlist.get('no-mw-parsing'), ) irc.reply(text) @internationalizeDocstring @wrap([additional('somethingWithoutSpaces')]) def random(self, irc, msg, args, site): """[<site>] Returns the first paragraph of a random wiki article. Optionally, the 'site' argument can be given to override the default (usually Wikipedia).""" baseurl = site or self.registryValue('url', msg.args[0]) text = self._wiki(irc, msg, 'Special:Random', baseurl) irc.reply(text)
class WunderWeather(callbacks.Plugin): """Uses the Wunderground XML API to get weather conditions for a given location. Always gets current conditions, and by default shows a 7-day forecast as well.""" threaded = True ########## GLOBAL VARIABLES ########## _weatherCurrentCondsURL = 'http://api.wunderground.com/auto/wui/geo/WXCurrentObXML/index.xml?query=%s' _weatherForecastURL = 'http://api.wunderground.com/auto/wui/geo/ForecastXML/index.xml?query=%s' ########## EXPOSED METHODS ########## def weather(self, irc, msg, args, options, location): """[--current|--forecast|--all] [US zip code | US/Canada city, state | Foreign city, country] Returns the approximate weather conditions for a given city from Wunderground. --current, --forecast, and --all control what kind of information the command shows. """ matchedLocation = self._commandSetup(irc, msg, location) locationName = self._getNodeValue(matchedLocation[0], 'full', 'Unknown Location') output = [] showCurrent = False showForecast = False if not options: # use default output showCurrent = self.registryValue('showCurrentByDefault', self.__channel) showForecast = self.registryValue('showForecastByDefault', self.__channel) else: for (type, arg) in options: if type == 'current': showCurrent = True elif type == 'forecast': showForecast = True elif type == 'all': showCurrent = True showForecast = True if showCurrent and showForecast: output.append(u('Weather for ') + locationName) elif showCurrent: output.append(u('Current weather for ') + locationName) elif showForecast: output.append(u('Forecast for ') + locationName) if showCurrent: output.append(self._getCurrentConditions(matchedLocation[0])) if showForecast: # _getForecast returns a list, so we have to call extend rather than append output.extend(self._getForecast(matchedLocation[1])) if not showCurrent and not showForecast: irc.error( "Something weird happened... I'm not supposed to show current conditions or a forecast!" ) irc.reply(self._formatUnicodeOutput(output)) weather = wrap(weather, [ getopts({ 'current': '', 'forecast': '', 'all': '' }), additional('text') ]) ########## SUPPORTING METHODS ########## def _checkLocation(self, location): if not location: location = self.userValue('lastLocation', self.__msg.prefix) if not location: raise callbacks.ArgumentError self.setUserValue('lastLocation', self.__msg.prefix, location, ignoreNoUser=True) # Check for shortforms, because Wunderground will attempt to check # for US locations without a full country name. # checkShortforms may return Unicode characters in the country name. # Need Latin 1 for Supybot's URL handlers to work webLocation = shortforms.checkShortforms(location) conditions = self._getDom(self._weatherCurrentCondsURL % utils.web.urlquote(webLocation)) observationLocation = conditions.getElementsByTagName( 'observation_location')[0] # if there's no city name in the XML, we didn't get a match if observationLocation.getElementsByTagName( 'city')[0].childNodes.length < 1: # maybe the country shortform given conflicts with a state shortform and wasn't replaced before webLocation = shortforms.checkConflictingShortforms(location) # if no conflicting short names match, we have the same query as before if webLocation == None: return None conditions = self._getDom(self._weatherCurrentCondsURL % utils.web.urlquote(webLocation)) observationLocation = conditions.getElementsByTagName( 'observation_location')[0] # if there's still no match, nothing more we can do if observationLocation.getElementsByTagName( 'city')[0].childNodes.length < 1: return None # if we get this far, we got a match. Return the DOM and location return (conditions, webLocation) def _commandSetup(self, irc, msg, location): channel = None if irc.isChannel(msg.args[0]): channel = msg.args[0] # set various variables for submethods use self.__irc = irc self.__msg = msg self.__channel = channel matchedLocation = self._checkLocation(location) if not matchedLocation: self._noLocation() return matchedLocation # format temperatures using _formatForMetricOrImperial def _formatCurrentConditionTemperatures(self, dom, string): tempC = self._getNodeValue(dom, string + '_c', u('N/A')) + u('\xb0C') tempF = self._getNodeValue(dom, string + '_f', u('N/A')) + u('\xb0F') return self._formatForMetricOrImperial(tempF, tempC) def _formatForecastTemperatures(self, dom, type): tempC = self._getNodeValue( dom.getElementsByTagName(type)[0], 'celsius', u('N/A')) + u('\xb0C') tempF = self._getNodeValue( dom.getElementsByTagName(type)[0], 'fahrenheit', u('N/A')) + u('\xb0F') return self._formatForMetricOrImperial(tempF, tempC) # formats any imperial or metric values according to the config def _formatForMetricOrImperial(self, imperial, metric): returnValues = [] if self.registryValue('imperial', self.__channel): returnValues.append(imperial) if self.registryValue('metric', self.__channel): returnValues.append(metric) if not returnValues: returnValues = (imperial, metric) return u(' / ').join(returnValues) def _formatPressures(self, dom): # lots of function calls, but it just divides pressure_mb by 10 and rounds it pressureKpa = str( round( float(self._getNodeValue(dom, 'pressure_mb', u('0'))) / 10, 1)) + 'kPa' pressureIn = self._getNodeValue(dom, 'pressure_in', u('0')) + 'in' return self._formatForMetricOrImperial(pressureIn, pressureKpa) def _formatSpeeds(self, dom, string): mphValue = float(self._getNodeValue(dom, string, u('0'))) speedM = u('%dmph') % round(mphValue) speedK = u('%dkph') % round( mphValue * 1.609344) # thanks Wikipedia for the conversion rate return self._formatForMetricOrImperial(speedM, speedK) def _formatUpdatedTime(self, dom): observationTime = self._getNodeValue(dom, 'observation_epoch', None) localTime = self._getNodeValue(dom, 'local_epoch', None) if not observationTime or not localTime: return self._getNodeValue(dom, 'observation_time', 'Unknown Time').lstrip( u('Last Updated on ')) seconds = int(localTime) - int(observationTime) minutes = int(seconds / 60) seconds -= minutes * 60 hours = int(minutes / 60) minutes -= hours * 60 if seconds == 1: seconds = '1 sec' else: seconds = '%d secs' % seconds if minutes == 1: minutes = '1 min' else: minutes = '%d mins' % minutes if hours == 1: hours = '1 hr' else: hours = '%d hrs' % hours if hours == '0 hrs': if minutes == '0 mins': return '%s ago' % seconds return '%s, %s ago' % (minutes, seconds) return '%s, %s, %s ago' % (hours, minutes, seconds) def _getCurrentConditions(self, dom): output = [] temp = self._formatCurrentConditionTemperatures(dom, 'temp') if self._getNodeValue(dom, 'heat_index_string') != 'NA': temp += u(' (Heat Index: %s)' ) % self._formatCurrentConditionTemperatures( dom, 'heat_index') if self._getNodeValue(dom, 'windchill_string') != 'NA': temp += u(' (Wind Chill: %s)' ) % self._formatCurrentConditionTemperatures( dom, 'windchill') output.append(u('Temperature: ') + temp) output.append( u('Humidity: ') + self._getNodeValue(dom, 'relative_humidity', u('N/A%'))) if self.registryValue('showPressure', self.__channel): output.append(u('Pressure: ') + self._formatPressures(dom)) output.append( u('Conditions: ') + self._getNodeValue(dom, 'weather').capitalize()) output.append( u('Wind: ') + self._getNodeValue(dom, 'wind_dir', u('None')).capitalize() + ', ' + self._formatSpeeds(dom, 'wind_mph')) output.append(u('Updated: ') + self._formatUpdatedTime(dom)) return u('; ').join(output) def _getDom(self, url): try: xmlString = utils.web.getUrl(url) return dom.parseString(xmlString) except utils.web.Error as e: error = e.args[0].capitalize() if error[-1] != '.': error = error + '.' self.__irc.error(error, Raise=True) def _getForecast(self, location): dom = self._getDom(self._weatherForecastURL % utils.web.urlquote(location)) output = [] count = 0 max = self.registryValue('forecastDays', self.__channel) forecast = dom.getElementsByTagName('simpleforecast')[0] for day in forecast.getElementsByTagName('forecastday'): if count >= max and max != 0: break forecastOutput = [] forecastOutput.append( 'Forecast for ' + self._getNodeValue(day, 'weekday').capitalize() + ': ' + self._getNodeValue(day, 'conditions').capitalize()) forecastOutput.append( 'High of ' + self._formatForecastTemperatures(day, 'high')) forecastOutput.append('Low of ' + self._formatForecastTemperatures(day, 'low')) output.append('; '.join(forecastOutput)) count += 1 return output ########## STATIC METHODS ########## def _formatUnicodeOutput(output): # UTF-8 encoding is required for Supybot to handle \xb0 (degrees) and other special chars # We can't (yet) pass it a Unicode string on its own (an oddity, to be sure) s = u(' | ').join(output) if sys.version_info[0] < 3: s = s.encode('utf-8') return s _formatUnicodeOutput = staticmethod(_formatUnicodeOutput) def _getNodeValue(dom, value, default=u('Unknown')): subTag = dom.getElementsByTagName(value) if len(subTag) < 1: return default subTag = subTag[0].firstChild if subTag == None: return default return subTag.nodeValue _getNodeValue = staticmethod(_getNodeValue) def _noLocation(): raise NoLocation(noLocationError) _noLocation = staticmethod(_noLocation)
class WeatherBot(callbacks.Plugin): """A weather script that uses Darksky and Weatherstack. """ threaded = True @wrap(["owner"]) def createdb(self, irc: callbacks.NestedCommandsIrcProxy, msg: ircmsgs.IrcMsg, args: List[str]) -> None: """- takes no arguments. Creates a new user table. """ try: result: str = User.create_tables() irc.reply(result, prefixNick=False) except DatabaseError as exc: log.error(str(exc)) irc.reply("There was an error with the database. Check logs.", prefixNick=False) @wrap([getopts({"user": ""}), optional("text")]) def weather( self, irc: callbacks.NestedCommandsIrcProxy, msg: ircmsgs.IrcMsg, args: List[str], optlist: List[Tuple[str, bool]], text: str, ) -> None: """- optional <location> OR [--user] <username> Calls the current weather given an optional arg .e.g. .weather 70119 - If you leave out the location, it will try to use the user's set location that is saved - You can find another user's weather by using the --user flag. e.g. .weather --user Chad """ lookup_user: bool = False for opt, _ in optlist: if opt == "user": lookup_user = True try: optional_user = html.escape(text) if lookup_user and text else msg.nick if lookup_user and not text: irc.reply(f"Please specify the user name.", prefixNick=False) return user: Union[User, AnonymousUser] = get_user(optional_user) if lookup_user: if not isinstance(user, AnonymousUser): weather: str = query_current_weather("", user) irc.reply(weather, prefixNick=False) else: irc.reply(f"No such user by the name of {text}.", prefixNick=False) elif not text and isinstance(user, AnonymousUser): irc.reply(f"No weather location set by {msg.nick}", prefixNick=False) elif not text: weather: str = query_current_weather(text, user) irc.reply(weather, prefixNick=False) else: deserialized_location: Dict[str, str] = UserSchema().load({"location": html.escape(text)}, partial=True) weather: str = query_current_weather(deserialized_location["location"], user) irc.reply(weather, prefixNick=False) except ValidationError as exc: if "location" in exc.messages: message = exc.messages["location"][0] irc.reply(message, prefixNick=False) log.error(str(exc), exc_info=True) except DatabaseError as exc: log.error(str(exc), exc_info=True) if "not created" in str(exc): irc.reply(str(exc), prefixNick=True) else: irc.reply("There is an error. Contact admin.", prefixNick=False) except (LocationNotFound, WeatherNotFound) as exc: irc.reply(str(exc), prefixNick=False) except RequestException as exc: log.error(str(exc), exc_info=True) if exc.response.status_code == 400: irc.reply("Unable to find this location.", prefixNick=False) else: irc.reply("There is an error. Contact admin.", prefixNick=False) @wrap([getopts({"metric": ""}), "text"]) def setweather( self, irc: callbacks.NestedCommandsIrcProxy, msg: ircmsgs.IrcMsg, args: List[str], optlist: List[Tuple[str, bool]], text: str, ) -> None: """[--metric] <location> Sets the weather location for a user. Format is set to show imperial first by default. To show metric first, use the --metric option. e.g. setweather --metric 70118 """ try: format = 1 for option, _ in optlist: if option == "metric": format = 2 deserialized_location: Dict[str, str] = UserSchema().load( {"location": html.escape(text), "format": format}, partial=True, ) geo: Dict[str, str] = query_location(deserialized_location["location"]) geo.update({"nick": msg.nick, "host": f"{msg.user}@{msg.host}", "format": format}) if geo["location"] is None: raise LocationNotFound("Unable to find this location.") user_schema: Dict[str, str] = UserSchema().load(geo) user, created = User.get_or_create( nick=msg.nick, defaults={ "host": user_schema["host"], "format": user_schema["format"], "location": user_schema["location"], "region": user_schema["region"], "coordinates": user_schema["coordinates"], }, ) # If created is a boolean of 0, it means it found a user. # Updates the user fields and saves to db. if not created: user.host = user_schema["host"] user.format = user_schema["format"] user.location = user_schema["location"] user.region = user_schema["region"] user.coordinates = user_schema["coordinates"] user.save() units = "imperial" if format == 1 else "metric" log.info(f"{msg.nick} set their location to {text}") irc.reply(f"{msg.nick} set their weather to {units} first and {text}.", prefixNick=False) except ValidationError as exc: if "location" in exc.messages: message = exc.messages["location"][0] irc.reply(message, prefixNick=False) elif "format" in exc.messages: message = exc.messages["format"][0] irc.reply(message, prefixNick=False) log.error(str(exc), exc_info=True) except DatabaseError as exc: log.error(str(exc), exc_info=True) if "not created" in str(exc): irc.reply(str(exc), prefixNick=True) else: irc.reply("There is an error. Contact admin.", prefixNick=False) except LocationNotFound as exc: irc.reply(str(exc), prefixNick=False) except RequestException as exc: log.error(str(exc), exc_info=True) irc.reply("Unable to find this location.", prefixNick=False)
class Wikifetch(callbacks.Plugin): """Grabs data from Wikipedia and other MediaWiki-powered sites.""" threaded = True # This defines a series of suffixes this should be added after the domain name. SPECIAL_URLS = { 'wikia.com': '/wiki', 'wikipedia.org': '/wiki', 'wiki.archlinux.org': '/index.php', 'wiki.gentoo.org': '/wiki', 'mediawiki.org': '/wiki', 'wikimedia.org': '/wiki', } def _get_article_tree(self, baseurl, search): """ Returns the article tree given the base URL and search query. baseurl can be None, in which case, the search query will be treated as a raw string. """ if baseurl is None: addr = search else: # Different instances of MediaWiki use different URLs... This tries # to make the parser work for most sites, but still use resonable defaults # such as filling in http:// and appending /wiki to links... # Special cases: Wikia, Wikipedia, Wikimedia (i.e. Wikimedia Commons), Arch Linux Wiki if '/' not in search: baseurl = baseurl.lower() for match, suffix in self.SPECIAL_URLS.items(): if match in baseurl: baseurl += suffix break # Add http:// to the URL if a scheme isn't specified if not baseurl.startswith(('http://', 'https://')): baseurl = 'http://' + baseurl # first, we get the page addr = '%s/Special:Search?search=%s' % \ (baseurl, quote_plus(search)) self.log.debug('Wikifetch: using URL %s', addr) try: article = utils.web.getUrl(addr, timeout=3) except utils.web.Error: self.log.exception('Failed to fetch link %s', addr) raise if sys.version_info[0] >= 3: article = article.decode() tree = lxml.html.document_fromstring(article) return (tree, article, addr) def _wiki(self, irc, msg, search, baseurl): """Fetches and replies content from a MediaWiki-powered website.""" reply = '' # First, fetch and parse the page tree, article, addr = self._get_article_tree(baseurl, search) # check if it gives a "Did you mean..." redirect didyoumean = tree.xpath('//div[@class="searchdidyoumean"]/a' '[@title="Special:Search"]') if didyoumean: redirect = didyoumean[0].text_content().strip() if sys.version_info[0] < 3: if isinstance(redirect, unicode): redirect = redirect.encode('utf-8', 'replace') if isinstance(search, unicode): search = search.encode('utf-8', 'replace') if self.registryValue('showRedirects', msg.args[0]): reply += _('I didn\'t find anything for "%s". ' 'Did you mean "%s"? ') % (search, redirect) tree, article, addr = self._get_article_tree( baseurl, didyoumean[0].get('href')) search = redirect # check if it's a page of search results (rather than an article), and # if so, retrieve the first result searchresults = tree.xpath('//div[@class="searchresults"]/ul/li/a') or \ tree.xpath('//article/ul/li/a') # Special case for Wikia (2017-01-27) self.log.debug('Wikifetch: got search results %s', searchresults) if searchresults: redirect = searchresults[0].text_content().strip() if self.registryValue('showRedirects', msg.args[0]): reply += _('I didn\'t find anything for "%s", but here\'s the ' 'result for "%s": ') % (search, redirect) # Follow the search result and fetch that article. Note: use the original # base url to prevent prefixes like "/wiki" from being added twice. self.log.debug('Wikifetch: following search result:') tree, article, addr = self._get_article_tree( None, searchresults[0].get('href')) search = redirect # otherwise, simply return the title and whether it redirected elif self.registryValue('showRedirects', msg.args[0]): redirect = re.search( '\(%s <a href=[^>]*>([^<]*)</a>\)' % _('Redirected from'), article) if redirect: try: redirect = tree.xpath( '//span[@class="mw-redirectedfrom"]/a')[0] redirect = redirect.text_content().strip() title = tree.xpath('//*[@class="firstHeading"]') title = title[0].text_content().strip() if sys.version_info[0] < 3: if isinstance(title, unicode): title = title.encode('utf-8', 'replace') if isinstance(redirect, unicode): redirect = redirect.encode('utf-8', 'replace') reply += '"%s" (Redirected from "%s"): ' % (title, redirect) except IndexError: pass # extract the address we got it from - most sites have the perm link # inside the page itself try: addr = tree.find(".//div[@class='printfooter']/a").attrib['href'] addr = re.sub('([&?]|(amp;)?)oldid=\d+$', '', addr) except: # If any of the above post-processing tricks fail, just ignore pass # check if it's a disambiguation page disambig = tree.xpath('//table[@id="disambigbox"]') or \ tree.xpath('//table[@id="setindexbox"]') or \ tree.xpath('//div[contains(@class, "disambig")]') # Wikia (2017-01-27) if disambig: reply += format(_('%u is a disambiguation page. '), addr) disambig = tree.xpath('//div[@id="bodyContent"]/div/ul/li') disambig_results = [] for item in disambig: for link in item.findall('a'): if link.text is not None: # Hackishly bold all <a> tags link.text = "%s" % link.text item = item.text_content().replace('', '\x02') # Normalize and strip whitespace, to prevent newlines and such # from corrupting the display. item = utils.str.normalizeWhitespace(item).strip() disambig_results.append(item) if disambig_results: reply += format(_('Possible results include: %L'), disambig_results) # Catch talk pages elif 'ns-talk' in tree.find("body").attrib.get('class', ''): reply += format(_('This article appears to be a talk page: %u'), addr) else: p = tree.xpath("//div[@id='mw-content-text']/p[1]") if len(p) == 0 or 'wiki/Special:Search' in addr: if 'wikipedia:wikiproject' in addr.lower(): reply += format( _('This page appears to be a WikiProject page, ' 'but it is too complex for us to parse: %u'), addr) else: irc.error(_('Not found, or page malformed.'), Raise=True) else: p = p[0] # Replace <b> tags with IRC-style bold, this has to be # done indirectly because unescaped '\x02' is invalid in XML for b_tag in p.xpath('//b'): b_tag.text = "%s" % (b_tag.text or '') p = p.text_content() p = p.replace('', '\x02') # Get rid of newlines, etc., that can corrupt the output. p = utils.str.normalizeWhitespace(p) p = p.strip() if sys.version_info[0] < 3: if isinstance(p, unicode): p = p.encode('utf-8', 'replace') if isinstance(reply, unicode): reply = reply.encode('utf-8', 'replace') if not p: reply = _('<Page was too complex to parse>') reply += format('%s %s %u', p, _('Retrieved from'), addr) reply = reply.replace('&', '&') # Remove inline citations (text[1][2][3], etc.) reply = re.sub('\[\d+\]', '', reply) return reply @internationalizeDocstring @wrap([getopts({'site': 'somethingWithoutSpaces'}), 'text']) def wiki(self, irc, msg, args, optlist, search): """[--site <site>] <search term> Returns the first paragraph of a wiki article. Optionally, a --site argument can be given to override the default (usually Wikipedia) - try using '--site lyrics.wikia.com' or '--site wiki.archlinux.org'.""" optlist = dict(optlist) baseurl = optlist.get('site') or self.registryValue('url', msg.args[0]) text = self._wiki(irc, msg, search, baseurl) irc.reply(text) @internationalizeDocstring @wrap([additional('somethingWithoutSpaces')]) def random(self, irc, msg, args, site): """[<site>] Returns the first paragraph of a random wiki article. Optionally, the --site argument can be given to override the default (usually Wikipedia).""" baseurl = site or self.registryValue('url', msg.args[0]) text = self._wiki(irc, msg, 'Special:Random', baseurl) irc.reply(text)