class file(callbacks.Commands):
def plugin(self, irc):
return irc.getCallback('Apt')
@wrap([
getopts({
**FILTERS_OPTLIST,
}),
'something',
])
@add_filters_doc
def packages(self, irc, msg, args, opts, filename):
"""%s <filename>
Lists packages that contain the given filename. %s"""
opts = dict(opts)
plugin = self.plugin(irc)
plugin._get_cache() # open the cache
filename = filename.encode()
# You may want to add '^.*' at the beginning of the pattern; but
# don't do that because the regexp then becomes much slower (6s for
# a 500MB list, instead of 0.3s).
# Given that we're matching end of lines, a line can't be matched
# twice anyway because matches can't overlap.
pattern = re.compile(
rb'%s.*\s+(\S+)/(\S+)$' % re.escape(filename),
re.MULTILINE)
packages = set()
# I can't find a way to do this with python-apt, so let's open and
# parse the files directly
rootdir = self.plugin(irc)._get_cache_dir()
lists = list_content_lists(plugin, irc, msg.channel, opts, rootdir)
with multiprocessing.Pool() as pool:
results = pool.imap_unordered(
search_file,
[(pattern, filename, opener)
for (filename, opener) in lists])
for result in results:
packages.update(result)
if packages:
irc.reply(format('%L', sorted(packages)))
else:
irc.error(_('No package found.'))
class file(callbacks.Commands):
def plugin(self, irc):
return irc.getCallback('Apt')
@wrap([
getopts({
'archs': commalist('something'),
}),
'something',
])
def packages(self, irc, msg, args, opts, filename):
"""<filename>
Lists packages that contain the given filename."""
opts = dict(opts)
plugin = self.plugin(irc)
plugin._get_cache() # open the cache
filename = filename.encode()
# You may want to add '^.*' at the beginning of the pattern; but
# don't do that because the regexp then becomes much slower (6s for
# a 500MB list, instead of 0.3s).
# Given that we're matching end of lines, a line can't be matched
# anyway because matches can't overlap.
line_entry_re = re.compile(
rb'%s.*\s+(\S+)/(\S+)$' % re.escape(filename), re.MULTILINE)
results = set()
# I can't find a way to do this with python-apt, so let's open and
# parse the files directly
rootdir = self.plugin(irc)._get_cache_dir()
lists = list_content_lists(plugin, irc, msg.channel, opts, rootdir)
for (list_filename, file_opener) in lists:
with file_opener(list_filename) as fd:
for match in search_lines(line_entry_re, fd):
results.add(match[1].decode())
if results:
irc.reply(format('%L', sorted(results)))
else:
irc.error(_('No package found.'))
class Owner(callbacks.Plugin):
"""Owner-only commands for core Supybot. This is a core Supybot module
that should not be removed!"""
# This plugin must be first; its priority must be lowest; otherwise odd
# things will happen when adding callbacks.
def __init__(self, irc=None):
if irc is not None:
assert not irc.getCallback(self.name())
self.__parent = super(Owner, self)
self.__parent.__init__(irc)
# Setup command flood detection.
self.commands = ircutils.FloodQueue(
conf.supybot.abuse.flood.interval())
conf.supybot.abuse.flood.interval.addCallback(
self.setFloodQueueTimeout)
# Setup plugins and default plugins for commands.
#
# This needs to be done before we connect to any networks so that the
# children of supybot.plugins (the actual plugins) exist and can be
# loaded.
for (name, s) in registry._cache.items():
if 'alwaysLoadDefault' in name or 'alwaysLoadImportant' in name:
continue
if name.startswith('supybot.plugins'):
try:
(_, _, name) = registry.split(name)
except ValueError: # unpack list of wrong size.
continue
# This is just for the prettiness of the configuration file.
# There are no plugins that are all-lowercase, so we'll at
# least attempt to capitalize them.
if name == name.lower():
name = name.capitalize()
conf.registerPlugin(name)
if name.startswith('supybot.commands.defaultPlugins'):
try:
(_, _, _, name) = registry.split(name)
except ValueError: # unpack list of wrong size.
continue
registerDefaultPlugin(name, s)
# Setup Irc objects, connected to networks. If world.ircs is already
# populated, chances are that we're being reloaded, so don't do this.
if not world.ircs:
for network in conf.supybot.networks():
try:
self._connect(network)
except socket.error as e:
self.log.error('Could not connect to %s: %s.', network, e)
except Exception as e:
self.log.exception('Exception connecting to %s:', network)
self.log.error('Could not connect to %s: %s.', network, e)
def callPrecedence(self, irc):
return ([], [cb for cb in irc.callbacks if cb is not self])
def outFilter(self, irc, msg):
if msg.command == 'PRIVMSG' and not world.testing:
if ircutils.strEqual(msg.args[0], irc.nick):
self.log.warning('Tried to send a message to myself: %r.', msg)
return None
return msg
def reset(self):
# This has to be done somewhere, I figure here is as good place as any.
callbacks.IrcObjectProxy._mores.clear()
self.__parent.reset()
def _connect(self, network, serverPort=None, password='', ssl=False):
try:
group = conf.supybot.networks.get(network)
group.servers()[0]
except (registry.NonExistentRegistryEntry, IndexError):
if serverPort is None:
raise ValueError('connect requires a (server, port) ' \
'if the network is not registered.')
conf.registerNetwork(network, password, ssl)
server = '%s:%s' % serverPort
conf.supybot.networks.get(network).servers.append(server)
assert conf.supybot.networks.get(network).servers(), \
'No servers are set for the %s network.' % network
self.log.debug('Creating new Irc for %s.', network)
newIrc = irclib.Irc(network)
driver = drivers.newDriver(newIrc)
self._loadPlugins(newIrc)
return newIrc
def _loadPlugins(self, irc):
self.log.debug('Loading plugins (connecting to %s).', irc.network)
alwaysLoadImportant = conf.supybot.plugins.alwaysLoadImportant()
important = conf.supybot.commands.defaultPlugins.importantPlugins()
for (name, value) in conf.supybot.plugins.getValues(fullNames=False):
if irc.getCallback(name) is None:
load = value()
if not load and name in important:
if alwaysLoadImportant:
s = '%s is configured not to be loaded, but is being '\
'loaded anyway because ' \
'supybot.plugins.alwaysLoadImportant is True.'
self.log.warning(s, name)
load = True
if load:
# We don't load plugins that don't start with a capital
# letter.
if name[0].isupper() and not irc.getCallback(name):
# This is debug because each log logs its beginning.
self.log.debug('Loading %s.', name)
try:
m = plugin.loadPluginModule(name,
ignoreDeprecation=True)
plugin.loadPluginClass(irc, m)
except callbacks.Error as e:
# This is just an error message.
log.warning(str(e))
except plugins.NoSuitableDatabase as e:
s = 'Failed to load %s: no suitable database(%s).' % (
name, e)
log.warning(s)
except ImportError as e:
e = str(e)
if e.endswith(name):
s = 'Failed to load {0}: No plugin named {0} exists.'.format(
utils.str.dqrepr(name))
elif "No module named 'config'" in e:
s = (
"Failed to load %s: This plugin may be incompatible "
"with your current Python version." % name)
else:
s = 'Failed to load %s: import error (%s).' % (
name, e)
log.warning(s)
except Exception as e:
log.exception('Failed to load %s:', name)
else:
# Let's import the module so configuration is preserved.
try:
_ = plugin.loadPluginModule(name)
except Exception as e:
log.debug(
'Attempted to load %s to preserve its '
'configuration, but load failed: %s', name, e)
world.starting = False
def do376(self, irc, msg):
msgs = conf.supybot.networks.get(irc.network).channels.joins()
if msgs:
for msg in msgs:
irc.queueMsg(msg)
do422 = do377 = do376
def setFloodQueueTimeout(self, *args, **kwargs):
self.commands.timeout = conf.supybot.abuse.flood.interval()
def doBatch(self, irc, msg):
if not conf.supybot.protocols.irc.experimentalExtensions():
return
batch = msg.tagged('batch') # Always not-None on a BATCH message
if msg.args[0].startswith('+'):
# Start of a batch, we're not interested yet.
return
if batch.type != 'draft/multiline':
# This is not a multiline batch, also not interested.
return
assert msg.args[0].startswith("-"), (
"BATCH's first argument should start with either - or +, but "
"it is %s.") % msg.args[0]
# End of multiline batch. It may be a long command.
payloads = []
first_privmsg = None
for message in batch.messages:
if message.command != "PRIVMSG":
# We're only interested in PRIVMSGs for the payloads.
# (eg. exclude NOTICE)
continue
elif not payloads:
# This is the first PRIVMSG of the batch
first_privmsg = message
payloads.append(message.args[1])
elif 'draft/multiline-concat' in message.server_tags:
# This message is not a new line, but the continuation
# of the previous one.
payloads.append(message.args[1])
else:
# New line; stop here. We're not processing extra lines
# either as the rest of the command or as new commands.
# This may change in the future.
break
payload = ''.join(payloads)
if not payload:
self.log.error(
'Got empty multiline payload. This is a bug, please '
'report it along with logs.')
return
assert first_privmsg, "This shouldn't be None unless payload is empty"
# Let's build a synthetic message from the various parts of the
# batch, to look like the multiline batch was a single (large)
# PRIVMSG:
# * copy the tags and server tags of the 'BATCH +' command,
# * copy the prefix and channel of any of the PRIVMSGs
# inside the batch
# * create a new args[1]
target = first_privmsg.args[0]
synthetic_msg = ircmsgs.IrcMsg(
msg=batch.messages[0], # tags, server_tags, time
prefix=first_privmsg.prefix,
command='PRIVMSG',
args=(target, payload))
self._doPrivmsgs(irc, synthetic_msg)
def doPrivmsg(self, irc, msg):
if 'batch' in msg.server_tags:
parent_batches = irc.state.getParentBatches(msg)
parent_batch_types = [batch.type for batch in parent_batches]
if 'draft/multiline' in parent_batch_types \
and conf.supybot.protocols.irc.experimentalExtensions():
# We will handle the message in doBatch when the entire
# batch ends.
return
if 'chathistory' in parent_batch_types:
# Either sent automatically by the server upon join,
# or triggered by a plugin (why?!)
# Either way, replying to commands from the history would
# look weird, because it may have been sent a while ago,
# and we may have already answered to it.
return
self._doPrivmsgs(irc, msg)
def _doPrivmsgs(self, irc, msg):
"""If the given message is a command, triggers Limnoria's
command-dispatching for that command.
Takes the same arguments as ``doPrivmsg`` would, but ``msg`` can
potentially be an artificial message synthesized in doBatch
from a multiline batch.
Usually, a command is a single message, so ``payload=msg.params[0]``
However, when ``msg`` is part of a multiline message, the payload
is the concatenation of multiple messages.
See <https://ircv3.net/specs/extensions/multiline>.
"""
assert self is irc.callbacks[0], \
'Owner isn\'t first callback: %r' % irc.callbacks
if ircmsgs.isCtcp(msg):
return
s = callbacks.addressed(irc, msg)
if s:
ignored = ircdb.checkIgnored(msg.prefix)
if ignored:
self.log.info('Ignoring command from %s.', msg.prefix)
return
maximum = conf.supybot.abuse.flood.command.maximum()
self.commands.enqueue(msg)
if conf.supybot.abuse.flood.command() \
and self.commands.len(msg) > maximum \
and not ircdb.checkCapability(msg.prefix, 'trusted'):
punishment = conf.supybot.abuse.flood.command.punishment()
banmask = conf.supybot.protocols.irc.banmask \
.makeBanmask(msg.prefix)
self.log.info(
'Ignoring %s for %s seconds due to an apparent '
'command flood.', banmask, punishment)
ircdb.ignores.add(banmask, time.time() + punishment)
if conf.supybot.abuse.flood.command.notify():
irc.reply('You\'ve given me %s commands within the last '
'%i seconds; I\'m now ignoring you for %s.' %
(maximum, conf.supybot.abuse.flood.interval(),
utils.timeElapsed(punishment, seconds=False)))
return
try:
tokens = callbacks.tokenize(s,
channel=msg.channel,
network=irc.network)
self.Proxy(irc, msg, tokens)
except SyntaxError as e:
if conf.supybot.reply.error.detailed():
irc.error(str(e))
else:
irc.replyError(msg=msg)
self.log.info('Syntax error: %s', e)
def logmark(self, irc, msg, args, text):
"""<text>
Logs <text> to the global Supybot log at critical priority. Useful for
marking logfiles for later searching.
"""
self.log.critical(text)
irc.replySuccess()
logmark = wrap(logmark, ['text'])
def announce(self, irc, msg, args, text):
"""<text>
Sends <text> to all channels the bot is currently on and not
lobotomized in.
"""
u = ircdb.users.getUser(msg.prefix)
template = self.registryValue('announceFormat')
text = ircutils.standardSubstitute(irc,
msg,
template,
env={
'owner': u.name,
'text': text
})
for channel in irc.state.channels:
c = ircdb.channels.getChannel(channel)
if not c.lobotomized:
irc.queueMsg(ircmsgs.privmsg(channel, text))
irc.noReply()
announce = wrap(announce, ['text'])
def defaultplugin(self, irc, msg, args, optlist, command, plugin):
"""[--remove] <command> [<plugin>]
Sets the default plugin for <command> to <plugin>. If --remove is
given, removes the current default plugin for <command>. If no plugin
is given, returns the current default plugin set for <command>. See
also, supybot.commands.defaultPlugins.importantPlugins.
"""
remove = False
for (option, arg) in optlist:
if option == 'remove':
remove = True
(_, cbs) = irc.findCallbacksForArgs([command])
if remove:
try:
conf.supybot.commands.defaultPlugins.unregister(command)
irc.replySuccess()
except registry.NonExistentRegistryEntry:
s = 'I don\'t have a default plugin set for that command.'
irc.error(s)
elif not cbs:
irc.errorInvalid('command', command)
elif plugin:
if not plugin.isCommand(command):
irc.errorInvalid('command in the %s plugin' % plugin.name(),
command)
registerDefaultPlugin(command, plugin.name())
irc.replySuccess()
else:
try:
irc.reply(conf.supybot.commands.defaultPlugins.get(command)())
except registry.NonExistentRegistryEntry:
s = 'I don\'t have a default plugin set for that command.'
irc.error(s)
defaultplugin = wrap(
defaultplugin,
[getopts({'remove': ''}), 'commandName',
additional('plugin')])
def ircquote(self, irc, msg, args, s):
"""<string to be sent to the server>
Sends the raw string given to the server.
"""
try:
m = ircmsgs.IrcMsg(s)
except Exception as e:
irc.error(utils.exnToString(e))
else:
irc.queueMsg(m)
irc.noReply()
ircquote = wrap(ircquote, ['text'])
def quit(self, irc, msg, args, text):
"""[<text>]
Exits the bot with the QUIT message <text>. If <text> is not given,
the default quit message (supybot.plugins.Owner.quitMsg) will be used.
If there is no default quitMsg set, your nick will be used. The standard
substitutions ($version, $nick, etc.) are all handled appropriately.
"""
text = text or self.registryValue('quitMsg') or msg.nick
text = ircutils.standardSubstitute(irc, msg, text)
irc.noReply()
m = ircmsgs.quit(text)
world.upkeep()
for irc in world.ircs[:]:
irc.queueMsg(m)
irc.die()
quit = wrap(quit, [additional('text')])
def flush(self, irc, msg, args):
"""takes no arguments
Runs all the periodic flushers in world.flushers. This includes
flushing all logs and all configuration changes to disk.
"""
world.flush()
irc.replySuccess()
flush = wrap(flush)
def upkeep(self, irc, msg, args, level):
"""[<level>]
Runs the standard upkeep stuff (flushes and gc.collects()). If given
a level, runs that level of upkeep (currently, the only supported
level is "high", which causes the bot to flush a lot of caches as well
as do normal upkeep stuff).
"""
L = []
if level == 'high':
L.append(
format('Regexp cache flushed: %n cleared.',
(len(re._cache), 'regexp')))
re.purge()
L.append(
format('Pattern cache flushed: %n cleared.',
(len(ircutils._patternCache), 'compiled pattern')))
ircutils._patternCache.clear()
L.append(
format('hostmaskPatternEqual cache flushed: %n cleared.',
(len(ircutils._hostmaskPatternEqualCache), 'result')))
ircutils._hostmaskPatternEqualCache.clear()
L.append(
format(
'ircdb username cache flushed: %n cleared.',
(len(ircdb.users._nameCache), 'username to id mapping')))
ircdb.users._nameCache.clear()
L.append(
format('ircdb hostmask cache flushed: %n cleared.', (len(
ircdb.users._hostmaskCache), 'hostmask to id mapping')))
ircdb.users._hostmaskCache.clear()
L.append(
format('linecache line cache flushed: %n cleared.',
(len(linecache.cache), 'line')))
linecache.clearcache()
if minisix.PY2:
sys.exc_clear()
collected = world.upkeep()
if gc.garbage:
L.append('Garbage! %r.' % gc.garbage)
if collected is not None:
# Some time between 5.2 and 7.1, Pypy (3?) started returning None
# when gc.collect() is called.
L.append(format('%n collected.', (collected, 'object')))
if L:
irc.reply(' '.join(L))
else:
irc.replySuccess()
upkeep = wrap(upkeep, [additional(('literal', ['high']))])
def load(self, irc, msg, args, optlist, name):
"""[--deprecated] <plugin>
Loads the plugin <plugin> from any of the directories in
conf.supybot.directories.plugins; usually this includes the main
installed directory and 'plugins' in the current directory.
--deprecated is necessary if you wish to load deprecated plugins.
"""
ignoreDeprecation = False
for (option, argument) in optlist:
if option == 'deprecated':
ignoreDeprecation = True
if name.endswith('.py'):
name = name[:-3]
if irc.getCallback(name):
irc.error('%s is already loaded.' % name.capitalize())
return
try:
module = plugin.loadPluginModule(name, ignoreDeprecation)
except plugin.Deprecated:
irc.error('%s is deprecated. Use --deprecated '
'to force it to load.' % name.capitalize())
return
except ImportError as e:
if str(e).endswith(name):
irc.error('No plugin named %s exists.' %
utils.str.dqrepr(name))
elif "No module named 'config'" in str(e):
irc.error(
'This plugin may be incompatible with your current Python '
'version. Try running 2to3 on it.')
else:
irc.error(str(e))
return
cb = plugin.loadPluginClass(irc, module)
name = cb.name() # Let's normalize this.
conf.registerPlugin(name, True)
irc.replySuccess()
load = wrap(load, [getopts({'deprecated': ''}), 'something'])
def reload(self, irc, msg, args, name):
"""<plugin>
Unloads and subsequently reloads the plugin by name; use the 'list'
command to see a list of the currently loaded plugins.
"""
if ircutils.strEqual(name, self.name()):
irc.error('You can\'t reload the %s plugin.' % name)
return
callbacks = irc.removeCallback(name)
if callbacks:
module = sys.modules[callbacks[0].__module__]
if hasattr(module, 'reload'):
x = module.reload()
try:
module = plugin.loadPluginModule(name)
if hasattr(module, 'reload') and 'x' in locals():
module.reload(x)
if hasattr(module, 'config'):
from importlib import reload
reload(module.config)
for callback in callbacks:
callback.die()
del callback
gc.collect() # This makes sure the callback is collected.
callback = plugin.loadPluginClass(irc, module)
irc.replySuccess()
except ImportError:
for callback in callbacks:
irc.addCallback(callback)
irc.error('No plugin named %s exists.' % name)
else:
irc.error('There was no plugin %s.' % name)
reload = wrap(reload, ['something'])
def unload(self, irc, msg, args, name):
"""<plugin>
Unloads the callback by name; use the 'list' command to see a list
of the currently loaded plugins. Obviously, the Owner plugin can't
be unloaded.
"""
if ircutils.strEqual(name, self.name()):
irc.error('You can\'t unload the %s plugin.' % name)
return
# Let's do this so even if the plugin isn't currently loaded, it doesn't
# stay attempting to load.
old_callback = irc.getCallback(name)
if old_callback:
# Normalize the plugin case to prevent duplicate registration
# entries, https://github.com/progval/Limnoria/issues/1295
name = old_callback.name()
conf.registerPlugin(name, False)
callbacks = irc.removeCallback(name)
if callbacks:
for callback in callbacks:
callback.die()
del callback
gc.collect()
irc.replySuccess()
return
irc.error('There was no plugin %s.' % name)
unload = wrap(unload, ['something'])
def defaultcapability(self, irc, msg, args, action, capability):
"""{add|remove} <capability>
Adds or removes (according to the first argument) <capability> from the
default capabilities given to users (the configuration variable
supybot.capabilities stores these).
"""
if action == 'add':
conf.supybot.capabilities().add(capability)
irc.replySuccess()
elif action == 'remove':
try:
conf.supybot.capabilities().remove(capability)
irc.replySuccess()
except KeyError:
if ircdb.isAntiCapability(capability):
irc.error('That capability wasn\'t in '
'supybot.capabilities.')
else:
anticap = ircdb.makeAntiCapability(capability)
conf.supybot.capabilities().add(anticap)
irc.replySuccess()
defaultcapability = wrap(defaultcapability,
[('literal', ['add', 'remove']), 'capability'])
def disable(self, irc, msg, args, plugin, command):
"""[<plugin>] <command>
Disables the command <command> for all users (including the owners).
If <plugin> is given, only disables the <command> from <plugin>. If
you want to disable a command for most users but not for yourself, set
a default capability of -plugin.command or -command (if you want to
disable the command in all plugins).
"""
if command in ('enable', 'identify'):
irc.error('You can\'t disable %s.' % command)
return
if plugin:
if plugin.isCommand(command):
pluginCommand = '%s.%s' % (plugin.name(), command)
conf.supybot.commands.disabled().add(pluginCommand)
plugin._disabled.add(command, plugin.name())
else:
irc.error('%s is not a command in the %s plugin.' %
(command, plugin.name()))
return
else:
conf.supybot.commands.disabled().add(command)
self._disabled.add(command)
irc.replySuccess()
disable = wrap(disable, [optional('plugin'), 'commandName'])
def enable(self, irc, msg, args, plugin, command):
"""[<plugin>] <command>
Enables the command <command> for all users. If <plugin>
if given, only enables the <command> from <plugin>. This command is
the inverse of disable.
"""
try:
if plugin:
plugin._disabled.remove(command, plugin.name())
command = '%s.%s' % (plugin.name(), command)
else:
self._disabled.remove(command)
conf.supybot.commands.disabled().remove(command)
irc.replySuccess()
except KeyError:
irc.error('That command wasn\'t disabled.')
enable = wrap(enable, [optional('plugin'), 'commandName'])
def rename(self, irc, msg, args, command_plugin, command, newName):
"""<plugin> <command> <new name>
Renames <command> in <plugin> to the <new name>.
"""
if not command_plugin.isCommand(command):
what = 'command in the %s plugin' % command_plugin.name()
irc.errorInvalid(what, command)
if hasattr(command_plugin, newName):
irc.error('The %s plugin already has an attribute named %s.' %
(command_plugin, newName))
return
plugin.registerRename(command_plugin.name(), command, newName)
plugin.renameCommand(command_plugin, command, newName)
irc.replySuccess()
rename = wrap(rename, ['plugin', 'commandName', 'commandName'])
def unrename(self, irc, msg, args, plugin):
"""<plugin>
Removes all renames in <plugin>. The plugin will be reloaded after
this command is run.
"""
try:
conf.supybot.commands.renames.unregister(plugin.name())
except registry.NonExistentRegistryEntry:
irc.errorInvalid('plugin', plugin.name())
self.reload(irc, msg, [plugin.name()]) # This makes the replySuccess.
unrename = wrap(unrename, ['plugin'])
def reloadlocale(self, irc, msg, args):
"""takes no argument
Reloads the locale of the bot."""
i18n.reloadLocales()
irc.replySuccess()
class Wikifetch(callbacks.Plugin):
"""Grabs data from Wikipedia and other MediaWiki-powered sites."""
threaded = True
def _wiki(self, irc, msg, search, baseurl):
"""Fetches and replies content from a MediaWiki-powered website."""
reply = ''
# Different instances of MediaWiki use different URLs... This tries
# to make the parser work for most sites, but still use resonable defaults
# such as filling in http:// and appending /wiki to links...
# Special cases: Wikia, Wikipedia, Wikimedia (i.e. Wikimedia Commons), Arch Linux Wiki
if any(sitename in baseurl
for sitename in ('wikia.com', 'wikipedia.org',
'wikimedia.org')):
baseurl += '/wiki'
elif 'wiki.archlinux.org' in baseurl:
baseurl += '/index.php'
if not baseurl.lower().startswith(('http://', 'https://')):
baseurl = 'http://' + baseurl
# first, we get the page
addr = '%s/Special:Search?search=%s' % \
(baseurl, quote_plus(search))
self.log.debug('Wikifetch: using URL %s', addr)
article = utils.web.getUrl(addr)
if sys.version_info[0] >= 3:
article = article.decode()
# parse the page
tree = lxml.html.document_fromstring(article)
# check if it gives a "Did you mean..." redirect
didyoumean = tree.xpath('//div[@class="searchdidyoumean"]/a'
'[@title="Special:Search"]')
if didyoumean:
redirect = didyoumean[0].text_content().strip()
if sys.version_info[0] < 3:
if isinstance(redirect, unicode):
redirect = redirect.encode('utf-8', 'replace')
if isinstance(search, unicode):
search = search.encode('utf-8', 'replace')
if self.registryValue('showRedirects', msg.args[0]):
reply += _('I didn\'t find anything for "%s". '
'Did you mean "%s"? ') % (search, redirect)
addr = "%s/%s" % (baseurl, didyoumean[0].get('href'))
article = utils.web.getUrl(addr)
if sys.version_info[0] >= 3:
article = article.decode()
tree = lxml.html.document_fromstring(article)
search = redirect
# check if it's a page of search results (rather than an article), and
# if so, retrieve the first result
searchresults = tree.xpath('//div[@class="searchresults"]/ul/li/a')
if searchresults:
redirect = searchresults[0].text_content().strip()
if self.registryValue('showRedirects', msg.args[0]):
reply += _('I didn\'t find anything for "%s", but here\'s the '
'result for "%s": ') % (search, redirect)
addr = self.registryValue('url', msg.args[0]) + \
searchresults[0].get('href')
article = utils.web.getUrl(addr)
if sys.version_info[0] >= 3:
article = article.decode()
tree = lxml.html.document_fromstring(article)
search = redirect
# otherwise, simply return the title and whether it redirected
elif self.registryValue('showRedirects', msg.args[0]):
redirect = re.search(
'\(%s <a href=[^>]*>([^<]*)</a>\)' % _('Redirected from'),
article)
if redirect:
try:
redirect = tree.xpath(
'//span[@class="mw-redirectedfrom"]/a')[0]
redirect = redirect.text_content().strip()
title = tree.xpath('//*[@class="firstHeading"]')
title = title[0].text_content().strip()
if sys.version_info[0] < 3:
if isinstance(title, unicode):
title = title.encode('utf-8', 'replace')
if isinstance(redirect, unicode):
redirect = redirect.encode('utf-8', 'replace')
reply += '"%s" (Redirected from "%s"): ' % (title,
redirect)
except IndexError:
pass
# extract the address we got it from - most sites have the perm link
# inside the page itself
try:
addr = tree.find(".//div[@class='printfooter']/a").attrib['href']
addr = re.sub('([&?]|(amp;)?)oldid=\d+$', '', addr)
except:
pass
# check if it's a disambiguation page
disambig = tree.xpath('//table[@id="disambigbox"]') or \
tree.xpath('//table[@id="setindexbox"]')
if disambig:
disambig = tree.xpath('//div[@id="bodyContent"]/div/ul/li')
# Hackishly bold all <a> tags
r = []
for item in disambig:
for link in item.findall('a'):
if link.text is not None:
link.text = "%s" % link.text
item = item.text_content().replace('', '\x02')
# Normalize and strip whitespace, to prevent newlines and such
# from corrupting the display.
item = utils.str.normalizeWhitespace(item).strip()
r.append(item)
reply += format(
_('%u is a disambiguation page. '
'Possible results include: %L'), addr, r)
# or just as bad, a page listing events in that year
elif re.search(
_('This article is about the year [\d]*\. '
'For the [a-zA-Z ]* [\d]*, see'), article):
reply += _('"%s" is a page full of events that happened in that '
'year. If you were looking for information about the '
'number itself, try searching for "%s_(number)", but '
'don\'t expect anything useful...') % (search, search)
# Catch talk pages
elif 'ns-talk' in tree.find("body").attrib['class']:
reply += format(_('This article appears to be a talk page: %u'),
addr)
else:
p = tree.xpath("//div[@id='mw-content-text']/p[1]")
if len(p) == 0 or 'wiki/Special:Search' in addr:
if 'wikipedia:wikiproject' in addr.lower():
reply += format(
_('This page appears to be a WikiProject page, '
'but it is too complex for us to parse: %u'), addr)
else:
irc.error(_('Not found, or page malformed.'), Raise=True)
else:
p = p[0]
# Replace <b> tags with IRC-style bold, this has to be
# done indirectly because unescaped '\x02' is invalid in XML
for b_tag in p.xpath('//b'):
b_tag.text = "%s" % (b_tag.text or '')
p = p.text_content()
p = p.replace('', '\x02')
# Get rid of newlines, etc., that can corrupt the output.
p = utils.str.normalizeWhitespace(p)
p = p.strip()
if sys.version_info[0] < 3:
if isinstance(p, unicode):
p = p.encode('utf-8', 'replace')
if isinstance(reply, unicode):
reply = reply.encode('utf-8', 'replace')
reply += format('%s %s %u', p, _('Retrieved from'), addr)
reply = reply.replace('&', '&')
# Remove inline citations (text[1][2][3], etc.)
reply = re.sub('\[\d+\]', '', reply)
return reply
@internationalizeDocstring
@wrap([getopts({'site': 'somethingWithoutSpaces'}), 'text'])
def wiki(self, irc, msg, args, optlist, search):
"""[--site <site>] <search term>
Returns the first paragraph of a wiki article. Optionally, a --site
argument can be given to override the default (usually Wikipedia) -
try using '--site lyrics.wikia.com' or '--site wiki.archlinux.org'."""
optlist = dict(optlist)
baseurl = optlist.get('site') or self.registryValue('url', msg.args[0])
text = self._wiki(irc, msg, search, baseurl)
irc.reply(text)
@internationalizeDocstring
@wrap([additional('somethingWithoutSpaces')])
def random(self, irc, msg, args, site):
"""[<site>]
Returns the first paragraph of a random wiki article. Optionally, the --site
argument can be given to override the default (usually Wikipedia)."""
baseurl = site or self.registryValue('url', msg.args[0])
text = self._wiki(irc, msg, 'Special:Random', baseurl)
irc.reply(text)
class package(callbacks.Commands):
def plugin(self, irc):
return irc.getCallback('Apt')
def _get_package(self, irc, package_name):
# TODO: add support for selecting the version
cache = self.plugin(irc)._get_cache()
pkg = cache.get(package_name)
if not pkg:
irc.error(_('Package not found.'), Raise=True)
return pkg
@wrap([
getopts({
**FILTERS_OPTLIST,
'types':
commalist('somethingWithoutSpaces'),
}),
'somethingWithoutSpaces',
])
@add_filters_doc
def depends(self, irc, msg, args, optlist, package_name):
"""%s [--types <type>,<type>,...] <package>
Lists dependencies of a package. <type>s are types of dependencies
that will be shown. Valid types are: Breaks, Conflicts, Depends,
Enhances, PreDepends, Recommends, Replaces, Suggests. %s
"""
opts = dict(optlist)
pkg = self._get_package(irc, package_name)
dep_types = opts.get('types', ['Depends', 'PreDepends'])
dep_types = [dep_type for dep_type in dep_types]
translations = get_dependency_reverse_translations()
dep_types = [
translations.get(dep_type, dep_type) for dep_type in dep_types
]
# TODO: better version selection
pkg_version = filter_versions(self.plugin(irc), irc, msg.channel,
opts, pkg.versions)[0]
deps = pkg_version.get_dependencies(*dep_types)
irc.reply(
format(_('%s: %L'), pkg_version, [dep.rawstr for dep in deps]))
@wrap([getopts({
**FILTERS_OPTLIST,
}), 'somethingWithoutSpaces'])
@add_filters_doc
def description(self, irc, msg, args, optlist, package_name):
"""%s <package>
Shows the long description of a package. %s"""
opts = dict(optlist)
pkg = self._get_package(irc, package_name)
# TODO: better version selection
pkg_version = filter_versions(self.plugin(irc), irc, msg.channel,
opts, pkg.versions)[0]
description = utils.str.normalizeWhitespace(
pkg_version.description)
irc.reply(
format(_('%s %s: %s'), pkg.shortname, pkg_version.version,
description))
@wrap([getopts({
**FILTERS_OPTLIST,
}), 'somethingWithoutSpaces'])
@add_filters_doc
def info(self, irc, msg, args, optlist, package_name):
"""%s <package>
Shows generic information about a package. %s"""
opts = dict(optlist)
pkg = self._get_package(irc, package_name)
# TODO: better version selection
pkg_version = filter_versions(self.plugin(irc), irc, msg.channel,
opts, pkg.versions)[0]
# source_name and priority shouldn't change too often, so I assume
# it's safe to call it a "generic info" in a UI
priority = get_priority_translations().get(pkg_version.priority,
pkg_version.priority)
generic_info = format(
_('%s (source: %s) is %s and in section "%s".'), pkg.shortname,
pkg_version.source_name, priority, pkg.section)
version_info = format(
_('Version %s package is %S and takes %S when installed.'),
pkg_version.version, pkg_version.size,
pkg_version.installed_size)
irc.reply(
format(_('%s %s Description: %s'), generic_info, version_info,
pkg_version.summary))
@wrap([
getopts({
**FILTERS_OPTLIST,
'with-version': '',
'description': '',
}), 'somethingWithoutSpaces'
])
@add_filters_doc
def search(self, irc, msg, args, optlist, package_pattern):
"""%s [--with-version] [--description] <package>
Shows generic information about a package. --with-version also
returns matching version numbers. --description searches in package
description instead of name. %s"""
opts = dict(optlist)
search_description = 'description' in opts
cache = self.plugin(irc)._get_cache()
pattern = re.compile(utils.python.glob2re(package_pattern),
re.DOTALL)
packages = iter(cache)
if not search_description:
packages = (pkg for pkg in cache
if pattern.match(pkg.shortname))
# Dirty trick to check if the iterator is empty without
# consuming it entirely
(packages, peek_packages) = itertools.tee(packages)
if not next(peek_packages, None):
irc.error(_('No package found.'), Raise=True)
versions = (version for pkg in packages
for version in pkg.versions)
if search_description:
versions = (version for version in versions
if pattern.match(version.description))
# filter_versions is rather slow, so we're putting this guard
# before running it.
versions = list(itertools.islice(versions, 10000))
if len(versions) >= 10000:
irc.error(_('Too many packages match this search.'),
Raise=True)
versions = filter_versions(self.plugin(irc), irc, msg.channel,
opts, versions)
if not opts.get('with-version'):
package_names = sorted(
{version.package.shortname
for version in versions})
irc.reply(format('%L', package_names))
return
items = []
for version in versions:
items.append(
format('%s %s (in %L)', version.package.shortname,
version.version,
{origin.codename
for origin in version.origins}))
irc.reply(format('%L', items))
class Wikifetch(callbacks.Plugin):
"""Grabs data from Wikipedia and other MediaWiki-powered sites."""
threaded = True
# This defines a series of suffixes this should be added after the domain name.
SPECIAL_URLS = {
'wikia.com': '/wiki',
'wikipedia.org': '/wiki',
'wiki.archlinux.org': '/index.php',
'wiki.gentoo.org': '/wiki',
'mediawiki.org': '/wiki',
'wikimedia.org': '/wiki',
}
def _get_article_tree(self, baseurl, query, use_mw_parsing=True):
"""
Returns a wiki article tree given the base URL and search query. baseurl can be None,
in which case, searching is skipped and the search query will be treated as a raw address.
"""
if baseurl is None:
addr = query
else:
# Different instances of MediaWiki use different URLs... This tries
# to make the parser work for most sites, but still use resonable defaults
# such as filling in http:// and appending /wiki to links...
baseurl = baseurl.lower()
for match, suffix in self.SPECIAL_URLS.items():
if match in baseurl:
baseurl += suffix
break
# Add http:// to the URL if a scheme isn't specified
if not baseurl.startswith(('http://', 'https://')):
baseurl = 'http://' + baseurl
if use_mw_parsing:
# first, we get the page
addr = '%s/Special:Search?search=%s' % \
(baseurl, quote_plus(query))
else:
addr = '%s/%s' % (baseurl, query)
self.log.debug('Wikifetch: using URL %s', addr)
try:
article = utils.web.getUrl(addr, timeout=3)
except utils.web.Error:
self.log.exception('Failed to fetch link %s', addr)
raise
article = article.decode()
tree = lxml.html.document_fromstring(article)
return (tree, article, addr)
def _wiki(self, irc, msg, search, baseurl, use_mw_parsing=True):
"""Fetches and replies content from a MediaWiki-powered website."""
reply = ''
# First, fetch and parse the page
tree, article, addr = self._get_article_tree(
baseurl, search, use_mw_parsing=use_mw_parsing)
# check if it gives a "Did you mean..." redirect
didyoumean = tree.xpath('//div[@class="searchdidyoumean"]/a'
'[@title="Special:Search"]')
if didyoumean:
redirect = didyoumean[0].text_content().strip()
if self.registryValue('showRedirects', msg.args[0]):
reply += _('I didn\'t find anything for "%s". '
'Did you mean "%s"? ') % (search, redirect)
tree, article, addr = self._get_article_tree(
baseurl, didyoumean[0].get('href'))
search = redirect
# check if it's a page of search results (rather than an article), and
# if so, retrieve the first result
searchresults = tree.xpath('//div[@class="searchresults"]/ul/li/a') or \
tree.xpath('//article/ul/li/a') # Special case for Wikia (2017-01-27)
self.log.debug('Wikifetch: got search results %s', searchresults)
if searchresults:
redirect = searchresults[0].text_content().strip()
if self.registryValue('showRedirects', msg.args[0]):
reply += _('I didn\'t find anything for "%s", but here\'s the '
'result for "%s": ') % (search, redirect)
# Follow the search result and fetch that article. Note: use the original
# base url to prevent prefixes like "/wiki" from being added twice.
self.log.debug('Wikifetch: following search result:')
tree, article, addr = self._get_article_tree(
None, searchresults[0].get('href'))
search = redirect
# otherwise, simply return the title and whether it redirected
elif self.registryValue('showRedirects', msg.args[0]):
redirect = re.search(
r'\(%s <a href=[^>]*>([^<]*)</a>\)' % _('Redirected from'),
article)
if redirect:
try:
redirect = tree.xpath(
'//span[@class="mw-redirectedfrom"]/a')[0]
redirect = redirect.text_content().strip()
title = tree.xpath('//*[@class="firstHeading"]')
title = title[0].text_content().strip()
reply += '"%s" (Redirected from "%s"): ' % (title,
redirect)
except IndexError:
pass
# extract the address we got it from - most sites have the perm link
# inside the page itself
try:
addr = tree.find(".//link[@rel='canonical']").attrib['href']
except (ValueError, AttributeError):
self.log.debug(
'Wikifetch: failed <link rel="canonical"> link extraction, skipping'
)
try:
addr = tree.find(
".//div[@class='printfooter']/a").attrib['href']
addr = re.sub(r'([&?]|(amp;)?)oldid=\d+$', '', addr)
except (ValueError, AttributeError):
self.log.debug(
'Wikifetch: failed printfooter link extraction, skipping')
# If any of the above post-processing tricks fail, just ignore
pass
text_content = tree
if use_mw_parsing:
text_content = tree.xpath(
"//div[@class='mw-parser-output']") or tree.xpath(
"//div[@id='mw-content-text']")
if text_content:
text_content = text_content[0]
self.log.debug('Wikifetch: Using %s as text_content', text_content)
# check if it's a disambiguation page
disambig = tree.xpath('//table[@id="disambigbox"]') or \
tree.xpath('//table[@id="setindexbox"]') or \
tree.xpath('//div[contains(@class, "disambig")]') # Wikia (2017-01-27)
if disambig:
reply += format(_('%u is a disambiguation page. '), addr)
disambig = text_content.xpath('./ul/li')
disambig_results = []
for item in disambig:
for link in item.findall('a'):
if link.text is not None:
# Hackishly bold all <a> tags
link.text = "%s" % link.text
item = item.text_content().replace('', '\x02')
# Normalize and strip whitespace, to prevent newlines and such
# from corrupting the display.
item = utils.str.normalizeWhitespace(item).strip()
disambig_results.append(item)
if disambig_results:
reply += format(_('Possible results include: %s'),
'; '.join(disambig_results))
# Catch talk pages
elif 'ns-talk' in tree.find("body").attrib.get('class', ''):
reply += format(_('This article appears to be a talk page: %u'),
addr)
else:
# Get the first paragraph as text.
paragraphs = []
for p in text_content.xpath("./p"):
self.log.debug('Wikifetch: looking at paragraph %s',
p.text_content())
# Skip geographic coordinates, e.g. on articles for countries
if p.xpath(".//span[@class='geo-dec']"):
continue
# 2018-07-19: some articles have an empty p tag with this class and no content (why?)
elif 'mw-empty-elt' in p.attrib.get('class', ''):
continue
# Skip <p> tags with no content, for obvious reasons
elif not p.text_content().strip():
continue
paragraphs.append(p)
if (not paragraphs) or 'wiki/Special:Search' in addr:
if 'wikipedia:wikiproject' in addr.lower():
reply += format(
_('This page appears to be a WikiProject page, '
'but it is too complex for us to parse: %u'), addr)
else:
irc.error(_('Not found, or page malformed.'), Raise=True)
else:
p = paragraphs[0]
# Replace <b> tags with IRC-style bold, this has to be
# done indirectly because unescaped '\x02' is invalid in XML
for b_tag in p.xpath('//b'):
b_tag.text = "%s" % (b_tag.text or '')
p = p.text_content()
p = p.replace('', '\x02')
# Get rid of newlines, etc., that can corrupt the output.
p = utils.str.normalizeWhitespace(p)
p = p.strip()
if not p:
reply = _('<Page was too complex to parse>')
reply += format('%s %s %u', p, _('Retrieved from'), addr)
reply = reply.replace('&', '&')
# Remove inline citations (text[1][2][3]) as well as inline notes (text[note 1]).
reply = re.sub(r'\[[a-z ]*?\d+\]', '', reply)
return reply
@internationalizeDocstring
@wrap([
getopts({
'site': 'somethingWithoutSpaces',
'no-mw-parsing': ''
}), 'text'
])
def wiki(self, irc, msg, args, optlist, search):
"""[--site <site>] [--no-mw-parsing] <search term>
Returns the first paragraph of a wiki article. Optionally, a --site
argument can be given to override the default (usually Wikipedia) -
try using '--site lyrics.wikia.com' or '--site wiki.archlinux.org'.
If the --no-mw-parsing option is given, MediaWiki-specific parsing is
disabled. This has the following effects:
1) No attempt at searching for a relevant Wiki page is made, and
an article with the same name as the search term is directly
retrieved.
2) The plugin will retrieve the first <p> tag found on a page,
regardless of where it's found, and print it as text. This may
not work on all sites, as some use <p> for navbars and headings
as well.
"""
optlist = dict(optlist)
baseurl = optlist.get('site') or self.registryValue('url', msg.args[0])
text = self._wiki(
irc,
msg,
search,
baseurl,
use_mw_parsing=not optlist.get('no-mw-parsing'),
)
irc.reply(text)
@internationalizeDocstring
@wrap([additional('somethingWithoutSpaces')])
def random(self, irc, msg, args, site):
"""[<site>]
Returns the first paragraph of a random wiki article. Optionally, the 'site'
argument can be given to override the default (usually Wikipedia)."""
baseurl = site or self.registryValue('url', msg.args[0])
text = self._wiki(irc, msg, 'Special:Random', baseurl)
irc.reply(text)
class WunderWeather(callbacks.Plugin):
"""Uses the Wunderground XML API to get weather conditions for a given location.
Always gets current conditions, and by default shows a 7-day forecast as well."""
threaded = True
########## GLOBAL VARIABLES ##########
_weatherCurrentCondsURL = 'http://api.wunderground.com/auto/wui/geo/WXCurrentObXML/index.xml?query=%s'
_weatherForecastURL = 'http://api.wunderground.com/auto/wui/geo/ForecastXML/index.xml?query=%s'
########## EXPOSED METHODS ##########
def weather(self, irc, msg, args, options, location):
"""[--current|--forecast|--all] [US zip code | US/Canada city, state | Foreign city, country]
Returns the approximate weather conditions for a given city from Wunderground.
--current, --forecast, and --all control what kind of information the command
shows.
"""
matchedLocation = self._commandSetup(irc, msg, location)
locationName = self._getNodeValue(matchedLocation[0], 'full',
'Unknown Location')
output = []
showCurrent = False
showForecast = False
if not options:
# use default output
showCurrent = self.registryValue('showCurrentByDefault',
self.__channel)
showForecast = self.registryValue('showForecastByDefault',
self.__channel)
else:
for (type, arg) in options:
if type == 'current':
showCurrent = True
elif type == 'forecast':
showForecast = True
elif type == 'all':
showCurrent = True
showForecast = True
if showCurrent and showForecast:
output.append(u('Weather for ') + locationName)
elif showCurrent:
output.append(u('Current weather for ') + locationName)
elif showForecast:
output.append(u('Forecast for ') + locationName)
if showCurrent:
output.append(self._getCurrentConditions(matchedLocation[0]))
if showForecast:
# _getForecast returns a list, so we have to call extend rather than append
output.extend(self._getForecast(matchedLocation[1]))
if not showCurrent and not showForecast:
irc.error(
"Something weird happened... I'm not supposed to show current conditions or a forecast!"
)
irc.reply(self._formatUnicodeOutput(output))
weather = wrap(weather, [
getopts({
'current': '',
'forecast': '',
'all': ''
}),
additional('text')
])
########## SUPPORTING METHODS ##########
def _checkLocation(self, location):
if not location:
location = self.userValue('lastLocation', self.__msg.prefix)
if not location:
raise callbacks.ArgumentError
self.setUserValue('lastLocation',
self.__msg.prefix,
location,
ignoreNoUser=True)
# Check for shortforms, because Wunderground will attempt to check
# for US locations without a full country name.
# checkShortforms may return Unicode characters in the country name.
# Need Latin 1 for Supybot's URL handlers to work
webLocation = shortforms.checkShortforms(location)
conditions = self._getDom(self._weatherCurrentCondsURL %
utils.web.urlquote(webLocation))
observationLocation = conditions.getElementsByTagName(
'observation_location')[0]
# if there's no city name in the XML, we didn't get a match
if observationLocation.getElementsByTagName(
'city')[0].childNodes.length < 1:
# maybe the country shortform given conflicts with a state shortform and wasn't replaced before
webLocation = shortforms.checkConflictingShortforms(location)
# if no conflicting short names match, we have the same query as before
if webLocation == None:
return None
conditions = self._getDom(self._weatherCurrentCondsURL %
utils.web.urlquote(webLocation))
observationLocation = conditions.getElementsByTagName(
'observation_location')[0]
# if there's still no match, nothing more we can do
if observationLocation.getElementsByTagName(
'city')[0].childNodes.length < 1:
return None
# if we get this far, we got a match. Return the DOM and location
return (conditions, webLocation)
def _commandSetup(self, irc, msg, location):
channel = None
if irc.isChannel(msg.args[0]):
channel = msg.args[0]
# set various variables for submethods use
self.__irc = irc
self.__msg = msg
self.__channel = channel
matchedLocation = self._checkLocation(location)
if not matchedLocation:
self._noLocation()
return matchedLocation
# format temperatures using _formatForMetricOrImperial
def _formatCurrentConditionTemperatures(self, dom, string):
tempC = self._getNodeValue(dom, string + '_c', u('N/A')) + u('\xb0C')
tempF = self._getNodeValue(dom, string + '_f', u('N/A')) + u('\xb0F')
return self._formatForMetricOrImperial(tempF, tempC)
def _formatForecastTemperatures(self, dom, type):
tempC = self._getNodeValue(
dom.getElementsByTagName(type)[0], 'celsius',
u('N/A')) + u('\xb0C')
tempF = self._getNodeValue(
dom.getElementsByTagName(type)[0], 'fahrenheit',
u('N/A')) + u('\xb0F')
return self._formatForMetricOrImperial(tempF, tempC)
# formats any imperial or metric values according to the config
def _formatForMetricOrImperial(self, imperial, metric):
returnValues = []
if self.registryValue('imperial', self.__channel):
returnValues.append(imperial)
if self.registryValue('metric', self.__channel):
returnValues.append(metric)
if not returnValues:
returnValues = (imperial, metric)
return u(' / ').join(returnValues)
def _formatPressures(self, dom):
# lots of function calls, but it just divides pressure_mb by 10 and rounds it
pressureKpa = str(
round(
float(self._getNodeValue(dom, 'pressure_mb', u('0'))) / 10,
1)) + 'kPa'
pressureIn = self._getNodeValue(dom, 'pressure_in', u('0')) + 'in'
return self._formatForMetricOrImperial(pressureIn, pressureKpa)
def _formatSpeeds(self, dom, string):
mphValue = float(self._getNodeValue(dom, string, u('0')))
speedM = u('%dmph') % round(mphValue)
speedK = u('%dkph') % round(
mphValue * 1.609344) # thanks Wikipedia for the conversion rate
return self._formatForMetricOrImperial(speedM, speedK)
def _formatUpdatedTime(self, dom):
observationTime = self._getNodeValue(dom, 'observation_epoch', None)
localTime = self._getNodeValue(dom, 'local_epoch', None)
if not observationTime or not localTime:
return self._getNodeValue(dom, 'observation_time',
'Unknown Time').lstrip(
u('Last Updated on '))
seconds = int(localTime) - int(observationTime)
minutes = int(seconds / 60)
seconds -= minutes * 60
hours = int(minutes / 60)
minutes -= hours * 60
if seconds == 1:
seconds = '1 sec'
else:
seconds = '%d secs' % seconds
if minutes == 1:
minutes = '1 min'
else:
minutes = '%d mins' % minutes
if hours == 1:
hours = '1 hr'
else:
hours = '%d hrs' % hours
if hours == '0 hrs':
if minutes == '0 mins':
return '%s ago' % seconds
return '%s, %s ago' % (minutes, seconds)
return '%s, %s, %s ago' % (hours, minutes, seconds)
def _getCurrentConditions(self, dom):
output = []
temp = self._formatCurrentConditionTemperatures(dom, 'temp')
if self._getNodeValue(dom, 'heat_index_string') != 'NA':
temp += u(' (Heat Index: %s)'
) % self._formatCurrentConditionTemperatures(
dom, 'heat_index')
if self._getNodeValue(dom, 'windchill_string') != 'NA':
temp += u(' (Wind Chill: %s)'
) % self._formatCurrentConditionTemperatures(
dom, 'windchill')
output.append(u('Temperature: ') + temp)
output.append(
u('Humidity: ') +
self._getNodeValue(dom, 'relative_humidity', u('N/A%')))
if self.registryValue('showPressure', self.__channel):
output.append(u('Pressure: ') + self._formatPressures(dom))
output.append(
u('Conditions: ') +
self._getNodeValue(dom, 'weather').capitalize())
output.append(
u('Wind: ') +
self._getNodeValue(dom, 'wind_dir', u('None')).capitalize() +
', ' + self._formatSpeeds(dom, 'wind_mph'))
output.append(u('Updated: ') + self._formatUpdatedTime(dom))
return u('; ').join(output)
def _getDom(self, url):
try:
xmlString = utils.web.getUrl(url)
return dom.parseString(xmlString)
except utils.web.Error as e:
error = e.args[0].capitalize()
if error[-1] != '.':
error = error + '.'
self.__irc.error(error, Raise=True)
def _getForecast(self, location):
dom = self._getDom(self._weatherForecastURL %
utils.web.urlquote(location))
output = []
count = 0
max = self.registryValue('forecastDays', self.__channel)
forecast = dom.getElementsByTagName('simpleforecast')[0]
for day in forecast.getElementsByTagName('forecastday'):
if count >= max and max != 0:
break
forecastOutput = []
forecastOutput.append(
'Forecast for ' +
self._getNodeValue(day, 'weekday').capitalize() + ': ' +
self._getNodeValue(day, 'conditions').capitalize())
forecastOutput.append(
'High of ' + self._formatForecastTemperatures(day, 'high'))
forecastOutput.append('Low of ' +
self._formatForecastTemperatures(day, 'low'))
output.append('; '.join(forecastOutput))
count += 1
return output
########## STATIC METHODS ##########
def _formatUnicodeOutput(output):
# UTF-8 encoding is required for Supybot to handle \xb0 (degrees) and other special chars
# We can't (yet) pass it a Unicode string on its own (an oddity, to be sure)
s = u(' | ').join(output)
if sys.version_info[0] < 3:
s = s.encode('utf-8')
return s
_formatUnicodeOutput = staticmethod(_formatUnicodeOutput)
def _getNodeValue(dom, value, default=u('Unknown')):
subTag = dom.getElementsByTagName(value)
if len(subTag) < 1:
return default
subTag = subTag[0].firstChild
if subTag == None:
return default
return subTag.nodeValue
_getNodeValue = staticmethod(_getNodeValue)
def _noLocation():
raise NoLocation(noLocationError)
_noLocation = staticmethod(_noLocation)
class WeatherBot(callbacks.Plugin):
"""A weather script that uses Darksky and Weatherstack.
"""
threaded = True
@wrap(["owner"])
def createdb(self, irc: callbacks.NestedCommandsIrcProxy, msg: ircmsgs.IrcMsg, args: List[str]) -> None:
"""- takes no arguments.
Creates a new user table.
"""
try:
result: str = User.create_tables()
irc.reply(result, prefixNick=False)
except DatabaseError as exc:
log.error(str(exc))
irc.reply("There was an error with the database. Check logs.", prefixNick=False)
@wrap([getopts({"user": ""}), optional("text")])
def weather(
self,
irc: callbacks.NestedCommandsIrcProxy,
msg: ircmsgs.IrcMsg,
args: List[str],
optlist: List[Tuple[str, bool]],
text: str,
) -> None:
"""- optional <location> OR [--user] <username>
Calls the current weather given an optional arg .e.g. .weather 70119 -
If you leave out the location, it will try to use the user's set location that is saved -
You can find another user's weather by using the --user flag. e.g. .weather --user Chad
"""
lookup_user: bool = False
for opt, _ in optlist:
if opt == "user":
lookup_user = True
try:
optional_user = html.escape(text) if lookup_user and text else msg.nick
if lookup_user and not text:
irc.reply(f"Please specify the user name.", prefixNick=False)
return
user: Union[User, AnonymousUser] = get_user(optional_user)
if lookup_user:
if not isinstance(user, AnonymousUser):
weather: str = query_current_weather("", user)
irc.reply(weather, prefixNick=False)
else:
irc.reply(f"No such user by the name of {text}.", prefixNick=False)
elif not text and isinstance(user, AnonymousUser):
irc.reply(f"No weather location set by {msg.nick}", prefixNick=False)
elif not text:
weather: str = query_current_weather(text, user)
irc.reply(weather, prefixNick=False)
else:
deserialized_location: Dict[str, str] = UserSchema().load({"location": html.escape(text)}, partial=True)
weather: str = query_current_weather(deserialized_location["location"], user)
irc.reply(weather, prefixNick=False)
except ValidationError as exc:
if "location" in exc.messages:
message = exc.messages["location"][0]
irc.reply(message, prefixNick=False)
log.error(str(exc), exc_info=True)
except DatabaseError as exc:
log.error(str(exc), exc_info=True)
if "not created" in str(exc):
irc.reply(str(exc), prefixNick=True)
else:
irc.reply("There is an error. Contact admin.", prefixNick=False)
except (LocationNotFound, WeatherNotFound) as exc:
irc.reply(str(exc), prefixNick=False)
except RequestException as exc:
log.error(str(exc), exc_info=True)
if exc.response.status_code == 400:
irc.reply("Unable to find this location.", prefixNick=False)
else:
irc.reply("There is an error. Contact admin.", prefixNick=False)
@wrap([getopts({"metric": ""}), "text"])
def setweather(
self,
irc: callbacks.NestedCommandsIrcProxy,
msg: ircmsgs.IrcMsg,
args: List[str],
optlist: List[Tuple[str, bool]],
text: str,
) -> None:
"""[--metric] <location>
Sets the weather location for a user. Format is set to show imperial first by default.
To show metric first, use the --metric option. e.g. setweather --metric 70118
"""
try:
format = 1
for option, _ in optlist:
if option == "metric":
format = 2
deserialized_location: Dict[str, str] = UserSchema().load(
{"location": html.escape(text), "format": format}, partial=True,
)
geo: Dict[str, str] = query_location(deserialized_location["location"])
geo.update({"nick": msg.nick, "host": f"{msg.user}@{msg.host}", "format": format})
if geo["location"] is None:
raise LocationNotFound("Unable to find this location.")
user_schema: Dict[str, str] = UserSchema().load(geo)
user, created = User.get_or_create(
nick=msg.nick,
defaults={
"host": user_schema["host"],
"format": user_schema["format"],
"location": user_schema["location"],
"region": user_schema["region"],
"coordinates": user_schema["coordinates"],
},
)
# If created is a boolean of 0, it means it found a user.
# Updates the user fields and saves to db.
if not created:
user.host = user_schema["host"]
user.format = user_schema["format"]
user.location = user_schema["location"]
user.region = user_schema["region"]
user.coordinates = user_schema["coordinates"]
user.save()
units = "imperial" if format == 1 else "metric"
log.info(f"{msg.nick} set their location to {text}")
irc.reply(f"{msg.nick} set their weather to {units} first and {text}.", prefixNick=False)
except ValidationError as exc:
if "location" in exc.messages:
message = exc.messages["location"][0]
irc.reply(message, prefixNick=False)
elif "format" in exc.messages:
message = exc.messages["format"][0]
irc.reply(message, prefixNick=False)
log.error(str(exc), exc_info=True)
except DatabaseError as exc:
log.error(str(exc), exc_info=True)
if "not created" in str(exc):
irc.reply(str(exc), prefixNick=True)
else:
irc.reply("There is an error. Contact admin.", prefixNick=False)
except LocationNotFound as exc:
irc.reply(str(exc), prefixNick=False)
except RequestException as exc:
log.error(str(exc), exc_info=True)
irc.reply("Unable to find this location.", prefixNick=False)
class Wikifetch(callbacks.Plugin):
"""Grabs data from Wikipedia and other MediaWiki-powered sites."""
threaded = True
# This defines a series of suffixes this should be added after the domain name.
SPECIAL_URLS = {
'wikia.com': '/wiki',
'wikipedia.org': '/wiki',
'wiki.archlinux.org': '/index.php',
'wiki.gentoo.org': '/wiki',
'mediawiki.org': '/wiki',
'wikimedia.org': '/wiki',
}
def _get_article_tree(self, baseurl, search):
"""
Returns the article tree given the base URL and search query. baseurl can be None,
in which case, the search query will be treated as a raw string.
"""
if baseurl is None:
addr = search
else:
# Different instances of MediaWiki use different URLs... This tries
# to make the parser work for most sites, but still use resonable defaults
# such as filling in http:// and appending /wiki to links...
# Special cases: Wikia, Wikipedia, Wikimedia (i.e. Wikimedia Commons), Arch Linux Wiki
if '/' not in search:
baseurl = baseurl.lower()
for match, suffix in self.SPECIAL_URLS.items():
if match in baseurl:
baseurl += suffix
break
# Add http:// to the URL if a scheme isn't specified
if not baseurl.startswith(('http://', 'https://')):
baseurl = 'http://' + baseurl
# first, we get the page
addr = '%s/Special:Search?search=%s' % \
(baseurl, quote_plus(search))
self.log.debug('Wikifetch: using URL %s', addr)
try:
article = utils.web.getUrl(addr, timeout=3)
except utils.web.Error:
self.log.exception('Failed to fetch link %s', addr)
raise
if sys.version_info[0] >= 3:
article = article.decode()
tree = lxml.html.document_fromstring(article)
return (tree, article, addr)
def _wiki(self, irc, msg, search, baseurl):
"""Fetches and replies content from a MediaWiki-powered website."""
reply = ''
# First, fetch and parse the page
tree, article, addr = self._get_article_tree(baseurl, search)
# check if it gives a "Did you mean..." redirect
didyoumean = tree.xpath('//div[@class="searchdidyoumean"]/a'
'[@title="Special:Search"]')
if didyoumean:
redirect = didyoumean[0].text_content().strip()
if sys.version_info[0] < 3:
if isinstance(redirect, unicode):
redirect = redirect.encode('utf-8', 'replace')
if isinstance(search, unicode):
search = search.encode('utf-8', 'replace')
if self.registryValue('showRedirects', msg.args[0]):
reply += _('I didn\'t find anything for "%s". '
'Did you mean "%s"? ') % (search, redirect)
tree, article, addr = self._get_article_tree(
baseurl, didyoumean[0].get('href'))
search = redirect
# check if it's a page of search results (rather than an article), and
# if so, retrieve the first result
searchresults = tree.xpath('//div[@class="searchresults"]/ul/li/a') or \
tree.xpath('//article/ul/li/a') # Special case for Wikia (2017-01-27)
self.log.debug('Wikifetch: got search results %s', searchresults)
if searchresults:
redirect = searchresults[0].text_content().strip()
if self.registryValue('showRedirects', msg.args[0]):
reply += _('I didn\'t find anything for "%s", but here\'s the '
'result for "%s": ') % (search, redirect)
# Follow the search result and fetch that article. Note: use the original
# base url to prevent prefixes like "/wiki" from being added twice.
self.log.debug('Wikifetch: following search result:')
tree, article, addr = self._get_article_tree(
None, searchresults[0].get('href'))
search = redirect
# otherwise, simply return the title and whether it redirected
elif self.registryValue('showRedirects', msg.args[0]):
redirect = re.search(
'\(%s <a href=[^>]*>([^<]*)</a>\)' % _('Redirected from'),
article)
if redirect:
try:
redirect = tree.xpath(
'//span[@class="mw-redirectedfrom"]/a')[0]
redirect = redirect.text_content().strip()
title = tree.xpath('//*[@class="firstHeading"]')
title = title[0].text_content().strip()
if sys.version_info[0] < 3:
if isinstance(title, unicode):
title = title.encode('utf-8', 'replace')
if isinstance(redirect, unicode):
redirect = redirect.encode('utf-8', 'replace')
reply += '"%s" (Redirected from "%s"): ' % (title,
redirect)
except IndexError:
pass
# extract the address we got it from - most sites have the perm link
# inside the page itself
try:
addr = tree.find(".//div[@class='printfooter']/a").attrib['href']
addr = re.sub('([&?]|(amp;)?)oldid=\d+$', '', addr)
except:
# If any of the above post-processing tricks fail, just ignore
pass
# check if it's a disambiguation page
disambig = tree.xpath('//table[@id="disambigbox"]') or \
tree.xpath('//table[@id="setindexbox"]') or \
tree.xpath('//div[contains(@class, "disambig")]') # Wikia (2017-01-27)
if disambig:
reply += format(_('%u is a disambiguation page. '), addr)
disambig = tree.xpath('//div[@id="bodyContent"]/div/ul/li')
disambig_results = []
for item in disambig:
for link in item.findall('a'):
if link.text is not None:
# Hackishly bold all <a> tags
link.text = "%s" % link.text
item = item.text_content().replace('', '\x02')
# Normalize and strip whitespace, to prevent newlines and such
# from corrupting the display.
item = utils.str.normalizeWhitespace(item).strip()
disambig_results.append(item)
if disambig_results:
reply += format(_('Possible results include: %L'),
disambig_results)
# Catch talk pages
elif 'ns-talk' in tree.find("body").attrib.get('class', ''):
reply += format(_('This article appears to be a talk page: %u'),
addr)
else:
p = tree.xpath("//div[@id='mw-content-text']/p[1]")
if len(p) == 0 or 'wiki/Special:Search' in addr:
if 'wikipedia:wikiproject' in addr.lower():
reply += format(
_('This page appears to be a WikiProject page, '
'but it is too complex for us to parse: %u'), addr)
else:
irc.error(_('Not found, or page malformed.'), Raise=True)
else:
p = p[0]
# Replace <b> tags with IRC-style bold, this has to be
# done indirectly because unescaped '\x02' is invalid in XML
for b_tag in p.xpath('//b'):
b_tag.text = "%s" % (b_tag.text or '')
p = p.text_content()
p = p.replace('', '\x02')
# Get rid of newlines, etc., that can corrupt the output.
p = utils.str.normalizeWhitespace(p)
p = p.strip()
if sys.version_info[0] < 3:
if isinstance(p, unicode):
p = p.encode('utf-8', 'replace')
if isinstance(reply, unicode):
reply = reply.encode('utf-8', 'replace')
if not p:
reply = _('<Page was too complex to parse>')
reply += format('%s %s %u', p, _('Retrieved from'), addr)
reply = reply.replace('&', '&')
# Remove inline citations (text[1][2][3], etc.)
reply = re.sub('\[\d+\]', '', reply)
return reply
@internationalizeDocstring
@wrap([getopts({'site': 'somethingWithoutSpaces'}), 'text'])
def wiki(self, irc, msg, args, optlist, search):
"""[--site <site>] <search term>
Returns the first paragraph of a wiki article. Optionally, a --site
argument can be given to override the default (usually Wikipedia) -
try using '--site lyrics.wikia.com' or '--site wiki.archlinux.org'."""
optlist = dict(optlist)
baseurl = optlist.get('site') or self.registryValue('url', msg.args[0])
text = self._wiki(irc, msg, search, baseurl)
irc.reply(text)
@internationalizeDocstring
@wrap([additional('somethingWithoutSpaces')])
def random(self, irc, msg, args, site):
"""[<site>]
Returns the first paragraph of a random wiki article. Optionally, the --site
argument can be given to override the default (usually Wikipedia)."""
baseurl = site or self.registryValue('url', msg.args[0])
text = self._wiki(irc, msg, 'Special:Random', baseurl)
irc.reply(text)