def execute(self, execstr, args=None, retry=2):
""" execute string on database. """
time.sleep(0.001)
result = 0
execstr = execstr.strip()
if 'sqlite' in self.dbtype: execstr = execstr.replace('%s', '?')
if self.dbtype == 'mysql':
try: self.ping()
except AttributeError: self.reconnect()
except Exception as ex:
try: self.reconnect()
except Exception as ex: logging.error('failed reconnect: %s' % str(ex)) ; return
logging.debug('exec %s %s' % (execstr, args))
got = False
counter = 0
for i in range(retry):
if not self.connection: self.reconnect()
if not self.connection: raise NoDbConnection()
if "sqlite" in self.dbtype:
try: result = self.doit(execstr, args) ; got = True ; break
except sqlite3.OperationalError as ex: logging.info(str(ex))
except Exception as ex: handle_exception() ; logging.error(str(ex))
else:
try: result = self.doit(execstr, args) ; got = True ; break
except Exception as ex: logging.error(str(ex))
counter += 1
if got:
if counter == 1: logging.warn("db query is ok - 1 retry" % counter)
else: logging.warn("db query is ok - %s retries" % counter)
return result
def doit(self, execstr, args=None):
logging.debug("%s/%s" % (execstr, str(args)))
cursor = self.cursor()
nr = 0
try:
if args != None:
if type(args) == tuple or type(args) == list: nr = cursor.execute(execstr, args)
else: nr = cursor.execute(execstr, (args, ))
else: nr = cursor.execute(execstr)
except Exception as ex:
logging.warn(str(ex))
if self.dbtype == 'postgres': cursor.execute(""" ROLLBACK """)
if 'sqlite' in self.dbtype: cursor.close() ; del cursor
raise
got = False
if execstr.startswith('INSERT'): nr = cursor.lastrowid or nr ; got = True
elif execstr.startswith('UPDATE'): nr = cursor.rowcount ; got = True
elif execstr.startswith('DELETE'): nr = cursor.rowcount ; got = True
if got: self.commit()
if 'sqlite' in self.dbtype and not got and type(nr) != int():
nr = cursor.rowcount or cursor.lastrowid
if nr == -1: nr = 0
result = None
try:
result = cursor.fetchall()
if not result: result = nr
except Exception as ex:
if 'no results to fetch' in str(ex): logging.warn("no results to fetch")
else: handle_exception()
result = nr
cursor.close()
return result
def plusscan(skip=False):
global teller
teller += 1
if teller % 5 != 0: return
logging.warn("running plus scan")
fleet = getfleet()
for id, channels in state.data.ids.items():
if not id in state.data.seen: state.data.seen[id] = []
for botname, chan in channels:
try:
res = getplus(id)
if not res: logging.warn("no result from %s" % id) ; continue
bot = fleet.byname(botname)
if bot:
todo = []
for r in res:
stamp = uuid.uuid3(uuid.NAMESPACE_URL, str(r)).hex
if stamp not in state.data.seen[id]:
state.data.seen[id].append(stamp)
todo.append(r)
if todo: bot.say(chan, "new plus update: " , todo)
else: logging.warn("no %s bot in fleet" % botname)
except AttributeError as ex: logging.error(str(ex))
except Exception as ex: handle_exception()
state.save()
def incoming(self, data):
""" process incoming hubbub data. """
result = feedparser.parse(data)
url = find_self_url(result.feed.links)
logging.debug("%s - %s" % (url, data))
try:
item = self.byurl(url)
if not item: logging.warn("can't find feed for %s" % url) ; return
if not item.data.running: logging.warn("%s is not in running mode" % item.data.url) ; return
if not item.data.url or item.data.url == 'urlnotset':
item.data.url = url
item.save()
if item:
loopover = item.data.watchchannels
name = item.data.name
else: logging.warn("can't find %s item" % url) ; return
logging.warn("loopover in %s peek is: %s" % (name, loopover))
counter = 1
for i in loopover:
if len(i) == 3:
try: (botname, type, channel) = i
except:
try: (botname, type, channel) = loads(i)
except:
logging.warn('%s is not in the format (botname, bottype, channel)' % str(item))
continue
else:
logging.warn('%s is not in the format (botname, bottype, channel)' % item.data.url)
continue
counter += 1
start_new_thread(self.work, (botname, type, channel, result.entries, url), {"_countdown": counter})
except Exception as ex: handle_exception(txt=url)
return True
def reloadcheck(self, bot, event, target=None):
""" check if plugin need to be reloaded for callback, """
from .boot import isdisabled
plugloaded = []
done = []
target = target or event.cbtype or event.cmnd
if not event.nolog: logging.debug("%s - checking for %s events" % (bot.cfg.name, target))
try:
from .boot import getcallbacktable
p = getcallbacktable()[target]
except KeyError:
if not event.nolog: logging.debug("can't find plugin to reload for %s" % event.cmnd)
return
if not event.nolog: logging.debug("found %s" % str(p))
for name in p:
if name in bot.plugs: done.append(name) ; continue
if isdisabled(name):
logging.info("%s - %s is disabled" % (bot.cfg.name, name))
continue
elif bot.cfg.loadlist and name not in bot.cfg.loadlist:
logging.info("%s - %s is not in loadlist" % (bot.cfg.name, name))
continue
if not event.nolog: logging.debug("%s - on demand reloading of %s" % (bot.cfg.name, name))
try:
mod = bot.plugs.reload(name, force=True, showerror=True)
if mod: plugloaded.append(mod) ; continue
except Exception as ex: handle_exception(event)
if done and not event.nolog: logging.debug("%s - %s is already loaded" % (bot.cfg.name, str(done)))
return plugloaded
def _listen(self):
""" listen for udp messages .. /msg via bot"""
if not cfg['udp']: return
fleet = getfleet()
for botname in cfg['udpbots']:
if not fleet.byname(botname): logging.info("udp - can't find %s bot" % botname)
try:
fleet.startok.wait(5)
logging.warn('udp listening on %s %s' % (cfg['udphost'], cfg['udpport']))
self.sock.bind((cfg['udphost'], cfg['udpport']))
self.stop = 0
except IOError:
handle_exception()
self.sock = None
self.stop = 1
return
# loop on listening udp socket
while not self.stop:
try: input, addr = self.sock.recvfrom(64000)
except socket.timeout: continue
except Exception as ex:
try: (errno, errstr) = ex
except ValueError: errno = 0 ; errstr = str(ex)
if errno == 4: logging.warn("udp - %s - %s" % (self.name, str(ex))) ; break
if errno == 35: continue
else: handle_exception() ; break
if self.stop: break
self.queue.put((input, addr))
logging.info('udp - shutting down main loop')
def docbs(self, bot, event):
""" do the actual callback .. put callback on the waitrunner for execution. """
if not self.cbs: return
logging.warn("%s - found wait match: %s" % (bot.cfg.name, event.dump()))
for cb in self.cbs:
try: cb(bot, event) #waitrunner.put(event.speed, (self.modname, cb, bot, event))
except Exception as ex: handle_exception()
def handle_rc(bot, event):
""" arguments: <file>|<url> - execute a .tl resource file with bot commands. """
if not event.rest: event.missing("<file>|<url>") ; return
if not getmainconfig().allowrc: event.reply("rc mode is not enabled") ; return
teller = 0
t = event.rest
waiting = []
try:
try:
if getmainconfig().allowremoterc and t.startswith("http"): data = geturl2(t)
else: data = open(t, 'r').read()
except IOError as ex: event.reply("I/O error: %s" % str(ex)) ; return
if not data: event.reply("can't get data from %s" % event.rest) ; return
for d in data.split("\n"):
i = d.strip()
if not i: continue
if i.startswith("#"): continue
e = cpy(event)
e.txt = "%s" % i.strip()
e.direct = True
bot.put(e)
waiting.append(e)
teller += 1
event.reply("%s commands executed" % teller)
except Exception as ex: event.reply("an error occured: %s" % str(ex)) ; handle_exception()
def work(self, botname, type, channel, entries, url, *args, **kwargs):
logging.debug("in work .. %s" % str(entries))
try:
item = self.byurl(url)
name = item.data.name
try:
fleet = getfleet()
bot = fleet.byname(botname)
if not bot: logging.warn("can't find %s bot in fleet" % botname) ; return
except NoSuchBotType as ex: logging.warn("%s" % str(ex)) ; return
if not bot: logging.error("can't find %s bot in fleet" % type) ; return
res2 = entries
if not res2: logging.info("no updates for %s (%s) feed available" % (item.data.name, channel)) ; return
if item.markup.get(jsonstring([name, channel]), 'reverse-order'): res2 = res2[::-1]
if item.markup.get(jsonstring([name, channel]), 'all-lines'):
for i in res2:
response = self.makeresponse(name, [i, ], channel)
try: bot.say(channel, response)
except Exception as ex: handle_exception()
else:
sep = item.markup.get(jsonstring([name, channel]), 'separator')
if sep: response = self.makeresponse(name, res2, channel, sep=sep)
else: response = self.makeresponse(name, res2, channel)
bot.say(channel, response)
except Exception as ex: handle_exception()
def handle_geoJOIN(bot, event):
event.reply("geo - doing query on %s" % event.hostname)
try:
result = querygeoipserver(host2ip(event.hostname))
if result: event.reply("%s lives in %s, %s (%s)" % (event.nick, result['city'], result['country_name'], result['country_code']))
else: event.reply("no result")
except: handle_exception()
def startshell(self, connect=True):
""" start the console bot. """
self.start(False)
print(getfullversion("CONSOLE"))
while not self.stopped:
try:
mainhandler.handle_one()
input = console.raw_input("\n> ")
if self.stopped: return
event = ConsoleEvent()
event.parse(self, input, console)
event.nooutput = True
event.nodispatch = False
e = self.put(event)
res = e.wait()
if res:
sys.stdout.write("\n")
txt = self.makeresponse(res, dot="<br>")
self.out(e.userhost, txt)
sys.stdout.flush()
except TLStop: break
except IOError: break
except NoInput: continue
except (KeyboardInterrupt, EOFError): break
except Exception as ex: handle_exception() ; break
console.save_history()
def handle_adminreloadconfig(bot, event):
""" no arguments - reload bot config and mainconfig files. """
try:
bot.cfg.reload()
getmainconfig().reload()
except Exception as ex: handle_exception()
event.done()
def handle(self, job):
speed, event, url, depth, spiderspeed = job.args
if not url: logging.error("no url provided") ; return
if depth < 0: return
if not self.url.base in url: logging.warn("skipping %s (%s)" % (url, self.url.base)) ; return
if url in self.errors: logging.warn("skipping %s" % url) ; return
urls = []
linknr = 0
follownr = 0
n = 0
try:
if url not in self.urls:
self.urls.append(url)
page = Url(url)
time.sleep(10-spiderspeed)
content = page.fetch()
event.reply("fetched %s - %s - %s" % (url, len(content), content.status))
try:
urldata = UrlData(url, striphtml(content))
if urldata.data.txt: urldata.save()
except Exception as ex: handle_exception()
for p in page.geturls():
if not p in self.errors:
self.put(6, event, p, depth-1, spiderspeed-1)
if not self.queue.qsize(): self.stop()
except urllib.error.URLError as ex: logging.warn("error fetching %s url: %s" % (url, str(ex)))
except Exception as e:
logging.warn("ERROR: Can't process url '%s' (%s)" % (url, e))
self.errors.append(url)
handle_exception()
if len(self.errors) > 10: self.stop()
def get_tinyurl(url):
""" grab a tinyurl. """
res = get(url, namespace='tinyurl') ; logging.debug('tinyurl - cache - %s' % str(res))
if res and res[0] == '[': return json.loads(res)
postarray = [
('submit', 'submit'),
('url', url),
]
postdata = urllib.parse.urlencode(postarray)
req = urllib.request.Request(url=plugcfg.url, data=bytes(postdata, "utf-8"))
req.add_header('User-agent', useragent())
try: res = urllib.request.urlopen(req).readlines()
except urllib.error.URLError as e: logging.warn('tinyurl - %s - URLError: %s' % (url, str(e))) ; return
except urllib.error.HTTPError as e: logging.warn('tinyurl - %s - HTTP error: %s' % (url, str(e))) ; return
except Exception as ex:
if "DownloadError" in str(ex): logging.warn('tinyurl - %s - DownloadError: %s' % (url, str(e)))
else: handle_exception()
return
urls = []
for line in res:
l = str(line, "utf-8")
if l.startswith('<blockquote><b>'): urls.append(striphtml(l.strip()).split('[Open')[0])
if len(urls) == 3: urls.pop(0)
set(url, json.dumps(urls), namespace='tinyurl')
return urls
def handle_alarmadd(bot, ievent):
""" alarm <txt-with-time> | <+delta> <txt> .. add an alarm """
if not ievent.rest: ievent.reply('alarm <txt-with-time> or alarm <+delta> <txt>') ; return
else: alarmtxt = ievent.rest
if alarmtxt[0] == '+':
try: sec = int(ievent.args[0][1:])
except ValueError: ievent.reply('use +nrofsecondstosleep') ; return
if len(ievent.args) < 2: ievent.reply('i need txt to remind you') ; return
try:
ttime = time.time() + sec
if ttime > 2**31: ievent.reply("time overflow") ; return
if bot.type == "xmpp":
if ievent.groupchat: nrid = alarms.add(bot.cfg.name, ievent.nick, ttime, ' '.join(ievent.args[1:]), ievent.channel)
else: nrid = alarms.add(bot.cfg.name, ievent.stripped, ttime, ' '.join(ievent.args[1:]))
elif bot.type == "irc":
if ievent.msg: nrid = alarms.add(bot.cfg.name, ievent.nick, ttime, ' '.join(ievent.args[1:]), ievent.nick)
else: nrid = alarms.add(bot.cfg.name, ievent.nick, ttime, ' '.join(ievent.args[1:]), ievent.channel)
else: nrid = alarms.add(bot.cfg.name, ievent.nick, ttime, ' '.join(ievent.args[1:]), ievent.channel)
ievent.reply("alarm %s set at %s" % (nrid, time.ctime(ttime)))
return
except Exception as ex: handle_exception(ievent) ; return
alarmtime = strtotime(alarmtxt)
if not alarmtime: ievent.reply("can't detect time") ; return
txt = striptime(alarmtxt).strip()
if not txt: ievent.reply('i need txt to remind you') ; return
if time.time() > alarmtime: ievent.reply("we are already past %s" % time.ctime(alarmtime)) ; return
if bot.jabber: nrid = alarms.add(bot.cfg.name, ievent.nick, alarmtime, txt, ievent.channel)
else: nrid = alarms.add(bot.cfg.name, ievent.nick, alarmtime, txt, ievent.channel)
ievent.reply("alarm %s set at %s" % (nrid, time.ctime(alarmtime)))
def loadall(self, paths=[], force=True):
"""
load all plugins from given paths, if force is true ..
otherwise load all plugins for default_plugins list.
"""
if not paths: paths = plugin_packages
imp = None
old = list(self.keys())
new = []
logging.warn("loading all from paths %s" % paths)
for module in paths:
try: imp = _import(module)
except ImportError as ex: logging.warn("no %s plugin package found - %s" % (module, str(ex))) ; continue
except Exception as ex: handle_exception() ; continue
logging.warn("got plugin package %s" % module)
try: todo = imp.__plugs__
except AttributeError as ex:
if "__plugs__" in str(ex):
logging.error("no plugins in %s .. define __plugs__ in __init__.py" % module)
continue
raise
todo.sort()
for plug in todo:
mod = "%s.%s" % (module, plug)
if not force and isdisabled(mod): logging.warn("%s is not enabled. not loading" % mod) ; continue
try: self.reload(mod, force=force, showerror=True)
except RequireError as ex: logging.info(str(ex)) ; continue
except KeyError: logging.debug("failed to load plugin package %s" % module) ; continue
except Exception as ex: handle_exception() ; continue
new.append(mod)
remove = [x for x in old if x not in new and x not in default_plugins]
logging.info("unload list is: %s" % ", ".join(remove))
for mod in remove: self.unload(mod)
return new
def handle_chatlogoff(bot, ievent):
""" no arguments - disable chatlog. """
try: cfg['channels'].remove([bot.cfg.name, ievent.channel]) ; cfg.save()
except ValueError: ievent.reply('chatlog is not enabled in (%s,%s)' % (bot.cfg.name, ievent.channel)) ; return
try: del loggers["%s-%s" % (bot.cfg.name, stripname(ievent.channel))]
except KeyError: pass
except Exception as ex: handle_exception()
ievent.reply('chatlog disabled on (%s,%s)' % (bot.cfg.name, ievent.channel))
def search(self, field, target):
res = []
for obj in list(self.objects().values()):
try: item = getattr(obj.data, field)
except AttributeError: handle_exception() ; continue
if not item: continue
if target in item: res.append(obj)
return res
def makeoptions(self):
""" check the given txt for options. """
try: self.options = makeeventopts(self.txt)
except: handle_exception() ; return
if not self.options: return
if self.options.channel: self.target = self.options.channel
logging.info("options - %s" % str(self.options))
self.txt = ' '.join(self.options.args)
self.makeargs()
def handle_size(bot, event):
res = []
mods = dict(sys.modules)
for name, mod in mods.items():
if not 'tl' in name: continue
try: res.append("<i><%s></i> %s" % (name.split(".")[-1], str(getattr(mod, 'size')())))
except (TypeError, AttributeError): continue
except Exception as ex: handle_exception()
event.reply("sizes in %s modules scanned: " % len(res), res, dot=", ")
def hubbub_post(bot, event):
"""Handles new content notifications."""
logging.warn("incoming POST")
try: watcher.incoming(event.request.body)
except IndexError: logging.error("hubbub plugin did not load properly")
except Exception as ex:
handle_exception()
event.handler.send_error(500)
event.finish()
def run(self):
""" run the thread. """
try:
logging.debug('threads - running thread %s' % self.name)
threading.Thread.run(self)
except URLNotEnabled as ex: logging.warn("url fetching is disabled - %s" % str(ex))
except Exception as ex:
handle_exception()
time.sleep(1)
def run(self):
""" run the bot command. """
try:
self.bot.benice()
result = threading.Thread.run(self)
self.ievent.ready()
except Exception as ex:
handle_exception(self.ievent)
time.sleep(1)
def handle_show(self, bot, ievent):
if bot.cfg.name not in self.data:
ievent.reply('no bugtracker configured')
return False
try:
self.instances[bot.cfg.name][ievent.channel].handle_show(bot, ievent)
except KeyError as ex: ievent.reply("key error: %s" % str(ex)) ; return False
except Exception as e:
handle_exception()
ievent.reply('failed to fetch bug information: %s' % str(e))
def twittercb(bot, event):
event.dontbind = False
event.bind(force=True)
try: twittername = event.chan.data.twittername
except Exception as ex: handle_exception() ; twittername = None
try: username = event.origin ; logging.info("using origin %s" % event.origin)
except Exception as ex: handle_exception() ; username = None
if twittername:
try: twitter_out(twittername or username or "tl", event.txt, event)
except Exception as ex: logging.warn("error posting to twitter: %s" % str(ex))
def handle_bddel(bot, ievent):
""" bd-del .. delete birthday """
global db
if not db: ievent.reply("plugin isnt initialised yet") ; return
name = bot.users.getname(ievent.userhost)
result = 0
try: result = db.execute(""" DELETE FROM birthday WHERE name = %s """, name)
except Exception as ex: handle_exception() ; return
if result: ievent.reply('birthday removed')
else: ievent.reply('no birthday known for %s' % ievent.nick)
def startall(self, bots, usethreads=True):
threads = []
target = bots or self.bots
for bot in target:
logging.debug('starting %s bot (%s)' % (bot.cfg.name, bot.type))
if usethreads or bot.type in ["xmpp", "tornado"]: threads.append(start_new_thread(bot.boot, ())) ; continue
try: bot.boot()
except Excepton as ex: handle_exception()
time.sleep(1)
for t in threads: t.join(15)
self.startok.set()
def getmodule(self, plugname):
for module in plugin_packages:
try:
imp = _import(module)
plugs = imp.__plugs__
except AttributeError: continue
except ImportError as ex:
logging.error("%s - %s" % (plugname, str(ex)))
continue
except Exception as ex: handle_exception() ; continue
if imp and plugname in imp.__plugs__: return "%s.%s" % (module, plugname)
def start_bot_command(func, arglist, kwargs={}):
""" start a new thread .. set name to function/method name. """
if not kwargs: kwargs = {}
try:
name = getname(func)
if not name: name = 'noname'
thread = Botcommand(group=None, target=func, name=name, args=arglist, kwargs=kwargs)
thread.start()
return thread
except:
handle_exception()
time.sleep(1)
def mainloop():
""" function to be used as mainloop. """
while 1:
try:
time.sleep(1)
mainhandler.handle_one()
except TLStop: break
except KeyboardInterrupt: break
except Exception as ex:
handle_exception()
break
globalshutdown()
