def __init__(self, id, botname=None, type="notset"): if not id: raise NoChannelSet() if not botname: Persist.__init__(self, getdatadir() + os.sep + 'channels' + os.sep + stripname(id)) else: Persist.__init__(self, getdatadir() + os.sep + 'fleet' + os.sep + stripname(botname) + os.sep + 'channels' + os.sep + stripname(id)) self.id = id self.type = type self.lastmodified = time.time() self.data.id = id self.data.enable = self.data.enable or False self.data.ops = self.data.ops or [] self.data.silentcommands = self.data.silentcommands or [] self.data.allowcommands = self.data.allowcommands or [] self.data.feeds = self.data.feeds or [] self.data.forwards = self.data.forwards or [] self.data.allowwatch = self.data.allowwatch or [] self.data.watched = self.data.watched or [] self.data.passwords = self.data.passwords or {} self.data.cc = self.data.cc or "!" self.data.nick = self.data.nick or "tl" self.data.key = self.data.key or "" self.data.denyplug = self.data.denyplug or [] self.data.createdfrom = whichmodule() self.data.cacheindex = 0 self.data.tokens = self.data.tokens or [] self.data.webchannels = self.data.webchannels or []
def oldname(name): from tl.lib.datadir import getdatadir if name.startswith("-"): name[0] = "+" name = name.replace("@", "+") if os.path.exists(getdatadir() + os.sep + name): return name name = name.replace("-", "#") name = prevchan.replace("+", "@") if os.path.exists(getdatadir() + os.sep + name): return name return ""
def get_token(username): global users if not users: users = TwitterUsers(getdatadir() + os.sep + "twitter" + os.sep + "users") if not users: raise Exception("can't get twitter users object") ; return key, secret = getcreds(getdatadir()) if not key: raise Exception(getdatadir()) if not secret: raise Exception(getdatadir()) token = twittertoken(key, secret, users, username) if not token: raise Exception("%s %s" % (str(user), username)) api = twitterapi(key, secret, token) if not api: raise Exception("%s %s" % (str(user), name)) return (token, api)
def handle_chanupgrade(bot, event): """ no arguments - upgrade the channel. """ prevchan = event.channel # 0.4.1 if prevchan.startswith("-"): prevchan[0] = "+" prevchan = prevchan.replace("@", "+") prev = Persist(getdatadir() + os.sep + "channels" + os.sep + prevchan) if prev.data: event.chan.data.update(prev.data) ; event.chan.save() ; event.reply("done") else: prevchan = event.channel prevchan = prevchan.replace("-", "#") prevchan = prevchan.replace("+", "@") prev = Persist(getdatadir() + os.sep + "channels" + os.sep + prevchan) if prev.data: event.chan.data.update(prev.data) ; event.chan.save() ; event.reply("done") else: event.reply("can't find previous channel data")
def startmcdaemon(): #if not getmainconfig().memcached: return logging.warn("starting memcached daemon") try: from tl.utils.popen import gozerpopen rundir = getdatadir() + os.sep + 'run' sock = os.path.abspath(rundir + os.sep + "memcached.socket") logging.warn("using unix socket %s" % sock) pidfile = sock[:-7] + ".pid" if os.path.exists(sock) and isactive(sock): logging.warn("memcached daemon is already running") ; return args = [[]]*4 args[0] = "memcached" args[1] = "-s%s" % sock args[2] = "-P%s" % pidfile args[3] = "-d" logging.debug("running %s" % " ".join(args)) proces = gozerpopen(args) except Exception as ex: if "No such file" in str(ex): logging.warn("no memcached installed") else: logging.error('error running popen: %s' % str(ex)) return data = proces.fromchild.readlines() returncode = proces.close() if returncode == 0: logging.warn("memcached started") else: logging.warn("can't start memcached (%s)" % returncode)
def handle_twittercmnd(bot, ievent): """ arguments: <API cmnd> - do a twitter API cmommand. """ go = getauth() if not go: ievent.reply("the twitter plugin needs the credentials.py file in the .tl/config dir. see the examples dir") ; return if not ievent.args: ievent.missing('<API cmnd>') ; return target = strippedtxt(ievent.args[0]) try: from tl.utils.twitter import get_token token = get_token(ievent.user.data.name) if not token: ievent.reply("you are not logged in yet .. run the twitter-auth command.") ; return key, secret = getcreds(getdatadir()) token = tweepy.oauth.OAuthToken(key, secret).from_string(token) twitter = twitterapi(key, secret, token) cmndlist = dir(twitter) cmnds = [] for cmnd in cmndlist: if cmnd.startswith("_") or cmnd == "auth": continue else: cmnds.append(cmnd) if target not in cmnds: ievent.reply("choose one of: %s" % ", ".join(cmnds)) ; return try: method = getattr(twitter, target) except AttributeError: ievent.reply("choose one of: %s" % ", ".join(cmnds)) ; return result = method() res = [] for item in result: try: res.append("%s - %s" % (item.screen_name, item.text)) except AttributeError: try: res.append("%s - %s" % (item.screen_name, item.description)) except AttributeError: try: res.append(str(item.__getstate__())) except AttributeError: res.append(dir(i)) ; res.append(str(item)) ievent.reply("result of %s: " % target, res) except KeyError: ievent.reply('you are not logged in yet. see the twitter-auth command.') except (tweepy.TweepError, urllib.error.HTTPError) as e: ievent.reply('twitter failed: %s' % (str(e),))
def stripdatadir(fname): res = [] ddir = getdatadir() for i in fname.split(os.sep): if i in ddir: continue res.append(i) return os.sep.join(res)
def getfleet(datadir=None, new=False): if not datadir: from tl.lib.datadir import getdatadir datadir = getdatadir() global fleet if not fleet or new: fleet = Fleet(datadir) fleet.loadall() return fleet
def getauth(datadir=None): """ get auth structure from datadir. """ global auth if auth: return auth if not datadir: datadir = getdatadir() try: key, secret = getcreds(datadir) except RequireError as ex: logging.warn(str(ex)) ; return None auth = tweepy.OAuthHandler(key, secret) return auth
def getplus(target): credentials = import_byfile("credentials", getdatadir() + os.sep + "config" + os.sep + "credentials.py") url = "https://www.googleapis.com/plus/v1/people/%s/activities/public?alt=json&pp=1&key=%s" % (target, credentials.googleclient_apikey) result = geturl2(url) data = json.loads(result) res = [] for item in data['items']: i = LazyDict(item) res.append("%s - %s - %s" % (i.actor['displayName'], i['title'], item['url'])) return res
def getaliases(ddir=None, force=True): """ return global aliases. """ global aliases if not aliases or force: from tl.lib.persist import Persist from tl.utils.lazydict import LazyDict d = ddir or getdatadir() p = Persist(d + os.sep + "run" + os.sep + "aliases") if not p.data: p.data = LazyDict() aliases = p.data return aliases
def handle_twitter_confirm(bot, ievent): """ arguments: <PIN code> - confirm auth with PIN. """ go = getauth() if not go: ievent.reply("the twitter plugin needs the credentials.py file in the %s/config dir. see the examples directory" % getdatadir()) ; return pin = ievent.args[0] if not pin: ievent.missing("<PIN> .. see the twitter-auth command.") ; return try: access_token = getauth(getdatadir()).get_access_token(pin) except (tweepy.TweepError, urllib.error.HTTPError) as e: ievent.reply('twitter failed: %s' % (str(e),)) ; return twitteruser = get_users() twitteruser.add(ievent.user.data.name, access_token.to_string()) ievent.reply("%s access token saved." % ievent.user.data.name)
def __init__(self, name, url="", owner="", itemslist=['title', 'link'], watchchannels=[], running=1): filebase = getdatadir() + os.sep + 'plugs' + os.sep + 'tl.plugs.extra.hubbub' + os.sep + name Persist.__init__(self, filebase + os.sep + name + '.core') if not self.data: self.data = {} self.data = LazyDict(self.data) self.data['name'] = self.data.name or str(name) self.data['url'] = self.data.url or str(url) self.data['owner'] = self.data.owner or str(owner) self.data['watchchannels'] = self.data.watchchannels or list(watchchannels) self.data['running'] = self.data.running or running self.itemslists = Pdol(filebase + os.sep + name + '.itemslists') self.markup = Pdod(filebase + os.sep + name + '.markup')
def savealiases(ddir=None): """ return global aliases. """ global aliases if aliases: logging.warn("saving aliases") from tl.lib.persist import Persist from tl.utils.lazydict import LazyDict d = ddir or getdatadir() p = Persist(d + os.sep + "run" + os.sep + "aliases") p.data = aliases p.save() return aliases
def mcboot(): #if not getmainconfig().memcached: return logging.warn("memcached is enabled") try: import tl.contrib.memcache as memcache rundir = getdatadir() + os.sep + 'run' sock = os.path.abspath(rundir + os.sep + "memcached.socket") global mc mc = memcache.Client(["unix:%s" % sock], debug=0) return isactive(sock) except ImportError as ex: logging.warn("using builtin cache - %s" % str(ex)) except Exception as ex: logging.warn("error starting memcached client: %s" % str(ex))
def handle_twitter_auth(bot, ievent): """ no arguments - get url to get the auth PIN needed for the twitter-confirm command. """ go = getauth() if not go: ievent.reply("the twitter plugin needs the credentials.py file in the .tl/config dir. see the examples directory") ; return try: auth_url = getauth(getdatadir()).get_authorization_url() except (tweepy.TweepError, urllib.error.HTTPError) as e: ievent.reply('twitter failed: %s' % (str(e),)) ; return if bot.type == "irc": bot.say(ievent.nick, "sign in at %s" % auth_url) bot.say(ievent.nick, "use the provided code in the twitter-confirm command.") else: ievent.reply("sign in at %s" % auth_url) ievent.reply("use the provided code in the twitter-confirm command.")
def getcredsmod(datadir=None, doraise=False): """ returnd credendtials.py as a module. """ if not datadir: datadir = getdatadir() try: mod = import_byfile("credentials", datadir + os.sep + "config" + os.sep + "credentials.py") global go go = True except (IOError, ImportError): if doraise: raise RequireError("credentials.py is needed in %s/config dir. see %s/examples" % (datadir, datadir)) else: logging.warn("credentials.py is needed in %s/config dir. see %s/examples" % (datadir, datadir)) return logging.info("found %s credentials" % str(mod)) return mod
def markovlearnlog(chan): """ learn a log """ lines = 0 logfiles = os.listdir(getdatadir() + os.sep + "chatlogs") for filename in logfiles: if chan[1:] not in filename: continue logging.warn("opening %s" % reversename(filename)) for line in open(getdatadir() + os.sep + "chatlogs" + os.sep + filename, "r"): if lines % 10 == 0: time.sleep(0.001) if not line: continue lines += 1 try: txt = " ".join( line.strip().split()[2:] ) # log format is: 2011-08-07 00:02:16 <botfather> love, peace and happiness markovtalk_learn(txt) except IndexError: continue logging.warn("learning %s log done. %s lines" % (chan, lines)) return lines
def handle_chatlogsearch(bot, event): """ arguments: <searchtxt> - search in the logs. """ if not event.rest: event.missing("<searchtxt>") ; return result = [] chatlogdir = getdatadir() + os.sep + "chatlogs" if event.options and event.options.channel: chan = event.options.channel else: chan = event.channel logs = os.listdir(chatlogdir) logs.sort() for f in logs: filename = stripname(f) if not stripname(chan) in filename: continue for line in open(chatlogdir + os.sep + filename, 'r'): if event.rest in line: result.append(line) if result: event.reply("search results for %s: " % event.rest, result, dot= " || ") else: event.reply("no result found for %s" % chan)
def __init__(self, dbname=None, dbhost=None, dbuser=None, dbpasswd=None, dbtype=None, ddir=None, doconnect=True): self.datadir = ddir or getdatadir() self.datadir = self.datadir + os.sep + "db" + os.sep if hasattr(os, 'mkdir'): if not os.path.isdir(self.datadir): try: os.mkdir(self.datadir) except OSError: pass cfg = getmainconfig() self.dbname = dbname or cfg.dbname if not self.dbname: raise Exception("no db name") self.dbhost = dbhost or cfg.dbhost or "" self.dbuser = dbuser or cfg.dbuser or "" self.dbpasswd = dbpasswd or cfg.dbpasswd or "" self.connection = None self.timeout = 15 self.dbtype = dbtype or cfg.dbtype or 'sqlite' if doconnect: self.connect()
def runapiserver(port=None, ddir=None): """ start running the API server. needs to be called from the main thread. """ from tl.drivers.tornado.bot import TornadoBot global bot bot = TornadoBot(botname="api-bot") if port: try: port = int(port) except ValueError: pass else: port = 10105 try: server = createserver(ddir or getdatadir()) server.bind(port) logging.warn("starting API server on port %s" % port) server.start() server.io_loop.start() except KeyboardInterrupt: globalshutdown() except Exception as ex: handle_exception() ; os._exit(1) else: globalshutdown()
def killmcdaemon(): #if not getmainconfig().memcached: return logging.warn("killing memcached daemon") rundir = getdatadir() + os.sep + 'run' sock = os.path.abspath(rundir + os.sep + "memcached.socket") pidfile = sock[:-7] + ".pid" try: pid = int(open(pidfile, "r").read().strip()) except Exception as ex : logging.warn("can't determine pid of memcached from %s - %s" % (pidfile,str(ex))) ; return False logging.warn("pid is %s" % pid) data = isactive(sock) if not data: logging.warn("memcached is not runniing") ; return False try: curr_connections = int(data[0][1]["curr_connections"]) except Exception as ex: logging.warn("can't determine current connections of memcached .. not killing - %s" % str(ex)) ; return False if curr_connections and curr_connections != 1: logging.warn("current connections of memcached is %s .. not killing" % curr_connections) ; return False try: os.kill(pid, 15) ; logging.warn("killed memcached with pid %s" % pid) except Exception as ex: logging.warn("failed to kill memcached (%s) - %s" % (pid, str(ex))) try: os.remove(pidfile) except: pass
def handle_twitterfriends(bot, ievent): """ no arguments - show friends timeline (your normal twitter feed). """ go = getauth() if not go: ievent.reply("the twitter plugin needs the credentials.py file in the .tl/config dir. see the examples directory") ; return try: token = get_token(ievent.user.data.name) if not token: ievent.reply("you are not logged in yet .. run the twitter-auth command.") ; return key , secret = getcreds(getdatadir()) token = tweepy.oauth.OAuthToken(key, secret).from_string(token) twitter = twitterapi(key, secret, token) method = getattr(twitter, "friends_timeline") result = method() res = [] for item in result: try: res.append("%s - %s" % (item.author.screen_name, item.text)) except Exception as ex: handle_exception() ievent.reply("results: ", res) except KeyError: ievent.reply('you are not logged in yet. see the twitter-auth command.') except (tweepy.TweepError, urllib.error.HTTPError) as e: ievent.reply('twitter failed: %s' % (str(e),))
def checkconfig(): changed = [] d = getdatadir() + os.sep + "config" if not os.path.isdir(d): return changed for f in os.listdir(d): if os.path.isdir(d + os.sep + f): dname = d + os.sep + f changed.extend(checktimestamps(d + os.sep + f)) continue m = d + os.sep + f if os.path.isdir(m): continue if "__init__" in f: continue global timestamps try: t = os.path.getmtime(m) if t > timestamps.data[m]: changed.append(m) ; timestamps.data[m] = t ; except KeyError: timestamps.data[m] = os.path.getmtime(m) ; changed.append(m) if changed: timestamps.save() return changed
def markovlearnspider(target): logging.warn("starting spider learn on %s" % target) coll = PersistCollection(getdatadir() + os.sep + "spider" + os.sep + "data") if target.startswith("spider://"): target = target[9:] objs = coll.search("url", target) for obj in objs: if not obj.data and obj.data.url: continue time.sleep(0.001) if target not in obj.data.url: continue logging.warn("url is %s" % obj.data.url) try: if obj.data and obj.data.txt: for line in obj.data.txt.split("\n"): if line.count(";") > 1: continue markovtalk_learn(striphtml(line)) except: handle_exception()
def handle_search(bot, event): if not event.options: event.makeoptions() all = event.options.all res = [] target = event.args if not target: event.missing("<search words seperated by space>") ; return coll = PersistCollection(getdatadir() + os.sep + 'spider' + os.sep + "data") files = coll.filenames(target) if files: for f in files: try: res.append(Persist(f).data.url) except AttributeError as ex: continue objs = coll.search('txt', event.rest) if not objs: objs = list(coll.objects().values()) stats = makestats(objs, target, res) urls = stats_response(stats, target) res.extend(urls) if res: if len(res) < 4 or all: event.reply("found %s urls: " % len(res), res, dot=" -or- ") else: event.reply("found %s urls, use --all for more: " % len(res), res[:3], dot=" -or- ") else: event.reply("no urls found")
def handle_chatlogstats(bot, event): """ no arguments - create log stats of the channel, possible options: --chan <channel> """ what = event.rest.strip() chatlogdir = getdatadir() + os.sep + "chatlogs" if event.options and event.options.channel: chan = event.options.channel else: chan = event.channel logs = os.listdir(chatlogdir) if not logs: event.reply("no logs available for %s" % chan) ; return now = time.time() if what: timetarget = strtotime2(what) ; what = striptime(what) else: timetarget = 0 ; what = None event.reply("creating stats for channel %s (%s)" % (chan, time.ctime(timetarget))) userstats = StatDict() wordstats = StatDict() stop = False for f in logs[::-1]: filename = stripname(f) channel = stripname(chan) if not channel in filename: continue for line in open(chatlogdir + os.sep + filename, 'r'): splitted = line.strip().split() if len(splitted) < 2: continue who = "unknown" for i in splitted: if i.startswith("<"): who = i[1:-1] if what and who != what: continue timestr = "%s %s" % (splitted[0], splitted[1]) logtime = strtotime2(timestr) if logtime: if logtime > timetarget: userstats.upitem(who) else: continue else: userstats.upitem(who) for word in splitted[4:]: wordstats.upitem(word) if what: result = wordstats.top() else: result = userstats.top() if result: res = ["%s: %s" % item for item in result] event.reply("stat results for %s: " % (what or chan), res) else: event.reply("no result found for %s" % (what or chan))
def enablelogging(botname, channel): """ set loglevel to level_name. """ global loggers LOGDIR = initlog(getdatadir()) logging.warn("enabling on (%s,%s)" % (botname, channel)) chan = channel channel = stripname(channel) logname = "%s_%s" % (botname, channel) #if logname in loggers: logging.warn("there is already a logger for %s" % logname) ; return try: filehandler = logging.handlers.TimedRotatingFileHandler(LOGDIR + os.sep + logname + ".log", 'midnight') formatter = logging.Formatter(format) filehandler.setFormatter(formatter) except IOError: filehandler = None chatlogger = logging.getLoggerClass()(logname) chatlogger.setLevel(logging.INFO) if chatlogger.handlers: for handler in chatlogger.handlers: chatlogger.removeHandler(handler) if filehandler: chatlogger.addHandler(filehandler) ; logging.warn("%s - logging enabled on %s" % (botname, chan)) else: logging.error("no file handler found - not enabling logging.") global lastlogger lastlogger = chatlogger loggers[logname] = lastlogger
def setloglevel(level_name="warn", colors=True, datadir=None): """ set loglevel to level_name. """ if not level_name: return global level global filehandler from tl.lib.datadir import getdatadir LOGDIR = init(getdatadir()) format_short = "\033[1m%(asctime)-8s\033[0m -=- %(name)-1s -=- \033[93m%(message)-75s\033[0m -=- \033[92m%(module)s.%(funcName)s.%(lineno)s\033[0m -=- \033[94m%(threadName)s\033[0m" format_short_plain = "%(asctime)-8s -=- %(name)-1s -=- %(message)-75s -=- %(module)s.%(funcName)s.%(lineno)s -=- %(threadName)s" datefmt = '%H:%M:%S' formatter_short = logging.Formatter(format_short, datefmt=datefmt) formatter_short_plain = logging.Formatter(format_short_plain, datefmt=datefmt) try: filehandler = logging.handlers.TimedRotatingFileHandler(LOGDIR + os.sep + "tl.log", 'midnight') except (IOError, AttributeError) as ex: logging.error("can't create file loggger %s" % str(ex)) filehandler = None docolors = colors or False level = LEVELS.get(str(level_name).lower(), logging.NOTSET) root = logging.getLogger() root.addFilter(MyFilter()) root.setLevel(level) if root and root.handlers: for handler in root.handlers: root.removeHandler(handler) ch = logging.StreamHandler() ch.setLevel(level) if True: ch.addFilter(MyFilter()) if docolors: ch.setFormatter(formatter_short) else: ch.setFormatter(formatter_short_plain) if filehandler: filehandler.setLevel(level) filehandler.setFormatter(formatter_short_plain) root.addHandler(ch) if filehandler: root.addHandler(filehandler) logging.warn("loglevel is %s (%s)" % (str(level), level_name))
def __init__(self, cfg=None, usersin=None, plugs=None, botname=None, nick=None, bottype=None, ordered=False, *args, **kwargs): logging.debug("type is %s" % str(type(self))) if cfg: self.cfg = cfg ; botname = botname or self.cfg.name if not botname: botname = "default-%s" % str(type(self)).split('.')[-1][:-2] if not botname: raise Exception("can't determine botname") self.fleetdir = 'fleet' + os.sep + stripname(botname) if not self.cfg: self.cfg = Config(self.fleetdir + os.sep + 'config') self.cfg.name = botname or self.cfg.name if not self.cfg.name: raise Exception("name is not set in %s config file" % self.fleetdir) logging.debug("name is %s" % self.cfg.name) LazyDict.__init__(self) self.bid = get_bid(self) logging.warn("created bot on %s" % self.bid) logging.debug("created bot with config %s" % self.cfg.tojson(full=True)) self.ecounter = 0 self.ignore = [] self.ids = [] self.stats = StatDict() logging.warn("stats dict set to %s" % str(self.stats)) self.aliases = getaliases() self.reconnectcount = 0 self.plugs = coreplugs self.gatekeeper = GateKeeper(self.cfg.name) self.gatekeeper.allow(self.user or self.jid or self.cfg.server or self.cfg.name) self.starttime = time.time() self.type = bottype or "base" self.status = "init" self.networkname = self.cfg.networkname or self.cfg.name or "" from tl.lib.datadir import getdatadir datadir = getdatadir() self.datadir = datadir + os.sep + self.fleetdir self.maincfg = getmainconfig() if not self.cfg.owner: logging.debug("owner is not set in %s - using mainconfig" % self.cfg.cfile) self.cfg.owner = self.maincfg.owner self.users = usersin or getusers() logging.debug("owner is %s" % self.cfg.owner) self.users.make_owner(self.cfg.owner) self.outcache = outcache self.userhosts = LazyDict() self.nicks = LazyDict() self.connectok = threading.Event() self.reconnectcount = 0 self.cfg.nick = nick or self.cfg.nick or 'tl' try: if not os.isdir(self.datadir): os.mkdir(self.datadir) except: pass self.setstate() self.outputlock = _thread.allocate_lock() if ordered: self.outqueue = queue.PriorityQueue() self.eventqueue = queue.PriorityQueue() else: self.outqueue = queue.Queue() self.eventqueue = queue.Queue() logging.debug("event queues is %s" % str(self.eventqueue)) self.encoding = self.cfg.encoding or "utf-8" self.cmndperms = getcmndperms() self.outputmorphs = outputmorphs self.inputmorphs = inputmorphs tickloop.start(self)