def __init__(self, id, botname=None, type="notset"):
if not id: raise NoChannelSet()
if not botname: Persist.__init__(self, getdatadir() + os.sep + 'channels' + os.sep + stripname(id))
else: Persist.__init__(self, getdatadir() + os.sep + 'fleet' + os.sep + stripname(botname) + os.sep + 'channels' + os.sep + stripname(id))
self.id = id
self.type = type
self.lastmodified = time.time()
self.data.id = id
self.data.enable = self.data.enable or False
self.data.ops = self.data.ops or []
self.data.silentcommands = self.data.silentcommands or []
self.data.allowcommands = self.data.allowcommands or []
self.data.feeds = self.data.feeds or []
self.data.forwards = self.data.forwards or []
self.data.allowwatch = self.data.allowwatch or []
self.data.watched = self.data.watched or []
self.data.passwords = self.data.passwords or {}
self.data.cc = self.data.cc or "!"
self.data.nick = self.data.nick or "tl"
self.data.key = self.data.key or ""
self.data.denyplug = self.data.denyplug or []
self.data.createdfrom = whichmodule()
self.data.cacheindex = 0
self.data.tokens = self.data.tokens or []
self.data.webchannels = self.data.webchannels or []
def oldname(name):
from tl.lib.datadir import getdatadir
if name.startswith("-"): name[0] = "+"
name = name.replace("@", "+")
if os.path.exists(getdatadir() + os.sep + name): return name
name = name.replace("-", "#")
name = prevchan.replace("+", "@")
if os.path.exists(getdatadir() + os.sep + name): return name
return ""
def get_token(username):
global users
if not users: users = TwitterUsers(getdatadir() + os.sep + "twitter" + os.sep + "users")
if not users: raise Exception("can't get twitter users object") ; return
key, secret = getcreds(getdatadir())
if not key: raise Exception(getdatadir())
if not secret: raise Exception(getdatadir())
token = twittertoken(key, secret, users, username)
if not token: raise Exception("%s %s" % (str(user), username))
api = twitterapi(key, secret, token)
if not api: raise Exception("%s %s" % (str(user), name))
return (token, api)
def handle_chanupgrade(bot, event):
""" no arguments - upgrade the channel. """
prevchan = event.channel
# 0.4.1
if prevchan.startswith("-"): prevchan[0] = "+"
prevchan = prevchan.replace("@", "+")
prev = Persist(getdatadir() + os.sep + "channels" + os.sep + prevchan)
if prev.data: event.chan.data.update(prev.data) ; event.chan.save() ; event.reply("done")
else:
prevchan = event.channel
prevchan = prevchan.replace("-", "#")
prevchan = prevchan.replace("+", "@")
prev = Persist(getdatadir() + os.sep + "channels" + os.sep + prevchan)
if prev.data: event.chan.data.update(prev.data) ; event.chan.save() ; event.reply("done")
else: event.reply("can't find previous channel data")
def startmcdaemon():
#if not getmainconfig().memcached: return
logging.warn("starting memcached daemon")
try:
from tl.utils.popen import gozerpopen
rundir = getdatadir() + os.sep + 'run'
sock = os.path.abspath(rundir + os.sep + "memcached.socket")
logging.warn("using unix socket %s" % sock)
pidfile = sock[:-7] + ".pid"
if os.path.exists(sock) and isactive(sock): logging.warn("memcached daemon is already running") ; return
args = [[]]*4
args[0] = "memcached"
args[1] = "-s%s" % sock
args[2] = "-P%s" % pidfile
args[3] = "-d"
logging.debug("running %s" % " ".join(args))
proces = gozerpopen(args)
except Exception as ex:
if "No such file" in str(ex): logging.warn("no memcached installed")
else: logging.error('error running popen: %s' % str(ex))
return
data = proces.fromchild.readlines()
returncode = proces.close()
if returncode == 0: logging.warn("memcached started")
else: logging.warn("can't start memcached (%s)" % returncode)
def handle_twittercmnd(bot, ievent):
""" arguments: <API cmnd> - do a twitter API cmommand. """
go = getauth()
if not go: ievent.reply("the twitter plugin needs the credentials.py file in the .tl/config dir. see the examples dir") ; return
if not ievent.args: ievent.missing('<API cmnd>') ; return
target = strippedtxt(ievent.args[0])
try:
from tl.utils.twitter import get_token
token = get_token(ievent.user.data.name)
if not token: ievent.reply("you are not logged in yet .. run the twitter-auth command.") ; return
key, secret = getcreds(getdatadir())
token = tweepy.oauth.OAuthToken(key, secret).from_string(token)
twitter = twitterapi(key, secret, token)
cmndlist = dir(twitter)
cmnds = []
for cmnd in cmndlist:
if cmnd.startswith("_") or cmnd == "auth": continue
else: cmnds.append(cmnd)
if target not in cmnds: ievent.reply("choose one of: %s" % ", ".join(cmnds)) ; return
try: method = getattr(twitter, target)
except AttributeError: ievent.reply("choose one of: %s" % ", ".join(cmnds)) ; return
result = method()
res = []
for item in result:
try: res.append("%s - %s" % (item.screen_name, item.text))
except AttributeError:
try: res.append("%s - %s" % (item.screen_name, item.description))
except AttributeError:
try: res.append(str(item.__getstate__()))
except AttributeError: res.append(dir(i)) ; res.append(str(item))
ievent.reply("result of %s: " % target, res)
except KeyError: ievent.reply('you are not logged in yet. see the twitter-auth command.')
except (tweepy.TweepError, urllib.error.HTTPError) as e: ievent.reply('twitter failed: %s' % (str(e),))
def stripdatadir(fname):
res = []
ddir = getdatadir()
for i in fname.split(os.sep):
if i in ddir: continue
res.append(i)
return os.sep.join(res)
def getfleet(datadir=None, new=False):
if not datadir:
from tl.lib.datadir import getdatadir
datadir = getdatadir()
global fleet
if not fleet or new:
fleet = Fleet(datadir)
fleet.loadall()
return fleet
def getauth(datadir=None):
""" get auth structure from datadir. """
global auth
if auth: return auth
if not datadir: datadir = getdatadir()
try: key, secret = getcreds(datadir)
except RequireError as ex: logging.warn(str(ex)) ; return None
auth = tweepy.OAuthHandler(key, secret)
return auth
def getplus(target):
credentials = import_byfile("credentials", getdatadir() + os.sep + "config" + os.sep + "credentials.py")
url = "https://www.googleapis.com/plus/v1/people/%s/activities/public?alt=json&pp=1&key=%s" % (target, credentials.googleclient_apikey)
result = geturl2(url)
data = json.loads(result)
res = []
for item in data['items']:
i = LazyDict(item)
res.append("%s - %s - %s" % (i.actor['displayName'], i['title'], item['url']))
return res
def getaliases(ddir=None, force=True):
""" return global aliases. """
global aliases
if not aliases or force:
from tl.lib.persist import Persist
from tl.utils.lazydict import LazyDict
d = ddir or getdatadir()
p = Persist(d + os.sep + "run" + os.sep + "aliases")
if not p.data: p.data = LazyDict()
aliases = p.data
return aliases
def handle_twitter_confirm(bot, ievent):
""" arguments: <PIN code> - confirm auth with PIN. """
go = getauth()
if not go: ievent.reply("the twitter plugin needs the credentials.py file in the %s/config dir. see the examples directory" % getdatadir()) ; return
pin = ievent.args[0]
if not pin: ievent.missing("<PIN> .. see the twitter-auth command.") ; return
try: access_token = getauth(getdatadir()).get_access_token(pin)
except (tweepy.TweepError, urllib.error.HTTPError) as e: ievent.reply('twitter failed: %s' % (str(e),)) ; return
twitteruser = get_users()
twitteruser.add(ievent.user.data.name, access_token.to_string())
ievent.reply("%s access token saved." % ievent.user.data.name)
def __init__(self, name, url="", owner="", itemslist=['title', 'link'], watchchannels=[], running=1):
filebase = getdatadir() + os.sep + 'plugs' + os.sep + 'tl.plugs.extra.hubbub' + os.sep + name
Persist.__init__(self, filebase + os.sep + name + '.core')
if not self.data: self.data = {}
self.data = LazyDict(self.data)
self.data['name'] = self.data.name or str(name)
self.data['url'] = self.data.url or str(url)
self.data['owner'] = self.data.owner or str(owner)
self.data['watchchannels'] = self.data.watchchannels or list(watchchannels)
self.data['running'] = self.data.running or running
self.itemslists = Pdol(filebase + os.sep + name + '.itemslists')
self.markup = Pdod(filebase + os.sep + name + '.markup')
def savealiases(ddir=None):
""" return global aliases. """
global aliases
if aliases:
logging.warn("saving aliases")
from tl.lib.persist import Persist
from tl.utils.lazydict import LazyDict
d = ddir or getdatadir()
p = Persist(d + os.sep + "run" + os.sep + "aliases")
p.data = aliases
p.save()
return aliases
def mcboot():
#if not getmainconfig().memcached: return
logging.warn("memcached is enabled")
try:
import tl.contrib.memcache as memcache
rundir = getdatadir() + os.sep + 'run'
sock = os.path.abspath(rundir + os.sep + "memcached.socket")
global mc
mc = memcache.Client(["unix:%s" % sock], debug=0)
return isactive(sock)
except ImportError as ex: logging.warn("using builtin cache - %s" % str(ex))
except Exception as ex: logging.warn("error starting memcached client: %s" % str(ex))
def handle_twitter_auth(bot, ievent):
""" no arguments - get url to get the auth PIN needed for the twitter-confirm command. """
go = getauth()
if not go: ievent.reply("the twitter plugin needs the credentials.py file in the .tl/config dir. see the examples directory") ; return
try: auth_url = getauth(getdatadir()).get_authorization_url()
except (tweepy.TweepError, urllib.error.HTTPError) as e: ievent.reply('twitter failed: %s' % (str(e),)) ; return
if bot.type == "irc":
bot.say(ievent.nick, "sign in at %s" % auth_url)
bot.say(ievent.nick, "use the provided code in the twitter-confirm command.")
else:
ievent.reply("sign in at %s" % auth_url)
ievent.reply("use the provided code in the twitter-confirm command.")
def getcredsmod(datadir=None, doraise=False):
""" returnd credendtials.py as a module. """
if not datadir: datadir = getdatadir()
try:
mod = import_byfile("credentials", datadir + os.sep + "config" + os.sep + "credentials.py")
global go
go = True
except (IOError, ImportError):
if doraise: raise RequireError("credentials.py is needed in %s/config dir. see %s/examples" % (datadir, datadir))
else: logging.warn("credentials.py is needed in %s/config dir. see %s/examples" % (datadir, datadir))
return
logging.info("found %s credentials" % str(mod))
return mod
def markovlearnlog(chan):
""" learn a log """
lines = 0
logfiles = os.listdir(getdatadir() + os.sep + "chatlogs")
for filename in logfiles:
if chan[1:] not in filename:
continue
logging.warn("opening %s" % reversename(filename))
for line in open(getdatadir() + os.sep + "chatlogs" + os.sep + filename, "r"):
if lines % 10 == 0:
time.sleep(0.001)
if not line:
continue
lines += 1
try:
txt = " ".join(
line.strip().split()[2:]
) # log format is: 2011-08-07 00:02:16 <botfather> love, peace and happiness
markovtalk_learn(txt)
except IndexError:
continue
logging.warn("learning %s log done. %s lines" % (chan, lines))
return lines
def handle_chatlogsearch(bot, event):
""" arguments: <searchtxt> - search in the logs. """
if not event.rest: event.missing("<searchtxt>") ; return
result = []
chatlogdir = getdatadir() + os.sep + "chatlogs"
if event.options and event.options.channel: chan = event.options.channel
else: chan = event.channel
logs = os.listdir(chatlogdir)
logs.sort()
for f in logs:
filename = stripname(f)
if not stripname(chan) in filename: continue
for line in open(chatlogdir + os.sep + filename, 'r'):
if event.rest in line: result.append(line)
if result: event.reply("search results for %s: " % event.rest, result, dot= " || ")
else: event.reply("no result found for %s" % chan)
def __init__(self, dbname=None, dbhost=None, dbuser=None, dbpasswd=None, dbtype=None, ddir=None, doconnect=True):
self.datadir = ddir or getdatadir()
self.datadir = self.datadir + os.sep + "db" + os.sep
if hasattr(os, 'mkdir'):
if not os.path.isdir(self.datadir):
try: os.mkdir(self.datadir)
except OSError: pass
cfg = getmainconfig()
self.dbname = dbname or cfg.dbname
if not self.dbname: raise Exception("no db name")
self.dbhost = dbhost or cfg.dbhost or ""
self.dbuser = dbuser or cfg.dbuser or ""
self.dbpasswd = dbpasswd or cfg.dbpasswd or ""
self.connection = None
self.timeout = 15
self.dbtype = dbtype or cfg.dbtype or 'sqlite'
if doconnect: self.connect()
def runapiserver(port=None, ddir=None):
""" start running the API server. needs to be called from the main thread. """
from tl.drivers.tornado.bot import TornadoBot
global bot
bot = TornadoBot(botname="api-bot")
if port:
try: port = int(port)
except ValueError: pass
else: port = 10105
try:
server = createserver(ddir or getdatadir())
server.bind(port)
logging.warn("starting API server on port %s" % port)
server.start()
server.io_loop.start()
except KeyboardInterrupt: globalshutdown()
except Exception as ex: handle_exception() ; os._exit(1)
else: globalshutdown()
def killmcdaemon():
#if not getmainconfig().memcached: return
logging.warn("killing memcached daemon")
rundir = getdatadir() + os.sep + 'run'
sock = os.path.abspath(rundir + os.sep + "memcached.socket")
pidfile = sock[:-7] + ".pid"
try: pid = int(open(pidfile, "r").read().strip())
except Exception as ex : logging.warn("can't determine pid of memcached from %s - %s" % (pidfile,str(ex))) ; return False
logging.warn("pid is %s" % pid)
data = isactive(sock)
if not data: logging.warn("memcached is not runniing") ; return False
try: curr_connections = int(data[0][1]["curr_connections"])
except Exception as ex: logging.warn("can't determine current connections of memcached .. not killing - %s" % str(ex)) ; return False
if curr_connections and curr_connections != 1: logging.warn("current connections of memcached is %s .. not killing" % curr_connections) ; return False
try: os.kill(pid, 15) ; logging.warn("killed memcached with pid %s" % pid)
except Exception as ex: logging.warn("failed to kill memcached (%s) - %s" % (pid, str(ex)))
try: os.remove(pidfile)
except: pass
def handle_twitterfriends(bot, ievent):
""" no arguments - show friends timeline (your normal twitter feed). """
go = getauth()
if not go: ievent.reply("the twitter plugin needs the credentials.py file in the .tl/config dir. see the examples directory") ; return
try:
token = get_token(ievent.user.data.name)
if not token: ievent.reply("you are not logged in yet .. run the twitter-auth command.") ; return
key , secret = getcreds(getdatadir())
token = tweepy.oauth.OAuthToken(key, secret).from_string(token)
twitter = twitterapi(key, secret, token)
method = getattr(twitter, "friends_timeline")
result = method()
res = []
for item in result:
try: res.append("%s - %s" % (item.author.screen_name, item.text))
except Exception as ex: handle_exception()
ievent.reply("results: ", res)
except KeyError: ievent.reply('you are not logged in yet. see the twitter-auth command.')
except (tweepy.TweepError, urllib.error.HTTPError) as e: ievent.reply('twitter failed: %s' % (str(e),))
def checkconfig():
changed = []
d = getdatadir() + os.sep + "config"
if not os.path.isdir(d): return changed
for f in os.listdir(d):
if os.path.isdir(d + os.sep + f):
dname = d + os.sep + f
changed.extend(checktimestamps(d + os.sep + f))
continue
m = d + os.sep + f
if os.path.isdir(m): continue
if "__init__" in f: continue
global timestamps
try:
t = os.path.getmtime(m)
if t > timestamps.data[m]: changed.append(m) ; timestamps.data[m] = t ;
except KeyError: timestamps.data[m] = os.path.getmtime(m) ; changed.append(m)
if changed: timestamps.save()
return changed
def markovlearnspider(target):
logging.warn("starting spider learn on %s" % target)
coll = PersistCollection(getdatadir() + os.sep + "spider" + os.sep + "data")
if target.startswith("spider://"):
target = target[9:]
objs = coll.search("url", target)
for obj in objs:
if not obj.data and obj.data.url:
continue
time.sleep(0.001)
if target not in obj.data.url:
continue
logging.warn("url is %s" % obj.data.url)
try:
if obj.data and obj.data.txt:
for line in obj.data.txt.split("\n"):
if line.count(";") > 1:
continue
markovtalk_learn(striphtml(line))
except:
handle_exception()
def handle_search(bot, event):
if not event.options: event.makeoptions()
all = event.options.all
res = []
target = event.args
if not target: event.missing("<search words seperated by space>") ; return
coll = PersistCollection(getdatadir() + os.sep + 'spider' + os.sep + "data")
files = coll.filenames(target)
if files:
for f in files:
try: res.append(Persist(f).data.url)
except AttributeError as ex: continue
objs = coll.search('txt', event.rest)
if not objs: objs = list(coll.objects().values())
stats = makestats(objs, target, res)
urls = stats_response(stats, target)
res.extend(urls)
if res:
if len(res) < 4 or all: event.reply("found %s urls: " % len(res), res, dot=" -or- ")
else: event.reply("found %s urls, use --all for more: " % len(res), res[:3], dot=" -or- ")
else: event.reply("no urls found")
def handle_chatlogstats(bot, event):
""" no arguments - create log stats of the channel, possible options: --chan <channel> """
what = event.rest.strip()
chatlogdir = getdatadir() + os.sep + "chatlogs"
if event.options and event.options.channel: chan = event.options.channel
else: chan = event.channel
logs = os.listdir(chatlogdir)
if not logs: event.reply("no logs available for %s" % chan) ; return
now = time.time()
if what: timetarget = strtotime2(what) ; what = striptime(what)
else: timetarget = 0 ; what = None
event.reply("creating stats for channel %s (%s)" % (chan, time.ctime(timetarget)))
userstats = StatDict()
wordstats = StatDict()
stop = False
for f in logs[::-1]:
filename = stripname(f)
channel = stripname(chan)
if not channel in filename: continue
for line in open(chatlogdir + os.sep + filename, 'r'):
splitted = line.strip().split()
if len(splitted) < 2: continue
who = "unknown"
for i in splitted:
if i.startswith("<"): who = i[1:-1]
if what and who != what: continue
timestr = "%s %s" % (splitted[0], splitted[1])
logtime = strtotime2(timestr)
if logtime:
if logtime > timetarget: userstats.upitem(who)
else: continue
else: userstats.upitem(who)
for word in splitted[4:]: wordstats.upitem(word)
if what: result = wordstats.top()
else: result = userstats.top()
if result:
res = ["%s: %s" % item for item in result]
event.reply("stat results for %s: " % (what or chan), res)
else: event.reply("no result found for %s" % (what or chan))
def enablelogging(botname, channel):
""" set loglevel to level_name. """
global loggers
LOGDIR = initlog(getdatadir())
logging.warn("enabling on (%s,%s)" % (botname, channel))
chan = channel
channel = stripname(channel)
logname = "%s_%s" % (botname, channel)
#if logname in loggers: logging.warn("there is already a logger for %s" % logname) ; return
try:
filehandler = logging.handlers.TimedRotatingFileHandler(LOGDIR + os.sep + logname + ".log", 'midnight')
formatter = logging.Formatter(format)
filehandler.setFormatter(formatter)
except IOError:
filehandler = None
chatlogger = logging.getLoggerClass()(logname)
chatlogger.setLevel(logging.INFO)
if chatlogger.handlers:
for handler in chatlogger.handlers: chatlogger.removeHandler(handler)
if filehandler: chatlogger.addHandler(filehandler) ; logging.warn("%s - logging enabled on %s" % (botname, chan))
else: logging.error("no file handler found - not enabling logging.")
global lastlogger
lastlogger = chatlogger
loggers[logname] = lastlogger
def setloglevel(level_name="warn", colors=True, datadir=None):
""" set loglevel to level_name. """
if not level_name: return
global level
global filehandler
from tl.lib.datadir import getdatadir
LOGDIR = init(getdatadir())
format_short = "\033[1m%(asctime)-8s\033[0m -=- %(name)-1s -=- \033[93m%(message)-75s\033[0m -=- \033[92m%(module)s.%(funcName)s.%(lineno)s\033[0m -=- \033[94m%(threadName)s\033[0m"
format_short_plain = "%(asctime)-8s -=- %(name)-1s -=- %(message)-75s -=- %(module)s.%(funcName)s.%(lineno)s -=- %(threadName)s"
datefmt = '%H:%M:%S'
formatter_short = logging.Formatter(format_short, datefmt=datefmt)
formatter_short_plain = logging.Formatter(format_short_plain, datefmt=datefmt)
try:
filehandler = logging.handlers.TimedRotatingFileHandler(LOGDIR + os.sep + "tl.log", 'midnight')
except (IOError, AttributeError) as ex:
logging.error("can't create file loggger %s" % str(ex))
filehandler = None
docolors = colors or False
level = LEVELS.get(str(level_name).lower(), logging.NOTSET)
root = logging.getLogger()
root.addFilter(MyFilter())
root.setLevel(level)
if root and root.handlers:
for handler in root.handlers: root.removeHandler(handler)
ch = logging.StreamHandler()
ch.setLevel(level)
if True:
ch.addFilter(MyFilter())
if docolors: ch.setFormatter(formatter_short)
else: ch.setFormatter(formatter_short_plain)
if filehandler:
filehandler.setLevel(level)
filehandler.setFormatter(formatter_short_plain)
root.addHandler(ch)
if filehandler: root.addHandler(filehandler)
logging.warn("loglevel is %s (%s)" % (str(level), level_name))
def __init__(self, cfg=None, usersin=None, plugs=None, botname=None, nick=None, bottype=None, ordered=False, *args, **kwargs):
logging.debug("type is %s" % str(type(self)))
if cfg: self.cfg = cfg ; botname = botname or self.cfg.name
if not botname: botname = "default-%s" % str(type(self)).split('.')[-1][:-2]
if not botname: raise Exception("can't determine botname")
self.fleetdir = 'fleet' + os.sep + stripname(botname)
if not self.cfg: self.cfg = Config(self.fleetdir + os.sep + 'config')
self.cfg.name = botname or self.cfg.name
if not self.cfg.name: raise Exception("name is not set in %s config file" % self.fleetdir)
logging.debug("name is %s" % self.cfg.name)
LazyDict.__init__(self)
self.bid = get_bid(self)
logging.warn("created bot on %s" % self.bid)
logging.debug("created bot with config %s" % self.cfg.tojson(full=True))
self.ecounter = 0
self.ignore = []
self.ids = []
self.stats = StatDict()
logging.warn("stats dict set to %s" % str(self.stats))
self.aliases = getaliases()
self.reconnectcount = 0
self.plugs = coreplugs
self.gatekeeper = GateKeeper(self.cfg.name)
self.gatekeeper.allow(self.user or self.jid or self.cfg.server or self.cfg.name)
self.starttime = time.time()
self.type = bottype or "base"
self.status = "init"
self.networkname = self.cfg.networkname or self.cfg.name or ""
from tl.lib.datadir import getdatadir
datadir = getdatadir()
self.datadir = datadir + os.sep + self.fleetdir
self.maincfg = getmainconfig()
if not self.cfg.owner:
logging.debug("owner is not set in %s - using mainconfig" % self.cfg.cfile)
self.cfg.owner = self.maincfg.owner
self.users = usersin or getusers()
logging.debug("owner is %s" % self.cfg.owner)
self.users.make_owner(self.cfg.owner)
self.outcache = outcache
self.userhosts = LazyDict()
self.nicks = LazyDict()
self.connectok = threading.Event()
self.reconnectcount = 0
self.cfg.nick = nick or self.cfg.nick or 'tl'
try:
if not os.isdir(self.datadir): os.mkdir(self.datadir)
except: pass
self.setstate()
self.outputlock = _thread.allocate_lock()
if ordered:
self.outqueue = queue.PriorityQueue()
self.eventqueue = queue.PriorityQueue()
else:
self.outqueue = queue.Queue()
self.eventqueue = queue.Queue()
logging.debug("event queues is %s" % str(self.eventqueue))
self.encoding = self.cfg.encoding or "utf-8"
self.cmndperms = getcmndperms()
self.outputmorphs = outputmorphs
self.inputmorphs = inputmorphs
tickloop.start(self)
