def __init__(self, id, botname=None, type="notset"):
if not id: raise NoChannelSet()
if not botname: Persist.__init__(self, getdatadir() + os.sep + 'channels' + os.sep + stripname(id))
else: Persist.__init__(self, getdatadir() + os.sep + 'fleet' + os.sep + stripname(botname) + os.sep + 'channels' + os.sep + stripname(id))
self.id = id
self.type = type
self.lastmodified = time.time()
self.data.id = id
self.data.enable = self.data.enable or False
self.data.ops = self.data.ops or []
self.data.silentcommands = self.data.silentcommands or []
self.data.allowcommands = self.data.allowcommands or []
self.data.feeds = self.data.feeds or []
self.data.forwards = self.data.forwards or []
self.data.allowwatch = self.data.allowwatch or []
self.data.watched = self.data.watched or []
self.data.passwords = self.data.passwords or {}
self.data.cc = self.data.cc or "!"
self.data.nick = self.data.nick or "tl"
self.data.key = self.data.key or ""
self.data.denyplug = self.data.denyplug or []
self.data.createdfrom = whichmodule()
self.data.cacheindex = 0
self.data.tokens = self.data.tokens or []
self.data.webchannels = self.data.webchannels or []
def save(self, attributes=[]):
target = {}
if attributes:
for key in attributes: target[key] = self[key]
else: target = cpy(self)
targetfile = getdatadir() + os.sep + "containers" + os.sep + str(self.createtime) + "_" + stripname(self.origin)
p = Persist(targetfile)
p.data = getjson().dumps(target)
p.save()
def __init__(self, fname):
Persist.__init__(self, fname)
if type(self.data) == list:
tmp = {}
for i in range(self.data):
tmp[i] = self.data[i]
self.data = tmp
for i in self.data:
z = Alarmitem(d=self.data[i])
periodical.addjob(z.time - time.time(), 1, self.alarmsay, z.nick, z)
def getaliases(ddir=None, force=True):
""" return global aliases. """
global aliases
if not aliases or force:
from tl.lib.persist import Persist
from tl.utils.lazydict import LazyDict
d = ddir or getdatadir()
p = Persist(d + os.sep + "run" + os.sep + "aliases")
if not p.data: p.data = LazyDict()
aliases = p.data
return aliases
def savealiases(ddir=None):
""" return global aliases. """
global aliases
if aliases:
logging.warn("saving aliases")
from tl.lib.persist import Persist
from tl.utils.lazydict import LazyDict
d = ddir or getdatadir()
p = Persist(d + os.sep + "run" + os.sep + "aliases")
p.data = aliases
p.save()
return aliases
def __init__(self, name, url="", owner="", itemslist=['title', 'link'], watchchannels=[], running=1):
filebase = getdatadir() + os.sep + 'plugs' + os.sep + 'tl.plugs.extra.hubbub' + os.sep + name
Persist.__init__(self, filebase + os.sep + name + '.core')
if not self.data: self.data = {}
self.data = LazyDict(self.data)
self.data['name'] = self.data.name or str(name)
self.data['url'] = self.data.url or str(url)
self.data['owner'] = self.data.owner or str(owner)
self.data['watchchannels'] = self.data.watchchannels or list(watchchannels)
self.data['running'] = self.data.running or running
self.itemslists = Pdol(filebase + os.sep + name + '.itemslists')
self.markup = Pdod(filebase + os.sep + name + '.markup')
def todb(self):
""" save config to database. """
cp = dict(self)
del cp['jsondb']
if not self.jsondb:
from tl.lib.persist import Persist
self.jsondb = Persist(self._cfile)
self.jsondb.data = cp
self.jsondb.save()
def __init__(self, keyname): Persist.__init__(self, getdatadir() + os.sep + 'keys' + os.sep + 'fish' + os.sep + stripname(keyname))
def __init__(self, url, txt=None, *args, **kwargs):
Persist.__init__(self, getdatadir() + os.sep + "spider" + os.sep + "data" + os.sep + stripname(url), *args, **kwargs)
self.data.url = url
self.data.txt = txt or self.data.txt or ""
class Config(LazyDict):
"""
config class is a dict containing json strings. is writable to file
and human editable.
"""
def __init__(self, filename, verbose=False, input={}, ddir=None, nolog=False, *args, **kwargs):
assert filename
if ddir: ddir = normdir(ddir)
if input: LazyDict.__init__(self, input, *args, **kwargs)
else: LazyDict.__init__(self, *args, **kwargs)
self.filename = filename
self.setcfile(ddir, filename)
if not self._comments: self._comments = {}
logging.info("%s from %s" % (self._logname, whichmodule(2)))
self.fromfile(self._cfile)
self.cid = get_cid(self)
self.init()
if self.owner: logging.info("owner is %s" % self.owner)
if "uuid" not in self: self.setuuid()
if not self._origdir in self._cfile: raise WrongFileName("%s not in %s" % (self._origdir, self._cfile))
def setcfile(self, ddir, filename):
self._origname = filename
self._logname = "config" + os.sep + filename
self._origdir = ddir or getdatadir()
self._filename = filename or 'mainconfig'
self._datadir = ddir or getdatadir()
if self._datadir[-1] != os.sep: self._datadir += os.sep
self._dir = self._datadir + 'config'
self._cfile = self._dir + os.sep + self._filename
logging.debug("self._cfile is %s" % self._cfile)
self.cfile = self._cfile
def setuuid(self, save=True):
logging.debug("setting uuid")
self.uuid = str(uuid.uuid4())
if save: self.save()
def __deepcopy__(self, a):
""" accessor function. """
cfg = Config(self._filename, input=self, nolog=True)
return cfg
def __getitem__(self, item):
""" accessor function. """
if item not in self: return None
else: return LazyDict.__getitem__(self, item)
def merge(self, cfg):
""" merge in another cfg. """
self.update(cfg)
def set(self, item, value):
""" set item to value. """
LazyDict.__setitem__(self, item, value)
def fromdb(self):
""" read config from database. """
from tl.lib.persist import Persist
tmp = Persist(self._cfile)
logging.debug("fromdb - %s - %s" % (self._cfile, tmp.data.tojson()))
self.update(tmp.data)
def todb(self):
""" save config to database. """
cp = dict(self)
del cp['jsondb']
if not self.jsondb:
from tl.lib.persist import Persist
self.jsondb = Persist(self._cfile)
self.jsondb.data = cp
self.jsondb.save()
def fromfile(self, filename=None):
""" read config object from filename. """
curline = ""
fname = filename or self._cfile
if not fname: raise Exception("no name found - %s - %s" % (self._cfile, self.dump()))
if not os.path.exists(fname): logging.warn("config file %s doesn't exist yet" % fname) ; return False
comment = ""
for line in open(fname, 'r'):
curline = line
curline = curline.strip()
if curline == "": continue
if curline.startswith('#'): comment = curline; continue
if True:
try:
key, value = curline.split('=', 1)
kkey = key.strip()
self[kkey] = json.loads(str(value.strip()))
if comment:
if not self._comments: self._comments = {}
self._comments[kkey] = comment
comment = ""
except ValueError: logging.error("skipping line - unable to parse: %s" % line)
#self.cfile = fname
return
def tofile(self, filename=None, stdout=False):
""" save config object to file. """
if not filename: filename = self._cfile
if not filename: raise Exception("no cfile found - %s" % whichmodule(3))
logging.warn("%s saved" % self._logname)
if filename.startswith(os.sep): d = [os.sep,]
else: d = []
for p in filename.split(os.sep)[:-1]:
if not p: continue
d.append(p)
ddir = os.sep.join(d)
if not os.path.isdir(ddir):
logging.debug("persist - creating %s dir" % ddir)
try: os.mkdir(ddir)
except OSError as ex:
logging.error("persist - not saving - failed to make %s - %s" % (ddir, str(ex)))
return
written = []
curitem = None
later = []
try:
if stdout: configtmp = sys.stdout
else: configtmp = open(filename + '.tmp', 'w')
configtmp.write('# ===========================================================\n#\n')
configtmp.write("# T I M E L I N E CONFIGURATION FILE - %s\n" % filename)
configtmp.write("#\n")
configtmp.write('# last changed on %s\n#\n' % time.ctime(time.time()))
configtmp.write("# This file contains configration data for the T I M E L I N E.\n")
configtmp.write('# Variables are defined by "name = json value" pairs.\n')
configtmp.write('# Make sure to use " in strings.\n#\n')
configtmp.write('# The bot can edit this file!.\n#\n')
configtmp.write('# ===========================================================\n\n')
teller = 0
keywords = list(self.keys())
keywords.sort()
for keyword in keywords:
value = self[keyword]
if keyword in written: continue
if keyword.startswith("_"): continue
if keyword in ['origdir', 'origname', 'issaved', 'blacklist', 'whitelist', 'followlist', 'uuid', 'whitelist', 'datadir', 'name', 'createdfrom', 'cfile', 'filename', 'dir', 'isdb']: later.append(keyword) ; continue
if keyword == 'jsondb': continue
if keyword == 'optionslist': continue
if keyword == 'gatekeeper': continue
if keyword == "_comments": continue
if self._comments and keyword in self._comments:
configtmp.write(self._comments[keyword] + "\n")
curitem = keyword
try: configtmp.write('%s = %s\n' % (keyword, json.dumps(value)))
except TypeError: logging.error("%s - can't serialize %s" % (filename, keyword)) ; continue
teller += 1
#configtmp.write("\n")
configtmp.write('\n\n# ============================================================\n#\n')
configtmp.write("# bot generated stuff.\n#\n")
configtmp.write('# ============================================================\n\n')
for keyword in later:
if self._comments and keyword in self._comments:
configtmp.write(self._comments[keyword] + "\n")
curitem = keyword
value = self[keyword]
try: configtmp.write(keyword + " = " + json.dumps(value) + "\n")
except TypeError: logging.error("%s - can't serialize %s" % (filename, keyword)) ; continue
teller += 1
#configtmp.write("\n")
if not "mainconfig" in filename and self._comments:
try:
configtmp.write('\n\n# ============================================================\n#\n')
configtmp.write("# possible other config variables.\n#\n")
configtmp.write('# ============================================================\n\n')
items = list(self._comments.keys())
keys = list(self.keys())
do = []
for var in items:
if var not in keys: do.append(var)
do.sort()
for var in do:
configtmp.write("# %s -=- %s\n" % (var, self._comments[var]))
configtmp.write("\n\n")
except Exception as ex: handle_exception()
else: configtmp.write("\n\n# tl can run multiple bots at once. see %s/config/fleet for their configurations.\n\n" % self.origdir)
if not stdout:
configtmp.close()
os.rename(filename + '.tmp', filename)
return teller
except Exception as ex:
handle_exception()
logging.error("ERROR WRITING %s CONFIG FILE: %s .. %s" % (filename, str(ex), curitem))
@savelocked
def save(self):
""" save the config. """
logging.info("save called from %s" % calledfrom(sys._getframe(2)))
self.issaved = True
if self.isdb: self.todb()
else: self.tofile(self._cfile)
def load_config(self, verbose=False):
""" load the config file. """
if self.isdb: self.fromdb()
else: self.fromfile(self._filename)
self.init()
if verbose: logging.debug('%s' % self.dump())
def init(self):
""" initialize the config object. """
if not self._comments: self._comments = {}
if self._filename == 'mainconfig':
self._comments["whitelist"] = "# - whitelist used to allow ips .. bot maintains this"
self.setdefault("whitelist", [])
self._comments["blacklist"] = "# - blacklist used to deny ips .. bot maintains this"
self.setdefault("blacklist", [])
self.setdefault('owner', [])
self._comments["loglist"] = "# - loglist .. maintained by the bot."
self.setdefault('loglist', [])
self._comments["loglevel"] = "# - loglevel of all bots"
self.setdefault('loglevel', "warn")
self._comments["loadlist"] = "# - loadlist .. not used yet."
self.setdefault('loadlist', [])
self._comments["quitmsg"] = "# - message to send on quit"
self.setdefault('quitmsg', "http://github.com/feedbackflow/tl")
self._comments["dotchars"] = "# - characters to used as seperator."
self.setdefault('dotchars', ", ")
self._comments["floodallow"] = "# - whether the bot is allowed to flood."
self.setdefault('floodallow', 1)
self._comments["auto_register"] = "# - enable automatic registration of new users."
self.setdefault('auto_register', 0)
self._comments["guestasuser"] = "******"
self.setdefault('guestasuser', 0)
self._comments["globalcc"] = "# - global control character"
self.setdefault('globalcc', "")
self._comments["app_id"] = "# - application id used by appengine."
self.setdefault('app_id', "tl")
self._comments["appname"] = "# - application name as used by the bot."
self.setdefault('appname', "T I M E L I N E")
self._comments["domain"] = "# - domain .. used for WAVE."
self.setdefault('domain', "")
self._comments["color"] = "# - color used in the webconsole."
self.setdefault('color', "")
self._comments["colors"] = "# - enable colors in logging."
self.setdefault('colors', "")
self._comments["memcached"] = "# - enable memcached."
self.setdefault('memcached', 0)
self._comments["allowrc"] = "# - allow execution of rc files."
self.setdefault('allowrc', 0)
self._comments["allowremoterc"] = "# - allow execution of remote rc files."
self.setdefault('allowremoterc', 0)
self._comments['dbenable'] = "# - enable database support"
self.setdefault('dbenable', 1)
self._comments['dbtype'] = "# - type of database .. sqlite or mysql at this time."
self.setdefault('dbtype', 'sqlite')
self._comments['dbname'] = "# - database name"
self.setdefault('dbname', "main.db")
self._comments['dbhost'] = "# - database hostname"
self.setdefault('dbhost', "localhost")
self._comments['dbuser'] = "******"
self.setdefault('dbuser', "bart")
self._comments['dbpasswd'] = "# - database password"
self.setdefault('dbpasswd', "mekker2")
self._comments['ticksleep'] = "# - nr of seconds to sleep before creating a TICK event."
self.setdefault('ticksleep', 1)
self._comments['bindhost'] = "# - host to bind to"
self.setdefault("bindhost", "")
self['createdfrom'] = whichmodule()
if 'xmpp' in self._cfile: self.setdefault('fulljids', 1)
if 'fleet' in self._cfile:
self.setdefault('nick', "timeliner")
self.setdefault('disable', 1)
self.setdefault("owner", [])
self.setdefault("user", "")
self.setdefault("host", "")
self.setdefault("server", "")
self.setdefault("ssl", 0)
self.setdefault("ipv6", 0)
self.setdefault("channels", [])
self.setdefault("port", "")
self.setdefault("password", "")
self._comments['datadir'] = "# - directory to store bot data in."
self._comments["owner"] = "# - owner of the bot."
self._comments["uuid"] = "# - bot generated uuid for this config file."
self._comments["user"] = "******"
self._comments["host"] = "# - host part of the user, derived from user var."
self._comments["server"] = "# - server to connect to (only when different from users host)."
self._comments["password"] = "******"
self._comments["port"] = "# - port to connect to (IRC)."
self._comments["ssl"] = "# - whether to enable ssl (set to 1 to enable)."
self._comments["ipv6"] = "# - whether to enable ssl (set to 1 to enable)."
self._comments["name"] = "# - the name of the bot."
self._comments["disable"] = "# - set this to 0 to enable the bot."
self._comments["followlist"] = "# - who to follow on the bot .. bot maintains this list."
self._comments["networkname"] = "# - networkname .. not used right now."
self._comments["type"] = "# - the bot's type."
self._comments["nick"] = "# - the bot's nick."
self._comments["channels"] = "# - channels to join."
self._comments["cfile"] = "# - filename of this config file. edit this when you move this file."
self._comments["createdfrom"] = "# - function that created this config file. bot generated"
self._comments["dir"] = "# - directory in which this config file lives."
self._comments["isdb"] = "# - whether this config file lives in the database and not on file."
self._comments["filename"] = "# - filename of this config file."
self._comments["username"] = "******"
self._comments["fulljids"] = "# - use fulljids of bot users (used in non anonymous conferences."
self._comments["servermodes"] = "# - string of modes to send to the server after connect."
self._comments["realname"] = "# - name used in the ident of the bot."
self._comments["onconnect"] = "# - string to send to server after connect."
self._comments["onconnectmode"] = "# - MODE string to send to server after connect."
self._comments["realname"] = "# - mode string to send to the server after connect."
self._comments["issaved"] = "# - whether this config file has been saved. "
self._comments["origdir"] = "# - original datadir for this configfile. "
self._comments["origname"] = "# - displayable name of the config file name. "
return self
def reload(self):
""" reload the config file. """
self.load_config()
return self
def boot(ddir=None, force=False, encoding="utf-8", umask=None, saveperms=True, fast=False, clear=False, loadall=False):
""" initialize the bot. """
global plugin_packages
try:
if os.getuid() == 0: print("don't run the bot as root") ; os._exit(1)
except AttributeError: pass
from tl.lib.datadir import getdatadir, setdatadir
if ddir: setdatadir(ddir)
origdir = ddir
ddir = getdatadir()
if not ddir: logging.error("can't determine datadir to boot from") ; raise Exception("can't determine datadir")
logging.warn("starting !!")
if not ddir in sys.path: sys.path.append(ddir)
makedirs(ddir)
try:
rundir = ddir + os.sep + "run"
k = open(rundir + os.sep + 'tl.pid','w')
k.write(str(os.getpid()))
k.close()
except IOError: pass
try:
if not umask: checkpermissions(getdatadir(), 0o700)
else: checkpermissions(getdatadir(), umask)
except: handle_exception()
from tl.lib.plugins import plugs
global loaded
global cmndtable
global retable
global plugins
global callbacktable
global shorttable
global cmndperms
global timestamps
if not retable: retable = Persist(rundir + os.sep + 'retable')
if clear: retable.data = {}
if not cmndtable: cmndtable = Persist(rundir + os.sep + 'cmndtable')
if clear: cmndtable.data = {}
if not plugins: plugins = Persist(rundir + os.sep + 'plugins')
if clear: plugins.data = {}
if not plugins.data.available: plugins.data.available = []
if not plugins.data.refused: plugins.data.refused = []
if not plugins.data.allowed: plugins.data.allowed = []
if not callbacktable: callbacktable = Persist(rundir + os.sep + 'callbacktable')
if clear: callbacktable.data = {}
if not shorttable: shorttable = Persist(rundir + os.sep + 'shorttable')
if clear: shorttable.data = {}
if not timestamps: timestamps = Persist(rundir + os.sep + 'timestamps')
if not cmndperms: cmndperms = Config('cmndperms', ddir=ddir)
changed = []
gotlocal = False
dosave = clear or False
maincfg = getmainconfig(ddir=ddir)
logging.warn("mainconfig used is %s" % maincfg.cfile)
packages = find_packages(ddir + os.sep + "myplugs")
for p in packages:
if p not in plugin_packages: plugin_packages.append(p)
packages = find_packages("tl" + os.sep + "plugs")
for p in packages:
if p not in plugin_packages: plugin_packages.append(p)
if os.path.isdir('tl'):
gotlocal = True
changed = scandir('tl-myplugs')
if changed:
logging.warn("tl-myplugs has changed -=- %s" % ", ".join(changed))
dosave = True
for plug in default_plugins: plugs.reload(plug, showerror=True, force=True)
changed = scandir(ddir + os.sep + 'myplugs')
if changed:
logging.warn("myplugs has changed -=- %s" % ', '.join(changed))
dosave = True
configchanges = checkconfig()
if configchanges:
logging.info("there are configuration changes: %s" % ', '.join(configchanges))
for f in configchanges:
if 'mainconfig' in f: force = True ; dosave = True
if os.path.isdir('tl'):
coreplugs = scandir("tl" + os.sep + "plugs")
if coreplugs:
logging.warn("core changed -=- %s" % ", ".join(coreplugs))
dosave = True
try:
from tl.db import getmaindb
from tl.db.tables import tablestxt
db = getmaindb()
if db: db.define(tablestxt)
except Exception as ex: logging.warn("could not initialize database %s" % str(ex))
if force or dosave or not cmndtable.data or len(cmndtable.data) < 100:
logging.debug("using target %s" % str(plugin_packages))
plugs.loadall(plugin_packages, force=True)
savecmndtable(saveperms=saveperms)
saveplugins()
savecallbacktable()
savealiases()
logging.warn(getfullversion(getdatadir()))
logging.warn("READY")
def __init__(self, fname):
Persist.__init__(self, fname)
if not self.data: self.data = {}
def __init__(self, url):
self.scantime = 0
self.url = Url(url)
self.fname = getdatadir() + os.sep + 'spider' + os.sep + 'stats' + os.sep + stripname(url)
Persist.__init__(self, self.fname)
def save(self):
""" save the hubbub items data. """
Persist.save(self)
self.itemslists.save()
self.markup.save()
def __init__(self, name):
self.name = name
if not os.path.exists(getdatadir() + os.sep +'gatekeeper'): os.mkdir(getdatadir() + os.sep + 'gatekeeper')
Persist.__init__(self, getdatadir() + os.sep + 'gatekeeper' + os.sep + name)
self.data.whitelist = self.data.whitelist or []