def rss(event):
o = Object(event)
o.prefix = "rss"
o.rss = event.rest
o.service = "rss"
path = o.save()
event.ok(1)
def first(self, *args, **kwargs):
fns = self.all(*args, **kwargs)
for fn in fns:
obj = Object().load(fn)
if "deleted" in obj and obj.deleted: continue
if args and len(args) > 1 and args[1] != obj.get(args[0], ""): continue
return obj
def test_iter(self):
o1 = Object()
o1.bla1 = 1
o1.bla2 = 2
o1.bla3 = 3
res = sorted(list(o1))
self.assertEqual(res, ["bla1","bla2","bla3"])
def ps(event):
res = []
nr = 1
for thr in sorted(launcher.running(), key=lambda x: name(x)):
obj = Object()
obj.update(vars(thr))
try:
obj = obj.__class__.__self__
except:
pass
if "sleep" in obj:
next = int(obj.sleep) - int(time.time() - int(obj._last))
else:
next = 0
thrname = naam(thr)
try:
stxt = obj._status.strip()
except:
stxt = "working"
try:
etxt = obj._error.strip()
except:
etxt = ""
try:
uptime = int(time.time()) - int(obj._starttime)
except:
uptime = int(time.time() - kernel._starttime)
txt = psformat % (elapsed(next), thrname, stxt, elapsed(uptime), etxt)
res.append((next, txt))
for next, txt in sorted(res, key=lambda x: x[0], reverse=True):
event.reply("%-5s %s" % (nr, txt))
nr += 1
if nr == 1: event.reply("no tasks running.")
def __init__(self, *args, **kwargs):
HTTPServer.__init__(self, *args, **kwargs)
Object.__init__(self)
self.host = args[0]
self._status = "start"
self._last = time.time()
self._start = time.time()
def shop(event):
if not event._parsed.rest:
event.reply("shop <txt>")
return
o = Object(event)
o.prefix = "shop"
o.shop = event._parsed.rest
path = o.save()
event.ok(1)
def log(event):
if not event._parsed.rest:
event.reply("log <txt>")
return
o = Object(event)
o.prefix = "log"
o.log = event._parsed.rest
path = o.save()
event.ok(1)
def last(self, *args, **kwargs):
fns = self.all(*args, **kwargs)[::-1]
res = None
for fn in fns[::-1]:
obj = Object().load(fn)
if "deleted" in obj and obj.deleted: continue
if args and len(args) > 1 and args[1] != obj.get(args[0], ""): continue
res = obj
return res
def todo(event):
if not event._parsed.rest:
event.reply("todo <txt>")
return
o = Object(event)
o.prefix = "todo"
o.todo = event._parsed.rest
path = o.save()
event.ok(1)
def __init__(self):
Object.__init__(self)
self._status = "start"
self._start = time.time()
self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try: self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
except: pass
self._sock.setblocking(1)
def first(self, *args, **kwargs):
""" return first object matching provided prefix. """
for fn in self.prefixed(*args, **kwargs):
try:
obj = Object().load(fn)
except:
logging.warn("fail %s" % fn)
continue
if "deleted" in obj and obj.deleted: continue
if len(args) > 1 and obj.get(args[0]) != args[1]: continue
return obj
def deleted(event):
if not event.args: return
store = Store()
key = event.args[0]
nr = 0
for fn in store.all(key):
obj = Object().load(fn)
if "deleted" not in obj: continue
res = obj.get(key, "")
if res: event.reply(res)
nr += 1
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._connected = Object()
self._poll = select.epoll()
self._resume = Object()
self._status = "running"
self._starttime = time.time()
thr = self.launch(self._start,
name="%s.engine" % sname(self),
daemon=True)
self._thrs.append(thr)
def last(self, *args, **kwargs):
""" return last record with a matching prefix. """
e = Event(**kwargs)
for fn in self.prefixed(*args, **kwargs)[::-1]:
try:
obj = Object().load(fn)
except:
logging.warn("fail %s" % fn)
continue
if "deleted" in obj and obj.deleted: continue
if len(args) > 1 and obj.get(args[0]) != args[1]: continue
return obj
def rm(event):
try: key, match = event.rest.split(" ", 1)
except ValueError: return
nr = 0
store = Store()
for fn in store.all(key):
obj = Object().load(fn)
if key not in obj: continue
if match not in obj[key]: continue
obj.deleted = True
obj.sync()
nr += 1
event.ok(nr)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._connected = Object()
self._queue = queue.Queue()
self._cbs = OOL()
self._table = Object()
self._handlers = OOL()
self._status = "running"
self._thrs = []
thr = self.launch(self.scheduling,
name="%s.schedule" % sname(self),
daemon=True)
self._thrs.append(thr)
def __getattr__(self, name):
if name == "cmnd": self.cmnd = ""
if name == "switch": self.switch = Object()
if name == "want": self.want = Object()
if name == "ignore": self.ignore = OOL()
if name == "notwant": self.notwant = Object()
if name == "args": self.args = []
if name == "rest": self.rest = ""
if name == "fields": self.fields = []
if name == "words": self.words = []
if name == "index": self.index = 0
if name not in self: raise AttributeError(name)
return self[name]
def restore(event):
try: key, match = event.args
except ValueError: return
nr = 0
store = Store()
for fn in store.all(*event.args):
obj = Object().load(fn)
if key not in obj: continue
if match not in obj[key]: continue
if "deleted" not in obj: continue
obj.deleted = False
obj.sync()
nr += 1
p.ok(nr)
class Engine(Scheduler):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._connected = Object()
self._poll = select.epoll()
self._resume = Object()
self._status = "running"
self._starttime = time.time()
thr = self.launch(self._start,
name="%s.engine" % sname(self),
daemon=True)
self._thrs.append(thr)
def _start(self):
self._connected.wait()
while self._status:
for event in self.events():
self.put(event)
def events(self):
self._poll.poll()
try:
yield self.event()
except (ConnectionResetError, BrokenPipeError, EDISCONNECT) as ex:
self.connect()
def event(self):
pass
def register_fd(self, f):
try:
fd = f.fileno()
except:
fd = f
logging.warn("# engine on %s" % str(fd))
self._poll.register(fd, select.EPOLLET)
self._resume.fd = fd
return fd
def resume(self):
if not self._resume.fd: raise ERESUME
logging.info("! resume on %s" % self._resume.fd)
self._poll = select.epoll.fromfd(self._resume.fd)
def stop(self):
self._status = ""
self._poll.close()
super().stop()
def ps(event):
results = []
nr = 1
for thr in sorted(launcher.running(), key=lambda x: str(x)):
obj = Object()
obj.update(vars(thr))
txt = thr.name.strip()
if event.rest and event.rest not in txt: continue
try: obj = obj.__class__.__self__
except: pass
if "_error" in obj: txt += " %s" % obj._error.strip()
if "_status" in obj: txt += " %s" % obj._status.strip()
if "_start" in obj: txt += " %s" % elapsed(int(time.time()) - int(obj._start))
if "sleep" in obj and "_last" in obj: txt += " next %s/%s" % (elapsed(int(obj.sleep) - int(time.time() - int(obj._last))), elapsed(int(obj.sleep)))
event.reply("%s %s" % (nr, txt.strip()))
nr += 1
def feed(text):
from meds.object import Object
result = []
chunks = text.split("\r\n")
for chunk in chunks:
obj = Object().feed(chunk)
result.append(obj)
return result
def find(self, prefix, *args):
""" find all objects stored with a prefix subdirectory. """
for fn in self.prefixed(prefix, *args):
try:
obj = Object().load(fn)
except:
logging.warn("fail %s" % fn)
continue
if "deleted" in obj and obj.deleted: continue
yield obj
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._cfg = Config(rss)
config = storage.last("cfg", "rss")
if config: self._cfg.update(config)
objs.seen = Object()
objs.seen.list = []
objs.seen.seen = "rss"
objs.seen.prefix = "seen"
obj = storage.last("seen", "rss")
if obj: objs.seen = obj
logging.warn("# seen %s" % len(objs.seen.list))
def since(self, start, *args, **kwargs):
""" return all object since a given time. """
e = Event(**kwargs)
e.start = start
for fn in self.prefixed(*args, **e):
try:
obj = Object().load(fn)
except:
logging.warn("fail %s" % fn)
continue
if "deleted" in obj and obj.deleted: continue
yield obj
def test_update(self):
o1 = Object()
o1._bla = "mekker"
o2 = Object()
o2._bla = "blaet"
o1.update(o2)
self.assertEqual(o1._bla, "blaet")
def mbox(event):
if not event.args: return
fn = os.path.expanduser(event.args[0])
nr = 0
try: object = mailbox.Maildir(fn, create=False)
except:
try: object = mailbox.mbox(fn, create=False)
except: event.reply("need a mbox or maildir.") ; return
for m in object:
o = Object()
o.update(m.items())
try: sdate = os.sep.join(short_date(o.Date).split())
except AttributeError: sdate = None
o.text = bytes()
for load in m.walk():
if load.get_content_type() == 'text/plain': o.text += load.get_payload(decode=True)
o.text = str(o.text, "latin-1")
o.prefix = "email"
if sdate: o.save(sdate)
else: o.save()
nr += 1
event.ok(nr)
return nr
def rss(event):
if not event._parsed.rest:
event.reply("rss <url>")
return
o = Object(event)
o.prefix = "rss"
o.rss = event._parsed.rest
o.service = "rss"
path = o.save()
event.ok(1)
def get_feed(url):
from meds.object import Object
if not gotparser:
logging.error("no feedparser available")
return []
if not "http" in url:
logging.error("%s is not an url." % url)
return []
try:
result = feedparser.parse(get_url(url))
except Exception as ex:
logging.error(get_exception(url))
return
if "entries" in result:
for entry in result["entries"][::-1]:
yield Object(entry)
def resume(self):
resume = Object().load(j(cfg.workdir, "resume"))
for bot in resume.fleet:
try:
fd = int(bot["_resume"]["fd"])
break
except:
fd = None
self._resume.fd = fd
if self._cfg.ipv6:
self._oldsock = socket.fromfd(fd, socket.AF_INET6,
socket.SOCK_STREAM)
else:
self._oldsock = socket.fromfd(fd, socket.AF_INET,
socket.SOCK_STREAM)
self._config()
self.announce("done")
def slice(obj, keys=[]):
from meds.object import Object
o = Object()
if not obj: return o
if not keys: keys = obj.keys()
for key in keys:
if key.startswith("_"): continue
try:
val = obj[key]
except KeyError:
continue
try:
val.keys()
o[key] = slice(val)
except:
o[key] = val
return o
def selected(self, event):
""" select objects based on a parsed event. """
nr = 0
if not event._parsed.args: return []
if event._parsed.args: event.prefix = event._parsed.args[0]
for fn in self.prefixed(event.prefix, **event._parsed):
try:
obj = Object().load(fn)
except:
logging.warn("fail %s" % fn)
continue
if not event.nodel and "deleted" in obj and obj.deleted: continue
if not selector(obj, event._parsed.fields): continue
if notwanted(obj, event._parsed.notwant): continue
if not wanted(obj, event._parsed.want): continue
if ignore(obj, event._parsed.ignore): continue
nr += 1
if event._parsed.index and event._parsed.index != nr: continue
yield obj
logging.warn("# selected %s" % nr)
def reboot(event):
""" reboot the bot, allowing statefull reboot (keeping connections alive). """
if not cfg.reboot:
event.reply("reboot is not enabled")
return
if not users.allowed(event.origin, "REBOOT"):
event.reply("you don't have reboot permissions.")
return
event.announce("rebooting")
resume = Object()
resume.kernel = kernel._resume
resume.fleet = fleet
resume.save("resume")
e = Event()
e._funcs.append(real_reboot)
print(e)
kernel.put(e)
def status(event):
o = Object(event)
o.prefix = "status"
o.status = event.rest
path = o.save()
event.ok(1)
""" rss module. """
__copyright__ = "Copyright 2015, Bart Thate"
from meds.misc import slice, get_url, strip_html, need_redirect, get_feed, short_date, matching, locked
from meds.mods.clock import Repeater
from meds.object import Object
from meds.store import Store
from meds.cfg import cfg
from meds.scheduler import launcher
from meds.bots import fleet
import logging
import time
seen = Object()
seen.list = []
seen.results = "rss"
seen.prefix = "results"
objs = Object()
def register(mods):
mods.register("fetcher", fetcher)
def stop():
if "rss" in objs:
objs.rss.exit()
def mbox(event):
if not event._parsed.rest:
event.reply("mbox <path>")
return
fn = os.path.expanduser(event._parsed.args[0])
nr = 0
if os.path.isdir(fn): thing = mailbox.Maildir(fn, create=False)
elif os.path.isfile(fn): thing = mailbox.mbox(fn, create=False)
else:
event.reply("need a mbox or maildir.")
return
for m in thing:
try:
o = Object()
o.update(m.items())
try:
sdate = os.sep.join(to_date(o.Date).split())
except AttributeError:
sdate = None
o.text = ""
for load in m.walk():
if load.get_content_type() == 'text/plain':
o.text += load.get_payload()
o.text = o.text.replace("\\n", "\n")
o.prefix = "email"
o.email = o.From
if sdate: o.save(sdate)
else: o.save()
nr += 1
except:
logging.error(get_exception())
if nr: event.ok(nr)
def test_undersopzet(self):
o = Object()
o._bla = "mekker"
self.assertEqual(o._bla, "mekker")
def test_settingattribute(self):
o = Object()
o.bla = "mekker"
self.assertEqual(o.bla, "mekker")
def timed2(self):
o = Object()
o.date = time.ctime(time.time())
self.assert_(o.timed())
if needed:
nrtimes = int(delta / needed)
txt = "%s #%s" % (name.upper(), nrtimes)
if name in omschrijving: txt += " (%s)" % omschrijving.get(name)
txt += " elke %s" % elapsed(seconds(nr(name)))
if name in soort:
txt += " door een %s" % soort.get(name, "slachtoffer")
txt += " bijv. in %s" % random.choice(gemeenten)
if name in tags: txt += " %s" % tags.get(name)
if cfg and cfg.showurl and name in urls:
txt += " - %s" % urls.get(name)
event.announce(txt)
#:
nrsec = Object()
nrsec.dag = 24 * 60 * 60.0
nrsec.jaar = 365 * nrsec.dag
nrsec.weekend = 2 / 7 * (24 * 60 * 60.0 * 365) / 52
nrsec.avond = 16 / 24 * (24 * 60 * 60.0)
#:
times = Object()
times.weekend = 2 / 7 * (24 * 60 * 60.0 * 365) / 52
times.avond = 16 / 24 * (24 * 60 * 60.0)
times.dag = 24 * 60 * 60.0
times.jaar = 365 * 24 * 60 * 60.0
#:
rechter = Object()
rechter.ibs = 8171
rss = Config()
rss.cfg = "rss"
rss.display_list = ["title", "link", "Date"]
rss.descriptions = [
"officiel",
]
rss.sleeptime = 600
rss.ignore = []
rss.dosave = []
rss.nosave = []
rss.showurl = False
#:
cli = Config()
cli.welcome = "mogge!!"
cli.cfg = "cli"
cli.server = "user@cli"
#:
input = Config()
input.server = "input@local"
templates = Object()
templates.xmpp = xmpp
templates.irc = irc
templates.main = main
templates.udp = udp
templates.rest = rest
templates.stats = stats
templates.rss = rss
templates.cli = cli
def log(event):
o = Object(event)
o.prefix = "log"
o.log = event.rest
path = o.save()
event.ok(1)
else: nr = 0 ;
txt = "%s #%s" % (name.upper(), nr)
if name in omschrijving: txt += " (%s)" % omschrijving.get(name)
if name in cijfers: txt += " %d/jaar elke %s" % (cijfers.get(name), elapsed(seconds(cijfers.get(name))))
if name in soort: txt += " %s" % soort.get(name)
if name in dbc: txt += " %s" % dbc.get(name)
txt += ", bijv. uit %s" % (random.choice(gemeenten))
if name in tags: txt += " %s" % tags.get(name)
txt += " (%s)" % elapsed(delta)
event.announce(txt)
startdate = "2012-11-23 00:00:00"
starttime = to_time(startdate)
source = "http://pypi.python.org/pypi/meds"
display = Object()
display.alarm = ["politie", "hap", "keten"]
display.oordeel = ["verwijs", "uitstroom", "opname"]
display.vergiftigingen = ["vergiftigingen", "tumor"]
display.neurotoxisch = ["neurotoxisch", ]
display.suicide = ["suicide",]
display.pogingen = ["pogingen",]
display.incidenten = ["incidenten",]
display.acuut = ["acuut",]
display.zorgmijder = ["zorgmijder", ]
display.opname = ["ibs", "rm", "vlm", "mvv", "vwm", "ev", "ob", "zb"]
tags = Object()
tags.opname = "#broodjepindakaas"
tags.crisis = "#triade"
tags.suicide = "#wetverplichteggz"
def todo(event):
o = Object(event)
o.prefix = "todo"
o.todo = event.rest
path = o.save()
event.ok(1)
#: cmnds = Cmnds() #: fleet = [] #: kernel = Kernel() #: launcher = Launcher() #: names = OOL() #: runner = Runner() #: storage = Storage() #: users = Users() #: cfgs = Config() #: objs = Object() objs.cfg = cfg objs.aliases = aliases objs.cmnds = cmnds objs.fleet = fleet objs.kernel = kernel objs.launcher = launcher objs.names = names objs.runner = runner objs.storage = storage objs.users = users
class Scheduler(Launcher):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._connected = Object()
self._queue = queue.Queue()
self._cbs = OOL()
self._table = Object()
self._handlers = OOL()
self._status = "running"
self._thrs = []
thr = self.launch(self.scheduling,
name="%s.schedule" % sname(self),
daemon=True)
self._thrs.append(thr)
def scheduling(self, timeout=None):
logging.info("! start %s" % sname(self))
self._connected.wait()
while self._status:
event = self._queue.get(timeout=timeout)
if not event: break
try:
self.handle_event(event)
except:
logging.error(get_exception())
logging.info("! stop %s" % sname(self))
self.ready()
def handle_event(self, event):
self.dispatch(event)
self.call_cb(event)
def dispatch(self, event):
if event._parsed.cmnd in self._handlers:
for func in self._handlers[event._parsed.cmnd]:
func(event)
def call_cb(self, event):
try:
cbs = self._cbs[event.origin]
except KeyError:
cbs = []
for cb in cbs:
cb(event)
def direct(self, name, package=None):
return importlib.import_module(name, package)
def load(self, modname, force=True):
logging.debug("! load %s" % modname)
mod = self.direct(modname)
if force: self._table[modname] = mod
return mod
def modules(self, packagename="", want=""):
if not packagename: packagename = "meds"
package = self.direct(packagename)
for pkg in pkgutil.walk_packages(package.__path__, packagename + "."):
yield pkg[1]
def prompt(self):
pass
def put(self, event):
self._queue.put_nowait(event)
def register(self, key, value):
self._handlers.register(key, value)
def reload(self, name, force=False, event=None):
from meds.event import Event
e = event or Event()
if name not in self._table: self.load(name, True)
n = name.split(".")[-1]
if name in self._table and "shutdown" in dir(self._table[name]):
self._table[name].shutdown(e)
e.ready()
if name not in self._table or force: self.load(name, True)
else: self._table[name] = importlib.reload(self._table[name])
if "init" in dir(self._table[name]):
self._table[name].init(e)
e.ready()
return self._table[name]
def start(self):
self._connected.ready()
def stop(self):
self._status = ""
self._queue.put_nowait(None)