def _stats(self, name, interval=60.0):
def counter(event):
counter.count += 1
return (event, )
counter.count = 0
@idiokit.stream
def logger():
while True:
try:
yield idiokit.sleep(interval)
finally:
if counter.count > 0:
self.log.info("Sent {0} events to room {1!r}".format(
counter.count, name),
event=events.Event({
"type":
"room",
"service":
self.bot_name,
"sent events":
unicode(counter.count),
"room":
name
}))
counter.count = 0
result = idiokit.map(counter)
idiokit.pipe(logger(), result)
return result
def _establish_session(self, jid, service_id, path, conf):
start = Element("start", xmlns=SERVICE_NS, id=service_id)
if path:
path_element = Element("path")
path_element.add(serialize.dump(path))
start.add(path_element)
conf_element = Element("config")
conf_element.add(serialize.dump(conf))
start.add(conf_element)
try:
result = yield self.xmpp.core.iq_set(start, to=jid)
except XMPPError as error:
if error.type != "cancel":
raise
if error.condition == "session-failure":
raise SessionError(
"session for service " + repr(service_id) +
" failed: " + error.text)
idiokit.stop()
except Unavailable:
idiokit.stop()
for start in result.children("start", SERVICE_NS).with_attrs("id"):
session_id = start.get_attr("id")
session = self._catch(jid, session_id)
idiokit.pipe(self.fork(), session)
sessions = self.jids.setdefault(jid, dict())
sessions[session_id] = session
idiokit.stop(session)
else:
raise SessionError(
"no session ID for service " + repr(service_id) + "received")
def session(self, _, src_room, dst_room, **keys):
keyset = yield idiokit.pipe(
self.transform_keys(src_room=src_room, dst_room=dst_room, **keys),
_collect_set())
pipes = [self._pipes.inc(src_room, dst_room, key) for key in keyset]
yield idiokit.pipe(*pipes)
def _stats(self, name, interval=60.0):
def counter(event):
counter.count += 1
return (event,)
counter.count = 0
@idiokit.stream
def logger():
while True:
try:
yield idiokit.sleep(interval)
finally:
if counter.count > 0:
self.log.info(
"Sent {0} events to room {1!r}".format(counter.count, name),
event=events.Event({
"type": "room",
"service": self.bot_name,
"sent events": unicode(counter.count),
"room": name}))
counter.count = 0
result = idiokit.map(counter)
idiokit.pipe(logger(), result)
return result
def inc(self, *args, **keys):
key = self._key(*args, **keys)
if self.counter.inc(key) and key not in self.tasks:
self.tasks[key] = self.task(*args, **keys)
task = self.tasks[key]
fork = task.fork()
idiokit.pipe(self._cleanup(key), fork)
return fork
def session(self, state, src_room, dst_room, window_time=60.0, rule=None):
if rule is None:
rule = rules.Anything()
rule = rules.rule(rule)
queue = collections.deque()
ids = dict()
to = self.to_room(dst_room)
idiokit.pipe(self.purge(ids, queue), events.events_to_elements(), to)
yield idiokit.pipe(self.from_room(src_room),
events.stanzas_to_events(), self.match(rule),
self.process(ids, queue, window_time),
events.events_to_elements(), to)
def test():
server, url = yield create_https_server("localhost")
with tmpfile(ca_data) as ca_certs:
_, fileobj = yield idiokit.pipe(
server, utils.fetch_url(url, verify=ca_certs))
self.assertEqual(fileobj.read(), "ok")
def main():
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s %(levelname)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%SZ"
)
parser = optparse.OptionParser()
parser.set_usage("usage: %prog [options] handler [path ...]")
options, args = parser.parse_args()
if len(args) < 1:
parser.error("expected handler")
try:
handler_spec = json.loads(args[0])
except ValueError:
handler_spec = args[0]
handler_type = handlers.load_handler(handler_spec)
def generate_events():
for line in fileinput.input(args[1:]):
line = line.strip()
if not line:
continue
yield events.Event(json.loads(line))
idiokit.main_loop(idiokit.pipe(
_feed(generate_events()),
handler_type(log=logging).transform(),
_print_events()
))
def _handle_augment(self, src_room, dst_room, args):
return idiokit.pipe(self.from_room(src_room),
events.stanzas_to_events(),
_ignore_augmentations(src_room == dst_room),
_create_eids(), self.augment(*args), _embed_eids(),
events.events_to_elements(),
self.to_room(dst_room))
def session(self, state, src_room, **keys):
keys["src_room"] = src_room
def _alert(_):
yield self.REPORT_NOW
@idiokit.stream
def _collect():
while True:
item = yield idiokit.next()
self.queue(0.0, item, **keys)
collector = idiokit.pipe(self.collect(state, **keys), _collect())
idiokit.pipe(self.alert(**keys), idiokit.map(_alert), collector)
result = yield idiokit.pipe(self._rooms.inc(src_room), collector)
idiokit.stop(result)
def test():
server, url = yield create_https_server("localhost")
try:
yield idiokit.pipe(server, utils.fetch_url(url, verify=True))
except utils.FetchUrlFailed:
return
self.fail("fetch_url should fail due to a certificate verification error")
def session(self, state, src_room, **keys):
keys["src_room"] = src_room
def _alert(_):
yield self.REPORT_NOW
@idiokit.stream
def _collect():
while True:
item = yield idiokit.next()
self.queue(0.0, item, **keys)
collector = idiokit.pipe(self.collect(state, **keys), _collect())
idiokit.pipe(self.alert(**keys), idiokit.map(_alert), collector)
result = yield idiokit.pipe(self._rooms.inc(src_room), collector)
idiokit.stop(result)
def test():
server, url = yield create_https_server("localhost")
request = urllib2.Request(url)
with tmpfile(ca_data) as ca_certs:
_, fileobj = yield idiokit.pipe(server, utils.fetch_url(request, verify=ca_certs))
self.assertEqual(fileobj.read(), "ok")
def _session(self, lobby, session, queues):
name = session.service
if session.path:
name += u"(" + ".".join(session.path) + ")"
conf = []
for key, value in session.conf.iteritems():
conf.append(repr(key) + "=" + repr(value))
attrs = events.Event({
"type":
"session",
"service":
session.service,
"path":
u".".join(session.path) if session.path else [],
"config":
u", ".join(conf)
})
session_id = session.path or [uuid.uuid4().hex]
with self.log.stateful(
attrs.value("service").encode("utf-8"), *session_id) as log:
while True:
waiter = queues.pop(session.service, None)
event = idiokit.Event()
queues[session.service] = event
try:
while waiter is not None:
waiter = yield waiter.fork()
stream = yield idiokit.pipe(
lobby.session(session.service, *session.path,
**session.conf),
self._delayed_log(log, name, attrs))
except Cancel:
return
finally:
if queues.get(session.service, None) is event:
queues.pop(session.service)
event.succeed(waiter)
conf_str = ", ".join(conf)
log.open("Sent {0!r} conf {1}".format(name, conf_str),
attrs,
status="running")
try:
yield stream
except services.Stop:
log.close("Lost connection to {0!r}".format(name),
attrs,
status="lost")
except Cancel:
log.close("Ended connection to {0!r}".format(name),
attrs,
status="removed")
return
def session(self, state, dst_room, **keys):
connections = []
for feed_key in self.feed_keys(dst_room=dst_room, **keys):
connections.append(self._connections.inc(feed_key, dst_room))
if connections:
return idiokit.pipe(*connections)
return idiokit.consume()
def test():
server, url = yield create_https_server("127.0.0.1")
with tmpfile(ca_data) as ca_certs:
try:
yield idiokit.pipe(server, utils.fetch_url(url, verify=ca_certs))
except idiokit.ssl.SSLCertificateError:
return
self.fail("fetch_url should fail due to a wrong hostname")
def session(self, state, dst_room, **keys):
connections = []
for feed_key in self.feed_keys(dst_room=dst_room, **keys):
connections.append(self._connections.inc(feed_key, dst_room))
if connections:
return idiokit.pipe(*connections)
return idiokit.consume()
def session(self, state, src_room, dst_room=None, **keys):
if dst_room is None:
dst_room = src_room
augments = list()
for args in self.augment_keys(src_room=src_room, dst_room=dst_room, **keys):
augments.append(self._augments.inc(src_room, dst_room, args))
yield idiokit.pipe(*augments)
def parse_csv(self, filename, fileobj):
match = re.match(self.filename_rex, filename)
if match is None:
self.log.error("Filename {0!r} did not match".format(filename))
idiokit.stop(False)
yield idiokit.pipe(utils.csv_to_events(fileobj),
_add_filename_info(match.groupdict()))
idiokit.stop(True)
def main(self):
xmpp = yield self.xmpp_connect()
room = yield xmpp.muc.join(self.room)
yield idiokit.pipe(
room,
events.stanzas_to_events(),
self._recv()
)
def main(self):
xmpp = yield self.xmpp_connect()
room = yield xmpp.muc.join(self.room)
yield idiokit.pipe(
room,
events.stanzas_to_events(),
self._recv()
)
def test():
server, url = yield create_https_server("localhost")
try:
yield idiokit.pipe(server, utils.fetch_url(url, verify=True))
except utils.FetchUrlFailed:
return
self.fail(
"fetch_url should fail due to a certificate verification error"
)
def _handle_room(self, room_name):
room = yield self.xmpp.muc.join(room_name, self.bot_name)
distributor = yield self._ready.fork()
yield idiokit.pipe(
room,
idiokit.map(self._map, room_name),
distributor.fork(),
idiokit.Event()
)
def test():
server, url = yield create_https_server("127.0.0.1")
with tmpfile(ca_data) as ca_certs:
try:
yield idiokit.pipe(server,
utils.fetch_url(url, verify=ca_certs))
except idiokit.ssl.SSLCertificateError:
return
self.fail("fetch_url should fail due to a wrong hostname")
def handle_room(self, name):
self.log.info("Joining room %r", name)
room = yield self.xmpp.muc.join(name, self.bot_name)
self.log.info("Joined room %r", name)
try:
yield idiokit.pipe(room, self.reply(room.jid),
events.events_to_elements())
finally:
self.log.info("Left room %r", name)
def _room(self, name):
room = yield idiokit.pipe(
self._delayed_log("Joining room " + repr(name)),
self.xmpp.muc.join(name, self.bot_name))
self.log.info("Joined room " + repr(name))
try:
yield room
finally:
self.log.info("Left room " + repr(name))
def session(self, state, src_room, dst_room=None, **keys):
if dst_room is None:
dst_room = src_room
augments = list()
for args in self.augment_keys(src_room=src_room,
dst_room=dst_room,
**keys):
augments.append(self._augments.inc(src_room, dst_room, args))
yield idiokit.pipe(*augments)
def session(self, state, src_room, dst_room, window_time=60.0, rule=None):
if rule is None:
rule = rules.Anything()
rule = rules.rule(rule)
queue = collections.deque()
ids = dict()
to = self.to_room(dst_room)
idiokit.pipe(self.purge(ids, queue), events.events_to_elements(), to)
yield idiokit.pipe(
self.from_room(src_room),
events.stanzas_to_events(),
self.match(rule),
self.process(ids, queue, window_time),
events.events_to_elements(),
to
)
def _handle_augment(self, src_room, dst_room, args):
return idiokit.pipe(
self.from_room(src_room),
events.stanzas_to_events(),
_ignore_augmentations(src_room == dst_room),
_create_eids(),
self.augment(*args),
_embed_eids(),
events.events_to_elements(),
self.to_room(dst_room),
)
def poll(self):
self.log.info("Downloading updates from {0!r}".format(self.url))
try:
info, fileobj = yield utils.fetch_url(self.url)
except utils.FetchUrlFailed as fuf:
raise bot.PollSkipped("Downloading {0!r} failed ({1})".format(self.url, fuf))
self.log.info("Updates downloaded from {0!r}".format(self.url))
yield idiokit.pipe(
utils.csv_to_events(fileobj, columns=self._columns),
idiokit.map(self._normalize))
def main(self):
xmpp = yield self.xmpp_connect()
room = yield xmpp.muc.join(self.room)
yield idiokit.pipe(
self._read_stdin(),
events.events_to_elements(),
_rate_limiter(self.rate_limit),
room,
idiokit.consume()
)
def parse_csv(self, filename, fileobj):
match = re.match(self.filename_rex, filename)
if match is None:
self.log.error("Filename {0!r} did not match".format(filename))
idiokit.stop(False)
yield idiokit.pipe(
utils.csv_to_events(fileobj),
_add_filename_info(match.groupdict())
)
idiokit.stop(True)
def poll(self):
self.log.info("Downloading updates from {0!r}".format(self.url))
try:
info, fileobj = yield utils.fetch_url(self.url)
except utils.FetchUrlFailed as fuf:
raise bot.PollSkipped("Downloading {0!r} failed ({1})".format(self.url, fuf))
self.log.info("Updates downloaded from {0!r}".format(self.url))
yield idiokit.pipe(
utils.csv_to_events(fileobj, columns=self._columns),
idiokit.map(self._normalize))
def parse_csv(self, headers, filename, fileobj):
match = re.match(self.filename_rex, filename)
if match is None:
self.log.error("Filename {0!r} did not match".format(filename))
idiokit.stop(False)
subject = imapbot.get_header(headers[0], "Subject", None)
yield idiokit.pipe(
utils.csv_to_events(fileobj),
self.normalize(subject, match.groupdict()))
idiokit.stop(True)
def main(self):
xmpp = yield self.xmpp_connect()
room = yield xmpp.muc.join(self.room)
yield idiokit.pipe(
self._read_stdin(),
events.events_to_elements(),
_rate_limiter(self.rate_limit),
room,
idiokit.consume()
)
def main(self):
try:
xmpp = yield self.xmpp_connect()
rooms = list()
for name in self.xmpp_rooms:
room = yield xmpp.muc.join(name, self.bot_name)
rooms.append(
room | self.xmpp_to_log(room.jid, room.participants))
yield idiokit.pipe(*rooms)
except idiokit.Signal:
pass
def handle_room(self, name):
self.log.info("Joining room %r", name)
room = yield self.xmpp.muc.join(name, self.bot_name)
self.log.info("Joined room %r", name)
try:
yield idiokit.pipe(
room,
self.reply(room.jid),
events.events_to_elements())
finally:
self.log.info("Left room %r", name)
def _handle_room(self, name):
msg = "room {0!r}".format(name)
attrs = events.Event(type="room", service=self.bot_name, room=name)
with self.log.stateful(repr(self.xmpp.jid), "room", repr(name)) as log:
log.open("Joining " + msg, attrs, status="joining")
room = yield self.xmpp.muc.join(name, self.bot_name)
log.open("Joined " + msg, attrs, status="joined")
try:
yield idiokit.pipe(room, events.stanzas_to_events())
finally:
log.close("Left " + msg, attrs, status="left")
def _handle_room(self, name):
msg = "room {0!r}".format(name)
attrs = events.Event(type="room", service=self.bot_name, room=name)
with self.log.stateful(repr(self.xmpp.jid), "room", repr(name)) as log:
log.open("Joining " + msg, attrs, status="joining")
room = yield self.xmpp.muc.join(name, self.bot_name)
log.open("Joined " + msg, attrs, status="joined")
try:
yield idiokit.pipe(room, events.stanzas_to_events())
finally:
log.close("Left " + msg, attrs, status="left")
def _session(self, lobby, session, queues):
name = session.service
if session.path:
name += u"(" + ".".join(session.path) + ")"
conf = []
for key, value in session.conf.iteritems():
conf.append(repr(key) + "=" + repr(value))
attrs = events.Event({
"type": "session",
"service": session.service,
"path": u".".join(session.path) if session.path else [],
"config": u", ".join(conf)
})
session_id = session.path or [uuid.uuid4().hex]
with self.log.stateful(attrs.value("service").encode("utf-8"), *session_id) as log:
while True:
waiter = queues.pop(session.service, None)
event = idiokit.Event()
queues[session.service] = event
try:
while waiter is not None:
waiter = yield waiter.fork()
stream = yield idiokit.pipe(
lobby.session(session.service, *session.path, **session.conf),
self._delayed_log(log, name, attrs))
except Cancel:
return
finally:
if queues.get(session.service, None) is event:
queues.pop(session.service)
event.succeed(waiter)
conf_str = ", ".join(conf)
log.open("Sent {0!r} conf {1}".format(name, conf_str), attrs, status="running")
try:
yield stream
except services.Stop:
log.close("Lost connection to {0!r}".format(name), attrs, status="lost")
except Cancel:
log.close("Ended connection to {0!r}".format(name), attrs, status="removed")
return
def poll(self):
self.log.info("Downloading {0}".format(self.feed_url))
try:
info, fileobj = yield utils.fetch_url(self.feed_url)
except utils.FetchUrlFailed as fuf:
raise bot.PollSkipped("Download failed: {0}".format(fuf))
lines = []
for line in fileobj:
line = line.strip()
if line and not line.startswith("#"):
lines.append(line)
yield idiokit.pipe(
utils.csv_to_events(tuple(lines),
columns=COLUMNS,
charset=info.get_param("charset", None)),
_parse())
def _archive(self, room_bare_jid):
compress = utils.WaitQueue()
room_name = _encode_room_jid(room_bare_jid)
_dir = os.path.join(self.archive_dir, room_name)
if _dir != os.path.normpath(_dir):
raise ValueError("incorrect room name lands outside the archive directory")
for root, _, filenames in os.walk(_dir):
for filename in filenames:
path = os.path.join(root, filename)
if _is_compress_path(path):
compress.queue(0.0, path)
return idiokit.pipe(
self._collect(room_name, compress),
self._compress(compress)
)
def session(self, state, src_room, dst_room,
augment_room=None, time_window=10.0):
if augment_room is None:
augment_room = src_room
ids = dict()
queue = collections.deque()
yield idiokit.pipe(
self.from_room(src_room),
events.stanzas_to_events(),
_ignore_augmentations(augment_room == src_room),
self.collect(ids, queue, time_window),
self.cleanup(ids, queue),
events.events_to_elements(),
self.to_room(dst_room),
self.from_room(augment_room),
events.stanzas_to_events(),
self.combine(ids, queue, time_window)
)
def session(self,
state,
src_room,
dst_room,
augment_room=None,
time_window=10.0):
if augment_room is None:
augment_room = src_room
ids = dict()
queue = collections.deque()
yield idiokit.pipe(self.from_room(src_room),
events.stanzas_to_events(),
_ignore_augmentations(augment_room == src_room),
self.collect(ids, queue,
time_window), self.cleanup(ids, queue),
events.events_to_elements(), self.to_room(dst_room),
self.from_room(augment_room),
events.stanzas_to_events(),
self.combine(ids, queue, time_window))
def poll(self):
self.log.info("Downloading {0}".format(self.feed_url))
try:
info, fileobj = yield utils.fetch_url(self.feed_url)
except utils.FetchUrlFailed as fuf:
raise bot.PollSkipped("Download failed: {0}".format(fuf))
lines = []
for line in fileobj:
line = line.strip()
if line and not line.startswith("#"):
lines.append(line)
yield idiokit.pipe(
utils.csv_to_events(tuple(lines),
columns=COLUMNS,
charset=info.get_param("charset", None)),
_parse()
)
def handle(handler_spec, input_events):
r"""
Return a list of event dictionaries collected by handling input_events
using handler_spec.
>>> from abusehelper.core.events import Event
>>> from abusehelper.core.transformation import Handler
>>>
>>> class MyHandler(Handler):
... @idiokit.stream
... def transform(self):
... while True:
... event = yield idiokit.next()
... event.add("a", "b")
... yield idiokit.send(event)
...
>>> handle(MyHandler, [{}])
[{u'a': [u'b']}]
Note that to simplify testing the output is a list of dictionaries
instead of abusehelper.core.events.Event objects.
"""
handler_type = handlers.load_handler(handler_spec)
log = logging.getLogger("Null")
log_handler = _NullHandler()
log.addHandler(log_handler)
try:
handler = handler_type(log=log)
return idiokit.main_loop(idiokit.pipe(
_feed(itertools.imap(events.Event, input_events)),
handler.transform(),
_collect_events()
))
finally:
log.removeHandler(log_handler)
def parse(self, db):
queries = QuerySet()
while True:
next = idiokit.next()
if queries:
idiokit.pipe(self._timeout(0.0), next)
try:
elements = yield next
except Timeout:
pass
else:
for element in elements.with_attrs("from"):
sender = JID(element.get_attr("from"))
if element.named("presence").with_attrs(type="unavailable"):
db.purge_jid(sender)
queries.discard_jid(sender)
for message in element.named("message"):
if not message.with_attrs(type="groupchat"):
continue
for event in events.Event.from_elements(message):
db.add_event(sender, event)
for query in element.named("message").children(ns=NS):
try:
args = json.loads(query.text)
except JSONDecodeError:
self.log.error("Invalid query data from %r: %r",
sender, query.text)
continue
if "id" not in args:
self.log.error("Query without an ID from %r: %r",
sender, args)
continue
id = args.get("id")
if query.named("start"):
start = args.get("start", None)
end = args.get("end", None)
queries.start(sender, id, db.query(start, end))
self.log.info("Start from %r: %r", sender, args)
elif query.named("load"):
if "size" in args:
queries.load(sender, id, args.get("size"))
self.log.debug("Load from %r: %r", sender, args)
else:
self.log.error("Load without an ID from %r: %r",
sender, args)
elif query.named("histogram"):
start = args.get("start", None)
end = args.get("end", None)
step = args.get("step", None)
if None not in (start, end, step):
element = db.histogram(id, start, end, step)
self.xmpp.core.message(sender, element)
self.log.debug("Histogram from %r: %r",
sender, args)
elif query.named("cancel"):
queries.cancel(sender, id)
self.log.info("Cancel from %r: %r", sender, args)
for sender, element in queries:
yield self.xmpp.core.message(sender, element)
def _dst(self, dst):
return idiokit.pipe(
self._rooms.inc(dst),
idiokit.consume())
def _handle_room(self, room_name):
room = yield self.xmpp.muc.join(room_name, self.bot_name)
distributor = yield self._ready.fork()
yield idiokit.pipe(room, idiokit.map(self._map, room_name),
distributor.fork(), idiokit.Event())
def test():
server, url = yield create_https_server("localhost")
_, fileobj = yield idiokit.pipe(server, utils.fetch_url(url, verify=False))
self.assertEqual(fileobj.read(), "ok")
def parse(self, db):
queries = QuerySet()
while True:
next = idiokit.next()
if queries:
idiokit.pipe(self._timeout(0.0), next)
try:
elements = yield next
except Timeout:
pass
else:
for element in elements.with_attrs("from"):
sender = JID(element.get_attr("from"))
if element.named("presence").with_attrs(
type="unavailable"):
db.purge_jid(sender)
queries.discard_jid(sender)
for message in element.named("message"):
if not message.with_attrs(type="groupchat"):
continue
for event in events.Event.from_elements(message):
db.add_event(sender, event)
for query in element.named("message").children(ns=NS):
try:
args = json.loads(query.text)
except JSONDecodeError:
self.log.error("Invalid query data from %r: %r",
sender, query.text)
continue
if "id" not in args:
self.log.error("Query without an ID from %r: %r",
sender, args)
continue
id = args.get("id")
if query.named("start"):
start = args.get("start", None)
end = args.get("end", None)
queries.start(sender, id, db.query(start, end))
self.log.info("Start from %r: %r", sender, args)
elif query.named("load"):
if "size" in args:
queries.load(sender, id, args.get("size"))
self.log.debug("Load from %r: %r", sender,
args)
else:
self.log.error(
"Load without an ID from %r: %r", sender,
args)
elif query.named("histogram"):
start = args.get("start", None)
end = args.get("end", None)
step = args.get("step", None)
if None not in (start, end, step):
element = db.histogram(id, start, end, step)
self.xmpp.core.message(sender, element)
self.log.debug("Histogram from %r: %r", sender,
args)
elif query.named("cancel"):
queries.cancel(sender, id)
self.log.info("Cancel from %r: %r", sender, args)
for sender, element in queries:
yield self.xmpp.core.message(sender, element)
def collect(self, room_name):
collect = self._collect(room_name)
idiokit.pipe(self._alert(), collect)
return collect
def test():
server, url = yield create_https_server("localhost")
_, fileobj = yield idiokit.pipe(server,
utils.fetch_url(url, verify=False))
self.assertEqual(fileobj.read(), "ok")
