def _stats(self, name, interval=60.0): def counter(event): counter.count += 1 return (event, ) counter.count = 0 @idiokit.stream def logger(): while True: try: yield idiokit.sleep(interval) finally: if counter.count > 0: self.log.info("Sent {0} events to room {1!r}".format( counter.count, name), event=events.Event({ "type": "room", "service": self.bot_name, "sent events": unicode(counter.count), "room": name })) counter.count = 0 result = idiokit.map(counter) idiokit.pipe(logger(), result) return result
def _establish_session(self, jid, service_id, path, conf): start = Element("start", xmlns=SERVICE_NS, id=service_id) if path: path_element = Element("path") path_element.add(serialize.dump(path)) start.add(path_element) conf_element = Element("config") conf_element.add(serialize.dump(conf)) start.add(conf_element) try: result = yield self.xmpp.core.iq_set(start, to=jid) except XMPPError as error: if error.type != "cancel": raise if error.condition == "session-failure": raise SessionError( "session for service " + repr(service_id) + " failed: " + error.text) idiokit.stop() except Unavailable: idiokit.stop() for start in result.children("start", SERVICE_NS).with_attrs("id"): session_id = start.get_attr("id") session = self._catch(jid, session_id) idiokit.pipe(self.fork(), session) sessions = self.jids.setdefault(jid, dict()) sessions[session_id] = session idiokit.stop(session) else: raise SessionError( "no session ID for service " + repr(service_id) + "received")
def session(self, _, src_room, dst_room, **keys): keyset = yield idiokit.pipe( self.transform_keys(src_room=src_room, dst_room=dst_room, **keys), _collect_set()) pipes = [self._pipes.inc(src_room, dst_room, key) for key in keyset] yield idiokit.pipe(*pipes)
def _stats(self, name, interval=60.0): def counter(event): counter.count += 1 return (event,) counter.count = 0 @idiokit.stream def logger(): while True: try: yield idiokit.sleep(interval) finally: if counter.count > 0: self.log.info( "Sent {0} events to room {1!r}".format(counter.count, name), event=events.Event({ "type": "room", "service": self.bot_name, "sent events": unicode(counter.count), "room": name})) counter.count = 0 result = idiokit.map(counter) idiokit.pipe(logger(), result) return result
def inc(self, *args, **keys): key = self._key(*args, **keys) if self.counter.inc(key) and key not in self.tasks: self.tasks[key] = self.task(*args, **keys) task = self.tasks[key] fork = task.fork() idiokit.pipe(self._cleanup(key), fork) return fork
def session(self, state, src_room, dst_room, window_time=60.0, rule=None): if rule is None: rule = rules.Anything() rule = rules.rule(rule) queue = collections.deque() ids = dict() to = self.to_room(dst_room) idiokit.pipe(self.purge(ids, queue), events.events_to_elements(), to) yield idiokit.pipe(self.from_room(src_room), events.stanzas_to_events(), self.match(rule), self.process(ids, queue, window_time), events.events_to_elements(), to)
def test(): server, url = yield create_https_server("localhost") with tmpfile(ca_data) as ca_certs: _, fileobj = yield idiokit.pipe( server, utils.fetch_url(url, verify=ca_certs)) self.assertEqual(fileobj.read(), "ok")
def main(): logging.basicConfig( level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s", datefmt="%Y-%m-%d %H:%M:%SZ" ) parser = optparse.OptionParser() parser.set_usage("usage: %prog [options] handler [path ...]") options, args = parser.parse_args() if len(args) < 1: parser.error("expected handler") try: handler_spec = json.loads(args[0]) except ValueError: handler_spec = args[0] handler_type = handlers.load_handler(handler_spec) def generate_events(): for line in fileinput.input(args[1:]): line = line.strip() if not line: continue yield events.Event(json.loads(line)) idiokit.main_loop(idiokit.pipe( _feed(generate_events()), handler_type(log=logging).transform(), _print_events() ))
def _handle_augment(self, src_room, dst_room, args): return idiokit.pipe(self.from_room(src_room), events.stanzas_to_events(), _ignore_augmentations(src_room == dst_room), _create_eids(), self.augment(*args), _embed_eids(), events.events_to_elements(), self.to_room(dst_room))
def session(self, state, src_room, **keys): keys["src_room"] = src_room def _alert(_): yield self.REPORT_NOW @idiokit.stream def _collect(): while True: item = yield idiokit.next() self.queue(0.0, item, **keys) collector = idiokit.pipe(self.collect(state, **keys), _collect()) idiokit.pipe(self.alert(**keys), idiokit.map(_alert), collector) result = yield idiokit.pipe(self._rooms.inc(src_room), collector) idiokit.stop(result)
def test(): server, url = yield create_https_server("localhost") try: yield idiokit.pipe(server, utils.fetch_url(url, verify=True)) except utils.FetchUrlFailed: return self.fail("fetch_url should fail due to a certificate verification error")
def test(): server, url = yield create_https_server("localhost") request = urllib2.Request(url) with tmpfile(ca_data) as ca_certs: _, fileobj = yield idiokit.pipe(server, utils.fetch_url(request, verify=ca_certs)) self.assertEqual(fileobj.read(), "ok")
def _session(self, lobby, session, queues): name = session.service if session.path: name += u"(" + ".".join(session.path) + ")" conf = [] for key, value in session.conf.iteritems(): conf.append(repr(key) + "=" + repr(value)) attrs = events.Event({ "type": "session", "service": session.service, "path": u".".join(session.path) if session.path else [], "config": u", ".join(conf) }) session_id = session.path or [uuid.uuid4().hex] with self.log.stateful( attrs.value("service").encode("utf-8"), *session_id) as log: while True: waiter = queues.pop(session.service, None) event = idiokit.Event() queues[session.service] = event try: while waiter is not None: waiter = yield waiter.fork() stream = yield idiokit.pipe( lobby.session(session.service, *session.path, **session.conf), self._delayed_log(log, name, attrs)) except Cancel: return finally: if queues.get(session.service, None) is event: queues.pop(session.service) event.succeed(waiter) conf_str = ", ".join(conf) log.open("Sent {0!r} conf {1}".format(name, conf_str), attrs, status="running") try: yield stream except services.Stop: log.close("Lost connection to {0!r}".format(name), attrs, status="lost") except Cancel: log.close("Ended connection to {0!r}".format(name), attrs, status="removed") return
def session(self, state, dst_room, **keys): connections = [] for feed_key in self.feed_keys(dst_room=dst_room, **keys): connections.append(self._connections.inc(feed_key, dst_room)) if connections: return idiokit.pipe(*connections) return idiokit.consume()
def test(): server, url = yield create_https_server("127.0.0.1") with tmpfile(ca_data) as ca_certs: try: yield idiokit.pipe(server, utils.fetch_url(url, verify=ca_certs)) except idiokit.ssl.SSLCertificateError: return self.fail("fetch_url should fail due to a wrong hostname")
def session(self, state, src_room, dst_room=None, **keys): if dst_room is None: dst_room = src_room augments = list() for args in self.augment_keys(src_room=src_room, dst_room=dst_room, **keys): augments.append(self._augments.inc(src_room, dst_room, args)) yield idiokit.pipe(*augments)
def parse_csv(self, filename, fileobj): match = re.match(self.filename_rex, filename) if match is None: self.log.error("Filename {0!r} did not match".format(filename)) idiokit.stop(False) yield idiokit.pipe(utils.csv_to_events(fileobj), _add_filename_info(match.groupdict())) idiokit.stop(True)
def main(self): xmpp = yield self.xmpp_connect() room = yield xmpp.muc.join(self.room) yield idiokit.pipe( room, events.stanzas_to_events(), self._recv() )
def test(): server, url = yield create_https_server("localhost") try: yield idiokit.pipe(server, utils.fetch_url(url, verify=True)) except utils.FetchUrlFailed: return self.fail( "fetch_url should fail due to a certificate verification error" )
def _handle_room(self, room_name): room = yield self.xmpp.muc.join(room_name, self.bot_name) distributor = yield self._ready.fork() yield idiokit.pipe( room, idiokit.map(self._map, room_name), distributor.fork(), idiokit.Event() )
def handle_room(self, name): self.log.info("Joining room %r", name) room = yield self.xmpp.muc.join(name, self.bot_name) self.log.info("Joined room %r", name) try: yield idiokit.pipe(room, self.reply(room.jid), events.events_to_elements()) finally: self.log.info("Left room %r", name)
def _room(self, name): room = yield idiokit.pipe( self._delayed_log("Joining room " + repr(name)), self.xmpp.muc.join(name, self.bot_name)) self.log.info("Joined room " + repr(name)) try: yield room finally: self.log.info("Left room " + repr(name))
def session(self, state, src_room, dst_room, window_time=60.0, rule=None): if rule is None: rule = rules.Anything() rule = rules.rule(rule) queue = collections.deque() ids = dict() to = self.to_room(dst_room) idiokit.pipe(self.purge(ids, queue), events.events_to_elements(), to) yield idiokit.pipe( self.from_room(src_room), events.stanzas_to_events(), self.match(rule), self.process(ids, queue, window_time), events.events_to_elements(), to )
def _handle_augment(self, src_room, dst_room, args): return idiokit.pipe( self.from_room(src_room), events.stanzas_to_events(), _ignore_augmentations(src_room == dst_room), _create_eids(), self.augment(*args), _embed_eids(), events.events_to_elements(), self.to_room(dst_room), )
def poll(self): self.log.info("Downloading updates from {0!r}".format(self.url)) try: info, fileobj = yield utils.fetch_url(self.url) except utils.FetchUrlFailed as fuf: raise bot.PollSkipped("Downloading {0!r} failed ({1})".format(self.url, fuf)) self.log.info("Updates downloaded from {0!r}".format(self.url)) yield idiokit.pipe( utils.csv_to_events(fileobj, columns=self._columns), idiokit.map(self._normalize))
def main(self): xmpp = yield self.xmpp_connect() room = yield xmpp.muc.join(self.room) yield idiokit.pipe( self._read_stdin(), events.events_to_elements(), _rate_limiter(self.rate_limit), room, idiokit.consume() )
def parse_csv(self, filename, fileobj): match = re.match(self.filename_rex, filename) if match is None: self.log.error("Filename {0!r} did not match".format(filename)) idiokit.stop(False) yield idiokit.pipe( utils.csv_to_events(fileobj), _add_filename_info(match.groupdict()) ) idiokit.stop(True)
def parse_csv(self, headers, filename, fileobj): match = re.match(self.filename_rex, filename) if match is None: self.log.error("Filename {0!r} did not match".format(filename)) idiokit.stop(False) subject = imapbot.get_header(headers[0], "Subject", None) yield idiokit.pipe( utils.csv_to_events(fileobj), self.normalize(subject, match.groupdict())) idiokit.stop(True)
def main(self): try: xmpp = yield self.xmpp_connect() rooms = list() for name in self.xmpp_rooms: room = yield xmpp.muc.join(name, self.bot_name) rooms.append( room | self.xmpp_to_log(room.jid, room.participants)) yield idiokit.pipe(*rooms) except idiokit.Signal: pass
def handle_room(self, name): self.log.info("Joining room %r", name) room = yield self.xmpp.muc.join(name, self.bot_name) self.log.info("Joined room %r", name) try: yield idiokit.pipe( room, self.reply(room.jid), events.events_to_elements()) finally: self.log.info("Left room %r", name)
def _handle_room(self, name): msg = "room {0!r}".format(name) attrs = events.Event(type="room", service=self.bot_name, room=name) with self.log.stateful(repr(self.xmpp.jid), "room", repr(name)) as log: log.open("Joining " + msg, attrs, status="joining") room = yield self.xmpp.muc.join(name, self.bot_name) log.open("Joined " + msg, attrs, status="joined") try: yield idiokit.pipe(room, events.stanzas_to_events()) finally: log.close("Left " + msg, attrs, status="left")
def _session(self, lobby, session, queues): name = session.service if session.path: name += u"(" + ".".join(session.path) + ")" conf = [] for key, value in session.conf.iteritems(): conf.append(repr(key) + "=" + repr(value)) attrs = events.Event({ "type": "session", "service": session.service, "path": u".".join(session.path) if session.path else [], "config": u", ".join(conf) }) session_id = session.path or [uuid.uuid4().hex] with self.log.stateful(attrs.value("service").encode("utf-8"), *session_id) as log: while True: waiter = queues.pop(session.service, None) event = idiokit.Event() queues[session.service] = event try: while waiter is not None: waiter = yield waiter.fork() stream = yield idiokit.pipe( lobby.session(session.service, *session.path, **session.conf), self._delayed_log(log, name, attrs)) except Cancel: return finally: if queues.get(session.service, None) is event: queues.pop(session.service) event.succeed(waiter) conf_str = ", ".join(conf) log.open("Sent {0!r} conf {1}".format(name, conf_str), attrs, status="running") try: yield stream except services.Stop: log.close("Lost connection to {0!r}".format(name), attrs, status="lost") except Cancel: log.close("Ended connection to {0!r}".format(name), attrs, status="removed") return
def poll(self): self.log.info("Downloading {0}".format(self.feed_url)) try: info, fileobj = yield utils.fetch_url(self.feed_url) except utils.FetchUrlFailed as fuf: raise bot.PollSkipped("Download failed: {0}".format(fuf)) lines = [] for line in fileobj: line = line.strip() if line and not line.startswith("#"): lines.append(line) yield idiokit.pipe( utils.csv_to_events(tuple(lines), columns=COLUMNS, charset=info.get_param("charset", None)), _parse())
def _archive(self, room_bare_jid): compress = utils.WaitQueue() room_name = _encode_room_jid(room_bare_jid) _dir = os.path.join(self.archive_dir, room_name) if _dir != os.path.normpath(_dir): raise ValueError("incorrect room name lands outside the archive directory") for root, _, filenames in os.walk(_dir): for filename in filenames: path = os.path.join(root, filename) if _is_compress_path(path): compress.queue(0.0, path) return idiokit.pipe( self._collect(room_name, compress), self._compress(compress) )
def session(self, state, src_room, dst_room, augment_room=None, time_window=10.0): if augment_room is None: augment_room = src_room ids = dict() queue = collections.deque() yield idiokit.pipe( self.from_room(src_room), events.stanzas_to_events(), _ignore_augmentations(augment_room == src_room), self.collect(ids, queue, time_window), self.cleanup(ids, queue), events.events_to_elements(), self.to_room(dst_room), self.from_room(augment_room), events.stanzas_to_events(), self.combine(ids, queue, time_window) )
def session(self, state, src_room, dst_room, augment_room=None, time_window=10.0): if augment_room is None: augment_room = src_room ids = dict() queue = collections.deque() yield idiokit.pipe(self.from_room(src_room), events.stanzas_to_events(), _ignore_augmentations(augment_room == src_room), self.collect(ids, queue, time_window), self.cleanup(ids, queue), events.events_to_elements(), self.to_room(dst_room), self.from_room(augment_room), events.stanzas_to_events(), self.combine(ids, queue, time_window))
def poll(self): self.log.info("Downloading {0}".format(self.feed_url)) try: info, fileobj = yield utils.fetch_url(self.feed_url) except utils.FetchUrlFailed as fuf: raise bot.PollSkipped("Download failed: {0}".format(fuf)) lines = [] for line in fileobj: line = line.strip() if line and not line.startswith("#"): lines.append(line) yield idiokit.pipe( utils.csv_to_events(tuple(lines), columns=COLUMNS, charset=info.get_param("charset", None)), _parse() )
def handle(handler_spec, input_events): r""" Return a list of event dictionaries collected by handling input_events using handler_spec. >>> from abusehelper.core.events import Event >>> from abusehelper.core.transformation import Handler >>> >>> class MyHandler(Handler): ... @idiokit.stream ... def transform(self): ... while True: ... event = yield idiokit.next() ... event.add("a", "b") ... yield idiokit.send(event) ... >>> handle(MyHandler, [{}]) [{u'a': [u'b']}] Note that to simplify testing the output is a list of dictionaries instead of abusehelper.core.events.Event objects. """ handler_type = handlers.load_handler(handler_spec) log = logging.getLogger("Null") log_handler = _NullHandler() log.addHandler(log_handler) try: handler = handler_type(log=log) return idiokit.main_loop(idiokit.pipe( _feed(itertools.imap(events.Event, input_events)), handler.transform(), _collect_events() )) finally: log.removeHandler(log_handler)
def parse(self, db): queries = QuerySet() while True: next = idiokit.next() if queries: idiokit.pipe(self._timeout(0.0), next) try: elements = yield next except Timeout: pass else: for element in elements.with_attrs("from"): sender = JID(element.get_attr("from")) if element.named("presence").with_attrs(type="unavailable"): db.purge_jid(sender) queries.discard_jid(sender) for message in element.named("message"): if not message.with_attrs(type="groupchat"): continue for event in events.Event.from_elements(message): db.add_event(sender, event) for query in element.named("message").children(ns=NS): try: args = json.loads(query.text) except JSONDecodeError: self.log.error("Invalid query data from %r: %r", sender, query.text) continue if "id" not in args: self.log.error("Query without an ID from %r: %r", sender, args) continue id = args.get("id") if query.named("start"): start = args.get("start", None) end = args.get("end", None) queries.start(sender, id, db.query(start, end)) self.log.info("Start from %r: %r", sender, args) elif query.named("load"): if "size" in args: queries.load(sender, id, args.get("size")) self.log.debug("Load from %r: %r", sender, args) else: self.log.error("Load without an ID from %r: %r", sender, args) elif query.named("histogram"): start = args.get("start", None) end = args.get("end", None) step = args.get("step", None) if None not in (start, end, step): element = db.histogram(id, start, end, step) self.xmpp.core.message(sender, element) self.log.debug("Histogram from %r: %r", sender, args) elif query.named("cancel"): queries.cancel(sender, id) self.log.info("Cancel from %r: %r", sender, args) for sender, element in queries: yield self.xmpp.core.message(sender, element)
def _dst(self, dst): return idiokit.pipe( self._rooms.inc(dst), idiokit.consume())
def _handle_room(self, room_name): room = yield self.xmpp.muc.join(room_name, self.bot_name) distributor = yield self._ready.fork() yield idiokit.pipe(room, idiokit.map(self._map, room_name), distributor.fork(), idiokit.Event())
def test(): server, url = yield create_https_server("localhost") _, fileobj = yield idiokit.pipe(server, utils.fetch_url(url, verify=False)) self.assertEqual(fileobj.read(), "ok")
def parse(self, db): queries = QuerySet() while True: next = idiokit.next() if queries: idiokit.pipe(self._timeout(0.0), next) try: elements = yield next except Timeout: pass else: for element in elements.with_attrs("from"): sender = JID(element.get_attr("from")) if element.named("presence").with_attrs( type="unavailable"): db.purge_jid(sender) queries.discard_jid(sender) for message in element.named("message"): if not message.with_attrs(type="groupchat"): continue for event in events.Event.from_elements(message): db.add_event(sender, event) for query in element.named("message").children(ns=NS): try: args = json.loads(query.text) except JSONDecodeError: self.log.error("Invalid query data from %r: %r", sender, query.text) continue if "id" not in args: self.log.error("Query without an ID from %r: %r", sender, args) continue id = args.get("id") if query.named("start"): start = args.get("start", None) end = args.get("end", None) queries.start(sender, id, db.query(start, end)) self.log.info("Start from %r: %r", sender, args) elif query.named("load"): if "size" in args: queries.load(sender, id, args.get("size")) self.log.debug("Load from %r: %r", sender, args) else: self.log.error( "Load without an ID from %r: %r", sender, args) elif query.named("histogram"): start = args.get("start", None) end = args.get("end", None) step = args.get("step", None) if None not in (start, end, step): element = db.histogram(id, start, end, step) self.xmpp.core.message(sender, element) self.log.debug("Histogram from %r: %r", sender, args) elif query.named("cancel"): queries.cancel(sender, id) self.log.info("Cancel from %r: %r", sender, args) for sender, element in queries: yield self.xmpp.core.message(sender, element)
def collect(self, room_name): collect = self._collect(room_name) idiokit.pipe(self._alert(), collect) return collect