Esempio n. 1
0
 def _handle(self):
     while True:
         prefix, command, params = yield idiokit.next()
         if self.filter(prefix, command, params):
             event = self.parse(prefix, command, params)
             if event is not None:
                 yield idiokit.send(event)
Esempio n. 2
0
 def _follow_config(self):
     while True:
         ok, obj = yield idiokit.next()
         if not ok:
             self.log.error(obj)
             continue
         yield idiokit.send(set(obj))
Esempio n. 3
0
 def read(self):
     try:
         while True:
             configs = yield idiokit.next()
             self._updated = set(iter_startups(config.flatten(configs)))
     finally:
         self._updated = None
Esempio n. 4
0
    def _listen_configs(self, error_event):
        closing = False
        term_count = 0
        configs = frozenset()

        try:
            while True:
                try:
                    while not closing:
                        if frozenset(self._handlers) == configs:
                            configs = yield idiokit.next()
                            configs = frozenset(iter_startups(config.flatten(configs)))
                        yield self._apply(configs, error_event)
                    yield self._wait(self._handlers.values())
                except idiokit.Signal as sig:
                    closing = True

                    if sig.signum == signal.SIGUSR1:
                        self._clean(signal.SIGTERM)
                        continue

                    if sig.signum == signal.SIGUSR2:
                        self._clean(signal.SIGKILL)
                        continue

                    if term_count == 0:
                        self._clean(signal.SIGTERM)
                        term_count += 1
                        continue
                break
        finally:
            self._check()
Esempio n. 5
0
    def collect(self, ids, queue, time_window):
        while True:
            event = yield idiokit.next()

            eid = events.hexdigest(event, sha1)
            unique, event_set, augment_set = self._add(ids, queue, time_window, eid)
            event_set[unique] = event.union(*augment_set.values())
Esempio n. 6
0
    def _listen_configs(self, error_event):
        closing = False
        term_count = 0
        configs = frozenset()

        try:
            while True:
                try:
                    while not closing:
                        if frozenset(self._handlers) == configs:
                            configs = yield idiokit.next()
                            configs = frozenset(
                                iter_startups(config.flatten(configs)))
                        yield self._apply(configs, error_event)
                    yield self._wait(self._handlers.values())
                except idiokit.Signal as sig:
                    closing = True

                    if sig.signum == signal.SIGUSR1:
                        self._clean(signal.SIGTERM)
                        continue

                    if sig.signum == signal.SIGUSR2:
                        self._clean(signal.SIGKILL)
                        continue

                    if term_count == 0:
                        self._clean(signal.SIGTERM)
                        term_count += 1
                        continue
                break
        finally:
            self._check()
Esempio n. 7
0
 def _follow_config(self):
     while True:
         ok, obj = yield idiokit.next()
         if not ok:
             self.log.error(obj)
             continue
         yield idiokit.send(set(obj))
Esempio n. 8
0
    def normalize(self, name):
        while True:
            event = yield idiokit.next()

            # A dict telling how to rename raw event keys.
            # A key is not renamed by default.
            # Mapping a key to None removes the key.
            key_mappings = {
                "time": "source time",
                "id": "cleanmx id",
                "phishtank": "phishtank id",
                "line": None,
                "firsttime": "first seen",
                "lasttime": "last seen"
            }

            new = events.Event()
            for key, value in event.items():
                key = key_mappings.get(key, key)
                if key is None:
                    continue

                value = unescape(value).strip()
                if not value:
                    continue

                new.add(key, value)

            if name:
                new.add("feed", name)

            yield idiokit.send(new)
Esempio n. 9
0
    def _collect(self, room_name, compress):
        event = yield idiokit.next()

        while True:
            current = datetime.utcnow().day

            with _open_archive(self.archive_dir, time.time(), room_name) as archive:
                self.log.info("Opened archive {0!r}".format(archive.name))

                while current == datetime.utcnow().day:
                    json_dict = dict((key, event.values(key)) for key in event.keys())
                    archive.write(json.dumps(json_dict) + os.linesep)

                    event = yield idiokit.next()

            yield compress.queue(0.0, _rename(archive.name))
Esempio n. 10
0
    def normalize(self, name):
        while True:
            event = yield idiokit.next()

            # A dict telling how to rename raw event keys.
            # A key is not renamed by default.
            # Mapping a key to None removes the key.
            key_mappings = {
                "time": "source time",
                "id": "cleanmx id",
                "phishtank": "phishtank id",
                "line": None,
                "firsttime": "first seen",
                "lasttime": "last seen"
            }

            new = events.Event()
            for key, value in event.items():
                key = key_mappings.get(key, key)
                if key is None:
                    continue

                value = unescape(value).strip()
                if not value:
                    continue

                new.add(key, value)

            if name:
                new.add("feed", name)

            yield idiokit.send(new)
Esempio n. 11
0
def encode(sock):
    while True:
        msg = yield idiokit.next()
        msg_bytes = cPickle.dumps(msg, cPickle.HIGHEST_PROTOCOL)
        data = struct.pack("!I", len(msg_bytes)) + msg_bytes

        with wrapped_socket_errnos(errno.ECONNRESET, errno.EPIPE):
            yield sock.sendall(data)
Esempio n. 12
0
def encode(sock):
    while True:
        msg = yield idiokit.next()
        msg_bytes = cPickle.dumps(msg, cPickle.HIGHEST_PROTOCOL)
        data = struct.pack("!I", len(msg_bytes)) + msg_bytes

        with wrapped_socket_errnos(errno.ECONNRESET, errno.EPIPE):
            yield sock.sendall(data)
Esempio n. 13
0
    def collect(self, ids, queue, time_window):
        while True:
            event = yield idiokit.next()

            eid = events.hexdigest(event, sha1)
            unique, event_set, augment_set = self._add(ids, queue, time_window,
                                                       eid)
            event_set[unique] = event.union(*augment_set.values())
Esempio n. 14
0
def _collect_events():
    results = []
    while True:
        try:
            event = yield idiokit.next()
        except StopIteration:
            idiokit.stop(results)
        else:
            results.append(_event_to_dict(event))
Esempio n. 15
0
def _collect_events():
    results = []
    while True:
        try:
            event = yield idiokit.next()
        except StopIteration:
            idiokit.stop(results)
        else:
            results.append(_event_to_dict(event))
Esempio n. 16
0
    def _forward_files(self):
        while True:
            input_name, output_name = yield idiokit.next()

            ack = idiokit.Event()
            node = yield self._queue.queue(0, (input_name, output_name, ack))
            try:
                yield ack
            finally:
                yield self._queue.cancel(node)
Esempio n. 17
0
def _add_filename_info(groupdict):
    while True:
        event = yield idiokit.next()

        for key, value in groupdict.items():
            if None in (key, value):
                continue
            event.add(key, value)

        yield idiokit.send(event)
Esempio n. 18
0
    def _forward_files(self):
        while True:
            input_name, output_name = yield idiokit.next()

            ack = idiokit.Event()
            node = yield self._queue.queue(0, (input_name, output_name, ack))
            try:
                yield ack
            finally:
                yield self._queue.cancel(node)
Esempio n. 19
0
def _add_filename_info(groupdict):
    while True:
        event = yield idiokit.next()

        for key, value in groupdict.items():
            if None in (key, value):
                continue
            event.add(key, value)

        yield idiokit.send(event)
Esempio n. 20
0
    def aggregate(self, group_keys, window_info):
        """
        Create aggregated events and ids for them.
        """

        group_keys = tuple(set(group_keys))
        key_groups = dict()

        while True:
            yield timer.sleep(1.0)

            event = yield idiokit.next()

            current_time = time.time()
            updated = set()

            key = tuple(tuple(sorted(event.values(x))) for x in group_keys)
            updated.add(key)

            if key not in key_groups:
                windows = []
                for constructor, keys, output_key in window_info:
                    windows.append((constructor(**keys), output_key))
                key_groups[key] = windows

            for window, output_key in key_groups[key]:
                window.push(current_time, event)

            for key, windows in list(key_groups.iteritems()):
                any_expired = False
                for window, _ in windows:
                    any_expired = window.expire(current_time) or any_expired

                if not (any_expired or key in updated):
                    continue

                output = None

                for window, output_key in windows:
                    value = window.value()
                    if value is None:
                        continue

                    if output is None:
                        output = events.Event()
                    output.add(output_key, unicode(value))

                id = hashlib.md5(repr(key)).hexdigest()
                if output is not None:
                    for group_key, group_values in zip(group_keys, key):
                        output.update(group_key, group_values)

                yield idiokit.send(id, output)
                if output is None:
                    del key_groups[key]
Esempio n. 21
0
def _collect_set():
    result_set = set()

    while True:
        try:
            value = yield idiokit.next()
        except StopIteration:
            break
        result_set.add(value)

    idiokit.stop(result_set)
Esempio n. 22
0
def _normalize(subject):
    while True:
        event = yield idiokit.next()
        if subject is not None:
            event.add("report_subject", subject)

        for key in event.keys():
            event.discard(key, "")
            event.discard(key, "-")

        yield idiokit.send(event)
Esempio n. 23
0
def _normalize(subject):
    while True:
        event = yield idiokit.next()
        if subject is not None:
            event.add("report_subject", subject)

        for key in event.keys():
            event.discard(key, "")
            event.discard(key, "-")

        yield idiokit.send(event)
Esempio n. 24
0
 def _asn_lookup(self, event, resolver=None, cache_time=0):
     lookup = cymruwhois.OriginLookup(resolver, cache_time=cache_time)
     asn_dict = {}
     event = yield idiokit.next()
     if event.contains(self.ip_key):
         ip = event.values(self.ip_key)
         asns = yield lookup.lookup(ip[0])
         for asn in asns:
             for key, value in asn:
                 event.add(key, value)
     yield idiokit.send(event)
Esempio n. 25
0
    def _recv(self):
        dumps = json.JSONEncoder(check_circular=False).encode

        while True:
            event = yield idiokit.next()

            out_dict = {}
            for key, value in event.items():
                out_dict.setdefault(key, []).append(value)

            print dumps(out_dict)
Esempio n. 26
0
    def _recv(self):
        dumps = json.JSONEncoder(check_circular=False).encode

        while True:
            event = yield idiokit.next()

            out_dict = {}
            for key, value in event.items():
                out_dict.setdefault(key, []).append(value)

            print dumps(out_dict)
Esempio n. 27
0
 def _asn_names_lookup(self, resolver=None, cache_time=4 * 60 * 60):
     lookup = cymruwhois.ASNameLookup(resolver, cache_time=cache_time)
     while True:
         asn_dict = {}
         event = yield idiokit.next()
         if event.contains("asn"):
             asn_value = event.value("asn")
             asns = yield lookup.lookup(asn_value)
             for asn in asns:
                 for key, value in asn:
                     event.add(key, value)
         yield idiokit.send(event)
Esempio n. 28
0
    def _recv(self):
        while True:
            event = yield idiokit.next()

            out_dict = {}
            for key, value in event.items():
                out_dict.setdefault(key, []).append(value)

            for key, values in out_dict.iteritems():
                if len(values) == 1:
                    out_dict[key] = values[0]

            print json.dumps(out_dict)
Esempio n. 29
0
    def augment(self, ip_key, prefix):
        while True:
            eid, event = yield idiokit.next()

            for ip in event.values(ip_key):
                items = yield cymruwhois.lookup(ip)
                if not items:
                    continue

                augmentation = events.Event()
                for key, value in items:
                    augmentation.add(prefix + key, value)
                yield idiokit.send(eid, augmentation)
Esempio n. 30
0
    def _distribute(self):
        while True:
            src, event, dsts = yield idiokit.next()

            count = 0
            for dst in dsts:
                dst_room = self._rooms.get(dst)
                if dst_room is not None:
                    count += 1
                    yield dst_room.send(event.to_elements())

            if count > 0:
                self._inc_stats(src, sent=1)
Esempio n. 31
0
    def combine(self, ids, queue, time_window):
        while True:
            augment = yield idiokit.next()
            augment = events.Event(augment)

            eids = augment.values(AUGMENT_KEY)
            augment = augment.difference({AUGMENT_KEY: eids})

            for eid in eids:
                unique, event_set, augment_set = self._add(ids, queue, time_window, eid)
                augment_set[unique] = augment
                for unique, event in event_set.items():
                    event_set[unique] = event.union(augment)
Esempio n. 32
0
def _rate_limiter(rate_limit):
    last_output = time.time()

    while True:
        if rate_limit is not None:
            delta = max(time.time() - last_output, 0)
            delay = 1.0 / rate_limit - delta
            if delay > 0.0:
                yield idiokit.sleep(delay)
            last_output = time.time()

        msg = yield idiokit.next()
        yield idiokit.send(msg)
Esempio n. 33
0
    def _distribute(self):
        while True:
            src, event, dsts = yield idiokit.next()

            count = 0
            for dst in dsts:
                dst_room = self._rooms.get(dst)
                if dst_room is not None:
                    count += 1
                    yield dst_room.send(event.to_elements())

            if count > 0:
                self._inc_stats(src, sent=1)
Esempio n. 34
0
def _rate_limiter(rate_limit):
    last_output = time.time()

    while True:
        if rate_limit is not None:
            delta = max(time.time() - last_output, 0)
            delay = 1.0 / rate_limit - delta
            if delay > 0.0:
                yield idiokit.sleep(delay)
            last_output = time.time()

        msg = yield idiokit.next()
        yield idiokit.send(msg)
Esempio n. 35
0
    def _output_rate_limiter(self):
        while self.xmpp_rate_limit <= 0.0:
            yield idiokit.sleep(60.0)

        while True:
            delta = max(time.time() - self._last_output, 0)
            delay = 1.0 / self.xmpp_rate_limit - delta
            if delay > 0.0:
                yield idiokit.sleep(delay)
            self._last_output = time.time()

            msg = yield idiokit.next()
            yield idiokit.send(msg)
Esempio n. 36
0
    def _output_rate_limiter(self):
        while self.xmpp_rate_limit <= 0.0:
            yield idiokit.sleep(60.0)

        while True:
            delta = max(time.time() - self._last_output, 0)
            delay = 1.0 / self.xmpp_rate_limit - delta
            if delay > 0.0:
                yield idiokit.sleep(delay)
            self._last_output = time.time()

            msg = yield idiokit.next()
            yield idiokit.send(msg)
Esempio n. 37
0
    def process(self, ids, queue, window_time):
        while True:
            event = yield idiokit.next()

            current_time = time.time()
            expire_time = current_time + window_time

            eid = events.hexdigest(event)
            count, items = ids.get(eid, (0, event.items()))
            ids[eid] = count + 1, items

            if count == 0:
                yield idiokit.send(event.union({"id:open": eid}))
            queue.append((expire_time, eid))
Esempio n. 38
0
    def augment(self, *ip_keys):
        while True:
            event = yield idiokit.next()
            if not ip_keys:
                values = event.values(parser=_parse_ip)
            else:
                values = self._ip_values(event, ip_keys)

            for ip in values:
                items = yield self.lookup(ip)
                for key, value in items:
                    event.add(key, value)

            yield idiokit.send(event)
Esempio n. 39
0
    def augment(self, *ip_keys):
        while True:
            event = yield idiokit.next()
            if not ip_keys:
                values = event.values(parser=_parse_ip)
            else:
                values = self._ip_values(event, ip_keys)

            for ip in values:
                items = yield self.lookup(ip)
                for key, value in items:
                    event.add(key, value)

            yield idiokit.send(event)
Esempio n. 40
0
    def combine(self, ids, queue, time_window):
        while True:
            augment = yield idiokit.next()
            augment = events.Event(augment)

            eids = augment.values(AUGMENT_KEY)
            augment = augment.difference({AUGMENT_KEY: eids})

            for eid in eids:
                unique, event_set, augment_set = self._add(
                    ids, queue, time_window, eid)
                augment_set[unique] = augment
                for unique, event in event_set.items():
                    event_set[unique] = event.union(augment)
Esempio n. 41
0
    def collect(self, state, **keys):
        if state is None:
            state = utils.CompressedCollection()

        try:
            while True:
                event = yield idiokit.next()

                if event is self.REPORT_NOW:
                    yield idiokit.send(state)
                    state = utils.CompressedCollection()
                else:
                    state.append(event)
        except services.Stop:
            idiokit.stop(state)
Esempio n. 42
0
    def collect(self, state, **keys):
        if state is None:
            state = utils.CompressedCollection()

        try:
            while True:
                event = yield idiokit.next()

                if event is self.REPORT_NOW:
                    yield idiokit.send(state)
                    state = utils.CompressedCollection()
                else:
                    state.append(event)
        except services.Stop:
            idiokit.stop(state)
Esempio n. 43
0
    def run_mailbox(self, min_delay=5.0, max_delay=60.0):
        mailbox = None

        try:
            while True:
                item = yield idiokit.next()

                while True:
                    delay = min(min_delay, max_delay)
                    while mailbox is None:
                        try:
                            mailbox = yield idiokit.thread(self.connect)
                        except (imaplib.IMAP4.abort, socket.error) as error:
                            self.log.error(
                                "Failed IMAP connection ({0})".format(
                                    utils.format_exception(error)))
                        else:
                            break

                        self.log.info(
                            "Retrying connection in {0:.2f} seconds".format(
                                delay))
                        yield idiokit.sleep(delay)
                        delay = min(2 * delay, max_delay)

                    event, name, args, keys = item
                    if event.result().unsafe_is_set():
                        break

                    try:
                        method = getattr(mailbox, name)
                        result = yield idiokit.thread(method, *args, **keys)
                    except (imaplib.IMAP4.abort, socket.error) as error:
                        yield idiokit.thread(self.disconnect, mailbox)
                        self.log.error("Lost IMAP connection ({0})".format(
                            utils.format_exception(error)))
                        mailbox = None
                        event.fail(LostConnection, "", None)
                        break
                    except imaplib.IMAP4.error as error:
                        event.fail(type(error), error, None)
                        break
                    else:
                        event.succeed(result)
                        break
        finally:
            if mailbox is not None:
                yield idiokit.thread(self.disconnect, mailbox)
Esempio n. 44
0
    def _follow_config(self, abspath):
        workdir = os.path.dirname(abspath)
        while True:
            ok, obj = yield idiokit.next()
            if not ok:
                self.log.error(obj)
                continue

            output = set()
            for conf in iter_startups(obj):
                if self.disable is not None and conf.name in self.disable:
                    continue
                if self.enable is not None and conf.name not in self.enable:
                    continue
                output.add(conf.with_workdir(workdir))
            yield idiokit.send(output)
    def normalize(self, subject, groupdict):
        while True:
            event = yield idiokit.next()
            if subject is not None:
                event.add("report_subject", subject)

            for key in event.keys():
                event.discard(key, "")
                event.discard(key, "-")

            for key, value in groupdict.items():
                if None in (key, value):
                    continue
                event.add(key, value)

            yield idiokit.send(event)
Esempio n. 46
0
    def _follow_config(self, abspath):
        workdir = os.path.dirname(abspath)
        while True:
            ok, obj = yield idiokit.next()
            if not ok:
                self.log.error(obj)
                continue

            output = set()
            for conf in iter_startups(obj):
                if self.disable is not None and conf.name in self.disable:
                    continue
                if self.enable is not None and conf.name not in self.enable:
                    continue
                output.add(conf.with_workdir(workdir))
            yield idiokit.send(output)
Esempio n. 47
0
    def process(self, ids, queue, window_time):
        while True:
            event = yield idiokit.next()

            current_time = time.time()
            expire_time = current_time + window_time

            eid = events.hexdigest(event)
            count, items = ids.get(eid, (0, event.items()))
            ids[eid] = count + 1, items

            if count == 0:
                yield idiokit.send(event.union({
                    "id:open": eid
                }))
            queue.append((expire_time, eid))
Esempio n. 48
0
def _collect_texts_and_attachments():
    texts = []
    attachments = []

    while True:
        try:
            msg_part = yield idiokit.next()
        except StopIteration:
            idiokit.stop(texts, attachments)

        content_type = msg_part.get_content_type()
        filename = msg_part.get_filename(None)

        if filename is not None:
            attachments.append(msg_part)
        elif content_type == "text/plain":
            texts.append(msg_part)
Esempio n. 49
0
def distribute_encode(socks):
    writable = []

    while True:
        to_all, msg = yield idiokit.next()
        msg_bytes = cPickle.dumps(msg, cPickle.HIGHEST_PROTOCOL)
        data = struct.pack("!I", len(msg_bytes)) + msg_bytes

        if to_all:
            for sock in socks:
                yield sock.sendall(data)
            writable = []
        else:
            while not writable:
                _, writable, _ = yield select.select((), socks, ())
                writable = list(writable)
            yield writable.pop().sendall(data)
Esempio n. 50
0
    def throttle(self, throttle_time):
        """
        Ensure that updates for a given event id are sent at least
        throttle_time apart.
        """

        sleeper = timer.sleep(1.0)

        ids = dict()
        queue = list()

        while True:
            yield timer.sleep(1.0)

            item = yield idiokit.next()
            current_time = time.time()

            id, event = item
            previous, _, update_time = ids.get(id, (None, None, None))
            if update_time is None:
                update_time = current_time + 1.0
                heapq.heappush(queue, (update_time, id))
            ids[id] = previous, event, update_time

            while queue and queue[0][0] <= current_time:
                _, id = heapq.heappop(queue)

                previous, next, _ = ids[id]
                if previous == next == None:
                    del ids[id]
                    continue

                if previous != next:
                    if next is None:
                        event = events.Event()
                    else:
                        event = events.Event(next)
                    event.add("id", id)
                    yield idiokit.send(event)

                if previous != next:
                    update_time = current_time + throttle_time
                    heapq.heappush(queue, (update_time, id))
                else:
                    update_time = None
                ids[id] = next, next, update_time
Esempio n. 51
0
    def _cutoff(self):
        while True:
            event = yield idiokit.next()

            latest = None
            for value in event.values("source time"):
                try:
                    source_time = time.strptime(value, "%Y-%m-%d %H:%M:%SZ")
                except ValueError:
                    continue
                latest = max(latest, source_time)

            cutoff = time.gmtime(time.time() - self.drop_older_than)
            if latest and latest < cutoff:
                continue

            yield idiokit.send(event)
Esempio n. 52
0
    def _cutoff(self):
        while True:
            event = yield idiokit.next()

            latest = None
            for value in event.values("source time"):
                try:
                    source_time = time.strptime(value, "%Y-%m-%d %H:%M:%SZ")
                except ValueError:
                    continue
                latest = max(latest, source_time)

            cutoff = time.gmtime(time.time() - self.drop_older_than)
            if latest and latest < cutoff:
                continue

            yield idiokit.send(event)
Esempio n. 53
0
def distribute_encode(socks):
    writable = []

    while True:
        to_all, msg = yield idiokit.next()
        msg_bytes = cPickle.dumps(msg, cPickle.HIGHEST_PROTOCOL)
        data = struct.pack("!I", len(msg_bytes)) + msg_bytes

        if to_all:
            for sock in socks:
                yield sock.sendall(data)
            writable = []
        else:
            while not writable:
                _, writable, _ = yield select.select((), socks, ())
                writable = list(writable)
            yield writable.pop().sendall(data)
Esempio n. 54
0
    def _run(self):
        with self.xmpp.core.iq_handler(self.handle_iq, "start", SERVICE_NS):
            while True:
                elements = yield idiokit.next()

                for message in elements.named("message").with_attrs("from"):
                    for end in message.children("end", SERVICE_NS).with_attrs("id"):
                        jid = JID(message.get_attr("from"))
                        session_id = end.get_attr("id")
                        self._discard_session(jid, session_id)

                presences = elements.named("presence").with_attrs("from")
                for presence in presences:
                    jid = JID(presence.get_attr("from"))
                    if presence.with_attrs(type="unavailable"):
                        self._discard_jid(jid, Unavailable())
                    else:
                        self._update_catalogue(jid, presence.children())
Esempio n. 55
0
    def _handle_updates(self, lobby, errors):
        queues = dict()
        sessions = dict()

        try:
            while True:
                configs = yield idiokit.next()

                added = set(iter_runtimes(config.flatten(configs)))

                for key in set(sessions) - added:
                    stream = sessions.pop(key)
                    stream.throw(Cancel())

                for session in added - set(sessions):
                    sessions[session] = self._session(lobby, session, queues) | self._catch(errors)
        finally:
            for stream in sessions.values():
                stream.throw(Cancel())