def _handle(self):
while True:
prefix, command, params = yield idiokit.next()
if self.filter(prefix, command, params):
event = self.parse(prefix, command, params)
if event is not None:
yield idiokit.send(event)
def _follow_config(self):
while True:
ok, obj = yield idiokit.next()
if not ok:
self.log.error(obj)
continue
yield idiokit.send(set(obj))
def read(self):
try:
while True:
configs = yield idiokit.next()
self._updated = set(iter_startups(config.flatten(configs)))
finally:
self._updated = None
def _listen_configs(self, error_event):
closing = False
term_count = 0
configs = frozenset()
try:
while True:
try:
while not closing:
if frozenset(self._handlers) == configs:
configs = yield idiokit.next()
configs = frozenset(iter_startups(config.flatten(configs)))
yield self._apply(configs, error_event)
yield self._wait(self._handlers.values())
except idiokit.Signal as sig:
closing = True
if sig.signum == signal.SIGUSR1:
self._clean(signal.SIGTERM)
continue
if sig.signum == signal.SIGUSR2:
self._clean(signal.SIGKILL)
continue
if term_count == 0:
self._clean(signal.SIGTERM)
term_count += 1
continue
break
finally:
self._check()
def collect(self, ids, queue, time_window):
while True:
event = yield idiokit.next()
eid = events.hexdigest(event, sha1)
unique, event_set, augment_set = self._add(ids, queue, time_window, eid)
event_set[unique] = event.union(*augment_set.values())
def _listen_configs(self, error_event):
closing = False
term_count = 0
configs = frozenset()
try:
while True:
try:
while not closing:
if frozenset(self._handlers) == configs:
configs = yield idiokit.next()
configs = frozenset(
iter_startups(config.flatten(configs)))
yield self._apply(configs, error_event)
yield self._wait(self._handlers.values())
except idiokit.Signal as sig:
closing = True
if sig.signum == signal.SIGUSR1:
self._clean(signal.SIGTERM)
continue
if sig.signum == signal.SIGUSR2:
self._clean(signal.SIGKILL)
continue
if term_count == 0:
self._clean(signal.SIGTERM)
term_count += 1
continue
break
finally:
self._check()
def _follow_config(self):
while True:
ok, obj = yield idiokit.next()
if not ok:
self.log.error(obj)
continue
yield idiokit.send(set(obj))
def normalize(self, name):
while True:
event = yield idiokit.next()
# A dict telling how to rename raw event keys.
# A key is not renamed by default.
# Mapping a key to None removes the key.
key_mappings = {
"time": "source time",
"id": "cleanmx id",
"phishtank": "phishtank id",
"line": None,
"firsttime": "first seen",
"lasttime": "last seen"
}
new = events.Event()
for key, value in event.items():
key = key_mappings.get(key, key)
if key is None:
continue
value = unescape(value).strip()
if not value:
continue
new.add(key, value)
if name:
new.add("feed", name)
yield idiokit.send(new)
def _collect(self, room_name, compress):
event = yield idiokit.next()
while True:
current = datetime.utcnow().day
with _open_archive(self.archive_dir, time.time(), room_name) as archive:
self.log.info("Opened archive {0!r}".format(archive.name))
while current == datetime.utcnow().day:
json_dict = dict((key, event.values(key)) for key in event.keys())
archive.write(json.dumps(json_dict) + os.linesep)
event = yield idiokit.next()
yield compress.queue(0.0, _rename(archive.name))
def normalize(self, name):
while True:
event = yield idiokit.next()
# A dict telling how to rename raw event keys.
# A key is not renamed by default.
# Mapping a key to None removes the key.
key_mappings = {
"time": "source time",
"id": "cleanmx id",
"phishtank": "phishtank id",
"line": None,
"firsttime": "first seen",
"lasttime": "last seen"
}
new = events.Event()
for key, value in event.items():
key = key_mappings.get(key, key)
if key is None:
continue
value = unescape(value).strip()
if not value:
continue
new.add(key, value)
if name:
new.add("feed", name)
yield idiokit.send(new)
def encode(sock):
while True:
msg = yield idiokit.next()
msg_bytes = cPickle.dumps(msg, cPickle.HIGHEST_PROTOCOL)
data = struct.pack("!I", len(msg_bytes)) + msg_bytes
with wrapped_socket_errnos(errno.ECONNRESET, errno.EPIPE):
yield sock.sendall(data)
def encode(sock):
while True:
msg = yield idiokit.next()
msg_bytes = cPickle.dumps(msg, cPickle.HIGHEST_PROTOCOL)
data = struct.pack("!I", len(msg_bytes)) + msg_bytes
with wrapped_socket_errnos(errno.ECONNRESET, errno.EPIPE):
yield sock.sendall(data)
def collect(self, ids, queue, time_window):
while True:
event = yield idiokit.next()
eid = events.hexdigest(event, sha1)
unique, event_set, augment_set = self._add(ids, queue, time_window,
eid)
event_set[unique] = event.union(*augment_set.values())
def _collect_events():
results = []
while True:
try:
event = yield idiokit.next()
except StopIteration:
idiokit.stop(results)
else:
results.append(_event_to_dict(event))
def _collect_events():
results = []
while True:
try:
event = yield idiokit.next()
except StopIteration:
idiokit.stop(results)
else:
results.append(_event_to_dict(event))
def _forward_files(self):
while True:
input_name, output_name = yield idiokit.next()
ack = idiokit.Event()
node = yield self._queue.queue(0, (input_name, output_name, ack))
try:
yield ack
finally:
yield self._queue.cancel(node)
def _add_filename_info(groupdict):
while True:
event = yield idiokit.next()
for key, value in groupdict.items():
if None in (key, value):
continue
event.add(key, value)
yield idiokit.send(event)
def _forward_files(self):
while True:
input_name, output_name = yield idiokit.next()
ack = idiokit.Event()
node = yield self._queue.queue(0, (input_name, output_name, ack))
try:
yield ack
finally:
yield self._queue.cancel(node)
def _add_filename_info(groupdict):
while True:
event = yield idiokit.next()
for key, value in groupdict.items():
if None in (key, value):
continue
event.add(key, value)
yield idiokit.send(event)
def aggregate(self, group_keys, window_info):
"""
Create aggregated events and ids for them.
"""
group_keys = tuple(set(group_keys))
key_groups = dict()
while True:
yield timer.sleep(1.0)
event = yield idiokit.next()
current_time = time.time()
updated = set()
key = tuple(tuple(sorted(event.values(x))) for x in group_keys)
updated.add(key)
if key not in key_groups:
windows = []
for constructor, keys, output_key in window_info:
windows.append((constructor(**keys), output_key))
key_groups[key] = windows
for window, output_key in key_groups[key]:
window.push(current_time, event)
for key, windows in list(key_groups.iteritems()):
any_expired = False
for window, _ in windows:
any_expired = window.expire(current_time) or any_expired
if not (any_expired or key in updated):
continue
output = None
for window, output_key in windows:
value = window.value()
if value is None:
continue
if output is None:
output = events.Event()
output.add(output_key, unicode(value))
id = hashlib.md5(repr(key)).hexdigest()
if output is not None:
for group_key, group_values in zip(group_keys, key):
output.update(group_key, group_values)
yield idiokit.send(id, output)
if output is None:
del key_groups[key]
def _collect_set():
result_set = set()
while True:
try:
value = yield idiokit.next()
except StopIteration:
break
result_set.add(value)
idiokit.stop(result_set)
def _normalize(subject):
while True:
event = yield idiokit.next()
if subject is not None:
event.add("report_subject", subject)
for key in event.keys():
event.discard(key, "")
event.discard(key, "-")
yield idiokit.send(event)
def _normalize(subject):
while True:
event = yield idiokit.next()
if subject is not None:
event.add("report_subject", subject)
for key in event.keys():
event.discard(key, "")
event.discard(key, "-")
yield idiokit.send(event)
def _asn_lookup(self, event, resolver=None, cache_time=0):
lookup = cymruwhois.OriginLookup(resolver, cache_time=cache_time)
asn_dict = {}
event = yield idiokit.next()
if event.contains(self.ip_key):
ip = event.values(self.ip_key)
asns = yield lookup.lookup(ip[0])
for asn in asns:
for key, value in asn:
event.add(key, value)
yield idiokit.send(event)
def _recv(self):
dumps = json.JSONEncoder(check_circular=False).encode
while True:
event = yield idiokit.next()
out_dict = {}
for key, value in event.items():
out_dict.setdefault(key, []).append(value)
print dumps(out_dict)
def _recv(self):
dumps = json.JSONEncoder(check_circular=False).encode
while True:
event = yield idiokit.next()
out_dict = {}
for key, value in event.items():
out_dict.setdefault(key, []).append(value)
print dumps(out_dict)
def _asn_names_lookup(self, resolver=None, cache_time=4 * 60 * 60):
lookup = cymruwhois.ASNameLookup(resolver, cache_time=cache_time)
while True:
asn_dict = {}
event = yield idiokit.next()
if event.contains("asn"):
asn_value = event.value("asn")
asns = yield lookup.lookup(asn_value)
for asn in asns:
for key, value in asn:
event.add(key, value)
yield idiokit.send(event)
def _recv(self):
while True:
event = yield idiokit.next()
out_dict = {}
for key, value in event.items():
out_dict.setdefault(key, []).append(value)
for key, values in out_dict.iteritems():
if len(values) == 1:
out_dict[key] = values[0]
print json.dumps(out_dict)
def augment(self, ip_key, prefix):
while True:
eid, event = yield idiokit.next()
for ip in event.values(ip_key):
items = yield cymruwhois.lookup(ip)
if not items:
continue
augmentation = events.Event()
for key, value in items:
augmentation.add(prefix + key, value)
yield idiokit.send(eid, augmentation)
def _distribute(self):
while True:
src, event, dsts = yield idiokit.next()
count = 0
for dst in dsts:
dst_room = self._rooms.get(dst)
if dst_room is not None:
count += 1
yield dst_room.send(event.to_elements())
if count > 0:
self._inc_stats(src, sent=1)
def combine(self, ids, queue, time_window):
while True:
augment = yield idiokit.next()
augment = events.Event(augment)
eids = augment.values(AUGMENT_KEY)
augment = augment.difference({AUGMENT_KEY: eids})
for eid in eids:
unique, event_set, augment_set = self._add(ids, queue, time_window, eid)
augment_set[unique] = augment
for unique, event in event_set.items():
event_set[unique] = event.union(augment)
def _rate_limiter(rate_limit):
last_output = time.time()
while True:
if rate_limit is not None:
delta = max(time.time() - last_output, 0)
delay = 1.0 / rate_limit - delta
if delay > 0.0:
yield idiokit.sleep(delay)
last_output = time.time()
msg = yield idiokit.next()
yield idiokit.send(msg)
def _distribute(self):
while True:
src, event, dsts = yield idiokit.next()
count = 0
for dst in dsts:
dst_room = self._rooms.get(dst)
if dst_room is not None:
count += 1
yield dst_room.send(event.to_elements())
if count > 0:
self._inc_stats(src, sent=1)
def _rate_limiter(rate_limit):
last_output = time.time()
while True:
if rate_limit is not None:
delta = max(time.time() - last_output, 0)
delay = 1.0 / rate_limit - delta
if delay > 0.0:
yield idiokit.sleep(delay)
last_output = time.time()
msg = yield idiokit.next()
yield idiokit.send(msg)
def _output_rate_limiter(self):
while self.xmpp_rate_limit <= 0.0:
yield idiokit.sleep(60.0)
while True:
delta = max(time.time() - self._last_output, 0)
delay = 1.0 / self.xmpp_rate_limit - delta
if delay > 0.0:
yield idiokit.sleep(delay)
self._last_output = time.time()
msg = yield idiokit.next()
yield idiokit.send(msg)
def _output_rate_limiter(self):
while self.xmpp_rate_limit <= 0.0:
yield idiokit.sleep(60.0)
while True:
delta = max(time.time() - self._last_output, 0)
delay = 1.0 / self.xmpp_rate_limit - delta
if delay > 0.0:
yield idiokit.sleep(delay)
self._last_output = time.time()
msg = yield idiokit.next()
yield idiokit.send(msg)
def process(self, ids, queue, window_time):
while True:
event = yield idiokit.next()
current_time = time.time()
expire_time = current_time + window_time
eid = events.hexdigest(event)
count, items = ids.get(eid, (0, event.items()))
ids[eid] = count + 1, items
if count == 0:
yield idiokit.send(event.union({"id:open": eid}))
queue.append((expire_time, eid))
def augment(self, *ip_keys):
while True:
event = yield idiokit.next()
if not ip_keys:
values = event.values(parser=_parse_ip)
else:
values = self._ip_values(event, ip_keys)
for ip in values:
items = yield self.lookup(ip)
for key, value in items:
event.add(key, value)
yield idiokit.send(event)
def augment(self, *ip_keys):
while True:
event = yield idiokit.next()
if not ip_keys:
values = event.values(parser=_parse_ip)
else:
values = self._ip_values(event, ip_keys)
for ip in values:
items = yield self.lookup(ip)
for key, value in items:
event.add(key, value)
yield idiokit.send(event)
def combine(self, ids, queue, time_window):
while True:
augment = yield idiokit.next()
augment = events.Event(augment)
eids = augment.values(AUGMENT_KEY)
augment = augment.difference({AUGMENT_KEY: eids})
for eid in eids:
unique, event_set, augment_set = self._add(
ids, queue, time_window, eid)
augment_set[unique] = augment
for unique, event in event_set.items():
event_set[unique] = event.union(augment)
def collect(self, state, **keys):
if state is None:
state = utils.CompressedCollection()
try:
while True:
event = yield idiokit.next()
if event is self.REPORT_NOW:
yield idiokit.send(state)
state = utils.CompressedCollection()
else:
state.append(event)
except services.Stop:
idiokit.stop(state)
def collect(self, state, **keys):
if state is None:
state = utils.CompressedCollection()
try:
while True:
event = yield idiokit.next()
if event is self.REPORT_NOW:
yield idiokit.send(state)
state = utils.CompressedCollection()
else:
state.append(event)
except services.Stop:
idiokit.stop(state)
def run_mailbox(self, min_delay=5.0, max_delay=60.0):
mailbox = None
try:
while True:
item = yield idiokit.next()
while True:
delay = min(min_delay, max_delay)
while mailbox is None:
try:
mailbox = yield idiokit.thread(self.connect)
except (imaplib.IMAP4.abort, socket.error) as error:
self.log.error(
"Failed IMAP connection ({0})".format(
utils.format_exception(error)))
else:
break
self.log.info(
"Retrying connection in {0:.2f} seconds".format(
delay))
yield idiokit.sleep(delay)
delay = min(2 * delay, max_delay)
event, name, args, keys = item
if event.result().unsafe_is_set():
break
try:
method = getattr(mailbox, name)
result = yield idiokit.thread(method, *args, **keys)
except (imaplib.IMAP4.abort, socket.error) as error:
yield idiokit.thread(self.disconnect, mailbox)
self.log.error("Lost IMAP connection ({0})".format(
utils.format_exception(error)))
mailbox = None
event.fail(LostConnection, "", None)
break
except imaplib.IMAP4.error as error:
event.fail(type(error), error, None)
break
else:
event.succeed(result)
break
finally:
if mailbox is not None:
yield idiokit.thread(self.disconnect, mailbox)
def _follow_config(self, abspath):
workdir = os.path.dirname(abspath)
while True:
ok, obj = yield idiokit.next()
if not ok:
self.log.error(obj)
continue
output = set()
for conf in iter_startups(obj):
if self.disable is not None and conf.name in self.disable:
continue
if self.enable is not None and conf.name not in self.enable:
continue
output.add(conf.with_workdir(workdir))
yield idiokit.send(output)
def normalize(self, subject, groupdict):
while True:
event = yield idiokit.next()
if subject is not None:
event.add("report_subject", subject)
for key in event.keys():
event.discard(key, "")
event.discard(key, "-")
for key, value in groupdict.items():
if None in (key, value):
continue
event.add(key, value)
yield idiokit.send(event)
def _follow_config(self, abspath):
workdir = os.path.dirname(abspath)
while True:
ok, obj = yield idiokit.next()
if not ok:
self.log.error(obj)
continue
output = set()
for conf in iter_startups(obj):
if self.disable is not None and conf.name in self.disable:
continue
if self.enable is not None and conf.name not in self.enable:
continue
output.add(conf.with_workdir(workdir))
yield idiokit.send(output)
def process(self, ids, queue, window_time):
while True:
event = yield idiokit.next()
current_time = time.time()
expire_time = current_time + window_time
eid = events.hexdigest(event)
count, items = ids.get(eid, (0, event.items()))
ids[eid] = count + 1, items
if count == 0:
yield idiokit.send(event.union({
"id:open": eid
}))
queue.append((expire_time, eid))
def _collect_texts_and_attachments():
texts = []
attachments = []
while True:
try:
msg_part = yield idiokit.next()
except StopIteration:
idiokit.stop(texts, attachments)
content_type = msg_part.get_content_type()
filename = msg_part.get_filename(None)
if filename is not None:
attachments.append(msg_part)
elif content_type == "text/plain":
texts.append(msg_part)
def distribute_encode(socks):
writable = []
while True:
to_all, msg = yield idiokit.next()
msg_bytes = cPickle.dumps(msg, cPickle.HIGHEST_PROTOCOL)
data = struct.pack("!I", len(msg_bytes)) + msg_bytes
if to_all:
for sock in socks:
yield sock.sendall(data)
writable = []
else:
while not writable:
_, writable, _ = yield select.select((), socks, ())
writable = list(writable)
yield writable.pop().sendall(data)
def throttle(self, throttle_time):
"""
Ensure that updates for a given event id are sent at least
throttle_time apart.
"""
sleeper = timer.sleep(1.0)
ids = dict()
queue = list()
while True:
yield timer.sleep(1.0)
item = yield idiokit.next()
current_time = time.time()
id, event = item
previous, _, update_time = ids.get(id, (None, None, None))
if update_time is None:
update_time = current_time + 1.0
heapq.heappush(queue, (update_time, id))
ids[id] = previous, event, update_time
while queue and queue[0][0] <= current_time:
_, id = heapq.heappop(queue)
previous, next, _ = ids[id]
if previous == next == None:
del ids[id]
continue
if previous != next:
if next is None:
event = events.Event()
else:
event = events.Event(next)
event.add("id", id)
yield idiokit.send(event)
if previous != next:
update_time = current_time + throttle_time
heapq.heappush(queue, (update_time, id))
else:
update_time = None
ids[id] = next, next, update_time
def _cutoff(self):
while True:
event = yield idiokit.next()
latest = None
for value in event.values("source time"):
try:
source_time = time.strptime(value, "%Y-%m-%d %H:%M:%SZ")
except ValueError:
continue
latest = max(latest, source_time)
cutoff = time.gmtime(time.time() - self.drop_older_than)
if latest and latest < cutoff:
continue
yield idiokit.send(event)
def _cutoff(self):
while True:
event = yield idiokit.next()
latest = None
for value in event.values("source time"):
try:
source_time = time.strptime(value, "%Y-%m-%d %H:%M:%SZ")
except ValueError:
continue
latest = max(latest, source_time)
cutoff = time.gmtime(time.time() - self.drop_older_than)
if latest and latest < cutoff:
continue
yield idiokit.send(event)
def distribute_encode(socks):
writable = []
while True:
to_all, msg = yield idiokit.next()
msg_bytes = cPickle.dumps(msg, cPickle.HIGHEST_PROTOCOL)
data = struct.pack("!I", len(msg_bytes)) + msg_bytes
if to_all:
for sock in socks:
yield sock.sendall(data)
writable = []
else:
while not writable:
_, writable, _ = yield select.select((), socks, ())
writable = list(writable)
yield writable.pop().sendall(data)
def _run(self):
with self.xmpp.core.iq_handler(self.handle_iq, "start", SERVICE_NS):
while True:
elements = yield idiokit.next()
for message in elements.named("message").with_attrs("from"):
for end in message.children("end", SERVICE_NS).with_attrs("id"):
jid = JID(message.get_attr("from"))
session_id = end.get_attr("id")
self._discard_session(jid, session_id)
presences = elements.named("presence").with_attrs("from")
for presence in presences:
jid = JID(presence.get_attr("from"))
if presence.with_attrs(type="unavailable"):
self._discard_jid(jid, Unavailable())
else:
self._update_catalogue(jid, presence.children())
def _handle_updates(self, lobby, errors):
queues = dict()
sessions = dict()
try:
while True:
configs = yield idiokit.next()
added = set(iter_runtimes(config.flatten(configs)))
for key in set(sessions) - added:
stream = sessions.pop(key)
stream.throw(Cancel())
for session in added - set(sessions):
sessions[session] = self._session(lobby, session, queues) | self._catch(errors)
finally:
for stream in sessions.values():
stream.throw(Cancel())