class BruteForceBlockerBot(bot.PollingBot): # Ignore the last column ("id"). COLUMNS = ["ip", "time", "count", None] use_cymru_whois = bot.BoolParam() def poll(self): if self.use_cymru_whois: return self._poll() | cymruwhois.augment("ip") return self._poll() @idiokit.stream def _poll( self, url="http://danger.rulez.sk/projects/bruteforceblocker/blist.php"): self.log.info("Downloading %s" % url) try: info, fileobj = yield utils.fetch_url(url) except utils.FetchUrlFailed, fuf: self.log.error("Download failed: %r", fuf) idiokit.stop(False) self.log.info("Downloaded") filtered = (x for x in fileobj if x.strip() and not x.startswith("#")) lines = (re.sub("\t+", "\t", x) for x in filtered) yield (utils.csv_to_events(lines, delimiter="\t", columns=self.COLUMNS, charset=info.get_param("charset")) | idiokit.map(self._normalize, url))
class AutoshunBot(bot.PollingBot): COLUMNS = ["ip", "time", "info"] time_offset = 5 feed_url = bot.Param(default=AUTOSHUN_CSV_URL) use_cymru_whois = bot.BoolParam() def poll(self): pipe = self._poll(url=self.feed_url) if self.use_cymru_whois: pipe = pipe | cymruwhois.augment("ip") return pipe | self._normalize() @idiokit.stream def _poll(self, url): self.log.info("Downloading %s" % url) try: info, fileobj = yield utils.fetch_url(url) except utils.FetchUrlFailed, fuf: self.log.error("Download failed: %r", fuf) idiokit.stop() self.log.info("Downloaded") # Grab time offset from first line of the CSV header = fileobj.readline() # Source file header row may sometimes be empty if header.startswith("Shunlist as of"): offset = -1 * int(header[-5:]) / 100 # ex: -0500 to 5 self.time_offset = offset if -12 <= offset <= 12 else 5 yield utils.csv_to_events(fileobj, columns=self.COLUMNS, charset=info.get_param("charset"))
class OpenBLBot(bot.PollingBot): feed_url = bot.Param(default=OPENBL_FEED_URL) use_cymru_whois = bot.BoolParam() def poll(self): pipe = self._poll(url=self.feed_url) if self.use_cymru_whois: pipe = pipe | cymruwhois.augment("ip") return pipe @idiokit.stream def _poll(self, url): self.log.info("Downloading %s" % url) try: info, fileobj = yield utils.fetch_url(url) except utils.FetchUrlFailed, fuf: raise bot.PollSkipped("failed to download {0!r} ({1})".format(url, fuf)) self.log.info("Downloaded") for line in fileobj: event = _parse_line(line) if event is None: continue event.add("feeder", "openbl.org") event.add("feed", "openbl") event.add("description url", self.feed_url) event.add("type", "brute-force") event.add( "description", "This host has most likely been performing brute-force " + "attacks on one of the following services: FTP, SSH, POP3, " + "IMAP, IMAPS or POP3S." ) yield idiokit.send(event)
class DataplaneBot(bot.PollingBot): url = bot.Param() use_cymru_whois = bot.BoolParam() # The first column values (ASN and AS name) are ignored. COLUMNS = [None, None, "ip", "time", "category"] def poll(self): if self.use_cymru_whois: return self._poll() | cymruwhois.augment("ip") return self._poll() @idiokit.stream def _poll(self): self.log.info("Downloading %s" % self.url) try: info, fileobj = yield utils.fetch_url(self.url) except utils.FetchUrlFailed, fuf: self.log.error("Download failed: %r", fuf) return self.log.info("Downloaded") charset = info.get_param("charset") filtered = (x for x in fileobj if x.strip() and not x.startswith("#")) yield utils.csv_to_events(filtered, delimiter="|", columns=self.COLUMNS, charset=charset)
class SpamhausDropBot(bot.PollingBot): use_cymru_whois = bot.BoolParam() http_headers = bot.ListParam("a list of http header (k, v) tuples", default=[]) @idiokit.stream def poll(self, url="https://www.spamhaus.org/drop/drop.lasso"): request = urllib2.Request(url) for key, value in self.http_headers: request.add_header(key, value) self.log.info("Downloading %s" % url) try: info, fileobj = yield utils.fetch_url(request) except utils.FetchUrlFailed as fuf: self.log.error("Download failed: %r", fuf) idiokit.stop(False) self.log.info("Downloaded") for line in fileobj.readlines(): if line.startswith(';'): continue data = line.split(';') if not data: continue netblock_sbl = [x.strip() for x in data] if len(netblock_sbl) != 2: continue netblock, sbl = netblock_sbl if not len(netblock.split('/')) == 2: continue new = events.Event() new.add('netblock', netblock) new.add('description url', "http://www.spamhaus.org/sbl/query/" + sbl) new.add('feeder', 'spamhaus') new.add('feed', 'spamhaus drop list') new.add('type', 'hijacked network') if self.use_cymru_whois: values = yield cymruwhois.lookup(netblock.split('/')[0]) for key, value in values: new.add(key, value) yield idiokit.send(new)
class BridgeBot(bot.Bot): xmpp_src_jid = bot.Param("the XMPP src JID") xmpp_src_password = bot.Param("the XMPP src password", default=None) xmpp_src_room = bot.Param("the XMPP src room") xmpp_src_host = bot.Param( "the XMPP src service host (default: autodetect)", default=None) xmpp_src_port = bot.IntParam( "the XMPP src service port (default: autodetect)", default=None) xmpp_src_ignore_cert = bot.BoolParam(""" do not perform any verification for the XMPP service's SSL certificate """) xmpp_src_extra_ca_certs = bot.Param(""" a PEM formatted file of CAs to be used in addition to the system CAs """, default=None) xmpp_dst_jid = bot.Param("the XMPP dst JID") xmpp_dst_password = bot.Param("the XMPP dst password", default=None) xmpp_dst_host = bot.Param( "the XMPP dst service host (default: autodetect)", default=None) xmpp_dst_port = bot.IntParam( "the XMPP dst service port (default: autodetect)", default=None) xmpp_dst_room = bot.Param("the XMPP dst room") xmpp_dst_ignore_cert = bot.BoolParam(""" do not perform any verification for the XMPP service's SSL certificate """) xmpp_dst_extra_ca_certs = bot.Param(""" a PEM formatted file of CAs to be used in addition to the system CAs """, default=None) def __init__(self, **keys): bot.Bot.__init__(self, **keys) if self.xmpp_src_password is None: self.xmpp_src_password = getpass.getpass("XMPP src password: "******"XMPP dst password: "******"Connecting to XMPP %s server with JID %r", _type, jid) connection = yield xmpp.connect(jid, password, host=host, port=port, ssl_verify_cert=verify_cert, ssl_ca_certs=ca_certs) self.log.info("Connected to XMPP %s server with JID %r", _type, jid) connection.core.presence() self.log.info("Joining %s room %r", _type, room_name) room = yield connection.muc.join(room_name, self.bot_name) self.log.info("Joined %s room %r", _type, room_name) idiokit.stop(room) @idiokit.stream def main(self): dst = yield self._join("dst", self.xmpp_dst_jid, self.xmpp_dst_password, self.xmpp_dst_host, self.xmpp_dst_port, self.xmpp_dst_ignore_cert, self.xmpp_dst_extra_ca_certs, self.xmpp_dst_room) src = yield self._join("src", self.xmpp_src_jid, self.xmpp_src_password, self.xmpp_src_host, self.xmpp_src_port, self.xmpp_src_ignore_cert, self.xmpp_src_extra_ca_certs, self.xmpp_src_room) yield src | peel_messages() | dst | idiokit.consume() def run(self): try: return idiokit.main_loop(self.main()) except idiokit.Signal: pass
class DotBot(bot.Bot): bot_name = None config = bot.Param("configuration module") show_startups = bot.BoolParam() show_attributes = bot.BoolParam() def run(self): print "digraph G {" print "node [ shape=box, style=filled, color=lightgrey ];" dot = Dot(config.load_configs(self.config)) services = dot.services() sessions = dot.sessions() def missing(x): return self.show_startups and x not in services for session in sessions: conf = dict(session.conf) path = session.path src = conf.pop("src_room", None) dst = conf.pop("dst_room", None) node = None if session.service != "roomgraph": node = "node " + session.service + " " + (src or "@") + " " + ( dst or "@") if missing(session.service): print line(node, label=session.service, shape="ellipse", fontsize=12, fontcolor="white", color="red") else: print line(node, label=session.service, shape="ellipse", fontsize=12, style="") label_list = list() if path: label_list.append(".".join(path)) if self.show_attributes: for item in conf.items(): label_list.append(" %r=%r" % item) label = "\\n".join(label_list) color = "red" if missing(session.service) else "black" if src is None: if node is not None and dst is not None: print line(node, dst, label=label, color=color, fontsize=10) else: if node is not None and dst is None: print line(src, node, label=label, color=color, fontsize=10) elif node is None and dst is not None: print line(src, dst, label=label, color=color, fontsize=10) elif node is not None and dst is not None: print line(src, node, label="", color=color, fontsize=10) print line(node, dst, label=label, color=color, fontsize=10) print '}'
class OpenCollabReader(bot.FeedBot): poll_interval = bot.IntParam(default=60) collab_url = bot.Param() collab_user = bot.Param() collab_password = bot.Param(default=None) collab_ignore_cert = bot.BoolParam() collab_extra_ca_certs = bot.Param(default=None) def __init__(self, *args, **keys): bot.FeedBot.__init__(self, *args, **keys) if self.collab_password is None: self.collab_password = getpass.getpass("Collab password: "******"utf8") + self.collab_url).hexdigest() @idiokit.stream def feed(self, query, feed_all): collab = wiki.GraphingWiki(self.collab_url, ssl_verify_cert=not self.collab_ignore_cert, ssl_ca_certs=self.collab_extra_ca_certs) yield idiokit.thread(collab.authenticate, self.collab_user, self.collab_password) yield idiokit.sleep(5) token = None current = dict() while True: try: result = yield idiokit.thread(collab.request, "IncGetMeta", query, token) except wiki.WikiFailure as fail: self.log.error("IncGetMeta failed: {0!r}".format(fail)) else: incremental, token, (removed, updates) = result removed = set(removed) if not incremental: removed.update(current) current.clear() for page, keys in updates.iteritems(): event = current.setdefault(page, events.Event()) event.add("id:open", self.page_id(page)) event.add("gwikipagename", page) event.add( "collab url", self.collab_url + urllib.quote(page.encode("utf8"))) removed.discard(page) for key, (discarded, added) in keys.iteritems(): for value in map(normalize, discarded): event.discard(key, value) for value in map(normalize, added): event.add(key, value) if not feed_all: yield idiokit.send(event) for page in removed: current.pop(page, None) event = events.Event() event.add("id:close", self.page_id(page)) event.add("gwikipagename", page) event.add("collab url", self.collab_url + page) yield idiokit.send(event) if feed_all: for page in current: yield idiokit.send(current[page]) yield idiokit.sleep(self.poll_interval)
class RSSBot(bot.PollingBot): feeds = bot.ListParam("a list of RSS feed URLs") use_cymru_whois = bot.BoolParam() http_headers = bot.ListParam("a list of http header (k,v) tuples", default=[]) def __init__(self, *args, **keys): bot.PollingBot.__init__(self, *args, **keys) def feed_keys(self, **_): for feed in self.feeds: yield (feed, ) def poll(self, url): if self.use_cymru_whois: return self._poll(url) | cymruwhois.augment() return self._poll(url) @idiokit.stream def _poll(self, url): request = urllib2.Request(url) for key, value in self.http_headers: request.add_header(key, value) try: self.log.info('Downloading feed from: "%s"', url) _, fileobj = yield utils.fetch_url(request) except utils.FetchUrlFailed as e: self.log.error('Failed to download feed "%s": %r', url, e) idiokit.stop(False) self.log.info("Finished downloading the feed.") byte = fileobj.read(1) while byte and byte != "<": byte = fileobj.read(1) if byte == "<": fileobj.seek(-1, 1) try: for _, elem in etree.iterparse(fileobj): for event in self._parse(elem, url): if event: yield idiokit.send(event) except ParseError as e: self.log.error('Invalid format on feed: "%s", "%r"', url, e) def _parse(self, elem, url): items = elem.findall("item") if not items: return for item in items: args = {"source": url} for element in list(item): if element.text and element.tag: args[element.tag] = element.text event = self.create_event(**args) yield event def create_event(self, **keys): event = events.Event() for key, value in keys.iteritems(): if value: event.add(key, value) return event
class Roomreader(bot.XMPPBot): bot_name = "roomreader" xmpp_rooms = bot.ListParam(""" comma separated list of XMPP rooms roomreader should watch. (e.g. [email protected], [email protected]) """) show_events = bot.BoolParam("print out events from channels") @idiokit.stream def main(self): try: xmpp = yield self.xmpp_connect() rooms = list() for name in self.xmpp_rooms: room = yield xmpp.muc.join(name, self.bot_name) rooms.append( room | self.xmpp_to_log(room.jid, room.participants)) yield idiokit.pipe(*rooms) except idiokit.Signal: pass @idiokit.stream def xmpp_to_log(self, own_jid, participants): in_room = set() for participant in participants: in_room.add(participant.name.resource) while True: elements = yield idiokit.next() for message in elements.with_attrs("from"): sender = JID(elements.get_attr("from")) if sender == own_jid or sender.resource is None: continue resource = sender.resource.encode("unicode-escape") bare = unicode(sender.bare()).encode("unicode-escape") type_ = message.get_attr("type", None) if type_ == "unavailable": if sender.resource in in_room: in_room.discard(sender.resource) self.log.info("* {0} left the room {1}.".format( resource, bare)) else: if sender.resource not in in_room: in_room.add(sender.resource) self.log.info("* {0} entered the room {1}.".format( resource, bare)) for body in message.children("body"): self.log.info("<{0}> {1}".format( unicode(sender).encode("unicode-escape"), body.text.encode("unicode-escape"))) if self.show_events: for event in events.Event.from_elements(message): self.log.info("<{0}> {1}".format( unicode(sender).encode("unicode-escape"), event))