class BruteForceBlockerBot(bot.PollingBot):
# Ignore the last column ("id").
COLUMNS = ["ip", "time", "count", None]
use_cymru_whois = bot.BoolParam()
def poll(self):
if self.use_cymru_whois:
return self._poll() | cymruwhois.augment("ip")
return self._poll()
@idiokit.stream
def _poll(
self,
url="http://danger.rulez.sk/projects/bruteforceblocker/blist.php"):
self.log.info("Downloading %s" % url)
try:
info, fileobj = yield utils.fetch_url(url)
except utils.FetchUrlFailed, fuf:
self.log.error("Download failed: %r", fuf)
idiokit.stop(False)
self.log.info("Downloaded")
filtered = (x for x in fileobj if x.strip() and not x.startswith("#"))
lines = (re.sub("\t+", "\t", x) for x in filtered)
yield (utils.csv_to_events(lines,
delimiter="\t",
columns=self.COLUMNS,
charset=info.get_param("charset"))
| idiokit.map(self._normalize, url))
class AutoshunBot(bot.PollingBot):
COLUMNS = ["ip", "time", "info"]
time_offset = 5
feed_url = bot.Param(default=AUTOSHUN_CSV_URL)
use_cymru_whois = bot.BoolParam()
def poll(self):
pipe = self._poll(url=self.feed_url)
if self.use_cymru_whois:
pipe = pipe | cymruwhois.augment("ip")
return pipe | self._normalize()
@idiokit.stream
def _poll(self, url):
self.log.info("Downloading %s" % url)
try:
info, fileobj = yield utils.fetch_url(url)
except utils.FetchUrlFailed, fuf:
self.log.error("Download failed: %r", fuf)
idiokit.stop()
self.log.info("Downloaded")
# Grab time offset from first line of the CSV
header = fileobj.readline()
# Source file header row may sometimes be empty
if header.startswith("Shunlist as of"):
offset = -1 * int(header[-5:]) / 100 # ex: -0500 to 5
self.time_offset = offset if -12 <= offset <= 12 else 5
yield utils.csv_to_events(fileobj,
columns=self.COLUMNS,
charset=info.get_param("charset"))
class OpenBLBot(bot.PollingBot):
feed_url = bot.Param(default=OPENBL_FEED_URL)
use_cymru_whois = bot.BoolParam()
def poll(self):
pipe = self._poll(url=self.feed_url)
if self.use_cymru_whois:
pipe = pipe | cymruwhois.augment("ip")
return pipe
@idiokit.stream
def _poll(self, url):
self.log.info("Downloading %s" % url)
try:
info, fileobj = yield utils.fetch_url(url)
except utils.FetchUrlFailed, fuf:
raise bot.PollSkipped("failed to download {0!r} ({1})".format(url, fuf))
self.log.info("Downloaded")
for line in fileobj:
event = _parse_line(line)
if event is None:
continue
event.add("feeder", "openbl.org")
event.add("feed", "openbl")
event.add("description url", self.feed_url)
event.add("type", "brute-force")
event.add(
"description",
"This host has most likely been performing brute-force " +
"attacks on one of the following services: FTP, SSH, POP3, " +
"IMAP, IMAPS or POP3S."
)
yield idiokit.send(event)
class DataplaneBot(bot.PollingBot):
url = bot.Param()
use_cymru_whois = bot.BoolParam()
# The first column values (ASN and AS name) are ignored.
COLUMNS = [None, None, "ip", "time", "category"]
def poll(self):
if self.use_cymru_whois:
return self._poll() | cymruwhois.augment("ip")
return self._poll()
@idiokit.stream
def _poll(self):
self.log.info("Downloading %s" % self.url)
try:
info, fileobj = yield utils.fetch_url(self.url)
except utils.FetchUrlFailed, fuf:
self.log.error("Download failed: %r", fuf)
return
self.log.info("Downloaded")
charset = info.get_param("charset")
filtered = (x for x in fileobj if x.strip() and not x.startswith("#"))
yield utils.csv_to_events(filtered,
delimiter="|",
columns=self.COLUMNS,
charset=charset)
class SpamhausDropBot(bot.PollingBot):
use_cymru_whois = bot.BoolParam()
http_headers = bot.ListParam("a list of http header (k, v) tuples",
default=[])
@idiokit.stream
def poll(self, url="https://www.spamhaus.org/drop/drop.lasso"):
request = urllib2.Request(url)
for key, value in self.http_headers:
request.add_header(key, value)
self.log.info("Downloading %s" % url)
try:
info, fileobj = yield utils.fetch_url(request)
except utils.FetchUrlFailed as fuf:
self.log.error("Download failed: %r", fuf)
idiokit.stop(False)
self.log.info("Downloaded")
for line in fileobj.readlines():
if line.startswith(';'):
continue
data = line.split(';')
if not data:
continue
netblock_sbl = [x.strip() for x in data]
if len(netblock_sbl) != 2:
continue
netblock, sbl = netblock_sbl
if not len(netblock.split('/')) == 2:
continue
new = events.Event()
new.add('netblock', netblock)
new.add('description url',
"http://www.spamhaus.org/sbl/query/" + sbl)
new.add('feeder', 'spamhaus')
new.add('feed', 'spamhaus drop list')
new.add('type', 'hijacked network')
if self.use_cymru_whois:
values = yield cymruwhois.lookup(netblock.split('/')[0])
for key, value in values:
new.add(key, value)
yield idiokit.send(new)
class BridgeBot(bot.Bot):
xmpp_src_jid = bot.Param("the XMPP src JID")
xmpp_src_password = bot.Param("the XMPP src password", default=None)
xmpp_src_room = bot.Param("the XMPP src room")
xmpp_src_host = bot.Param(
"the XMPP src service host (default: autodetect)", default=None)
xmpp_src_port = bot.IntParam(
"the XMPP src service port (default: autodetect)", default=None)
xmpp_src_ignore_cert = bot.BoolParam("""
do not perform any verification for
the XMPP service's SSL certificate
""")
xmpp_src_extra_ca_certs = bot.Param("""
a PEM formatted file of CAs to be
used in addition to the system CAs
""",
default=None)
xmpp_dst_jid = bot.Param("the XMPP dst JID")
xmpp_dst_password = bot.Param("the XMPP dst password", default=None)
xmpp_dst_host = bot.Param(
"the XMPP dst service host (default: autodetect)", default=None)
xmpp_dst_port = bot.IntParam(
"the XMPP dst service port (default: autodetect)", default=None)
xmpp_dst_room = bot.Param("the XMPP dst room")
xmpp_dst_ignore_cert = bot.BoolParam("""
do not perform any verification for
the XMPP service's SSL certificate
""")
xmpp_dst_extra_ca_certs = bot.Param("""
a PEM formatted file of CAs to be
used in addition to the system CAs
""",
default=None)
def __init__(self, **keys):
bot.Bot.__init__(self, **keys)
if self.xmpp_src_password is None:
self.xmpp_src_password = getpass.getpass("XMPP src password: "******"XMPP dst password: "******"Connecting to XMPP %s server with JID %r", _type, jid)
connection = yield xmpp.connect(jid,
password,
host=host,
port=port,
ssl_verify_cert=verify_cert,
ssl_ca_certs=ca_certs)
self.log.info("Connected to XMPP %s server with JID %r", _type, jid)
connection.core.presence()
self.log.info("Joining %s room %r", _type, room_name)
room = yield connection.muc.join(room_name, self.bot_name)
self.log.info("Joined %s room %r", _type, room_name)
idiokit.stop(room)
@idiokit.stream
def main(self):
dst = yield self._join("dst", self.xmpp_dst_jid,
self.xmpp_dst_password, self.xmpp_dst_host,
self.xmpp_dst_port, self.xmpp_dst_ignore_cert,
self.xmpp_dst_extra_ca_certs,
self.xmpp_dst_room)
src = yield self._join("src", self.xmpp_src_jid,
self.xmpp_src_password, self.xmpp_src_host,
self.xmpp_src_port, self.xmpp_src_ignore_cert,
self.xmpp_src_extra_ca_certs,
self.xmpp_src_room)
yield src | peel_messages() | dst | idiokit.consume()
def run(self):
try:
return idiokit.main_loop(self.main())
except idiokit.Signal:
pass
class DotBot(bot.Bot):
bot_name = None
config = bot.Param("configuration module")
show_startups = bot.BoolParam()
show_attributes = bot.BoolParam()
def run(self):
print "digraph G {"
print "node [ shape=box, style=filled, color=lightgrey ];"
dot = Dot(config.load_configs(self.config))
services = dot.services()
sessions = dot.sessions()
def missing(x):
return self.show_startups and x not in services
for session in sessions:
conf = dict(session.conf)
path = session.path
src = conf.pop("src_room", None)
dst = conf.pop("dst_room", None)
node = None
if session.service != "roomgraph":
node = "node " + session.service + " " + (src or "@") + " " + (
dst or "@")
if missing(session.service):
print line(node,
label=session.service,
shape="ellipse",
fontsize=12,
fontcolor="white",
color="red")
else:
print line(node,
label=session.service,
shape="ellipse",
fontsize=12,
style="")
label_list = list()
if path:
label_list.append(".".join(path))
if self.show_attributes:
for item in conf.items():
label_list.append(" %r=%r" % item)
label = "\\n".join(label_list)
color = "red" if missing(session.service) else "black"
if src is None:
if node is not None and dst is not None:
print line(node,
dst,
label=label,
color=color,
fontsize=10)
else:
if node is not None and dst is None:
print line(src,
node,
label=label,
color=color,
fontsize=10)
elif node is None and dst is not None:
print line(src, dst, label=label, color=color, fontsize=10)
elif node is not None and dst is not None:
print line(src, node, label="", color=color, fontsize=10)
print line(node,
dst,
label=label,
color=color,
fontsize=10)
print '}'
class OpenCollabReader(bot.FeedBot):
poll_interval = bot.IntParam(default=60)
collab_url = bot.Param()
collab_user = bot.Param()
collab_password = bot.Param(default=None)
collab_ignore_cert = bot.BoolParam()
collab_extra_ca_certs = bot.Param(default=None)
def __init__(self, *args, **keys):
bot.FeedBot.__init__(self, *args, **keys)
if self.collab_password is None:
self.collab_password = getpass.getpass("Collab password: "******"utf8") + self.collab_url).hexdigest()
@idiokit.stream
def feed(self, query, feed_all):
collab = wiki.GraphingWiki(self.collab_url,
ssl_verify_cert=not self.collab_ignore_cert,
ssl_ca_certs=self.collab_extra_ca_certs)
yield idiokit.thread(collab.authenticate, self.collab_user,
self.collab_password)
yield idiokit.sleep(5)
token = None
current = dict()
while True:
try:
result = yield idiokit.thread(collab.request, "IncGetMeta",
query, token)
except wiki.WikiFailure as fail:
self.log.error("IncGetMeta failed: {0!r}".format(fail))
else:
incremental, token, (removed, updates) = result
removed = set(removed)
if not incremental:
removed.update(current)
current.clear()
for page, keys in updates.iteritems():
event = current.setdefault(page, events.Event())
event.add("id:open", self.page_id(page))
event.add("gwikipagename", page)
event.add(
"collab url",
self.collab_url + urllib.quote(page.encode("utf8")))
removed.discard(page)
for key, (discarded, added) in keys.iteritems():
for value in map(normalize, discarded):
event.discard(key, value)
for value in map(normalize, added):
event.add(key, value)
if not feed_all:
yield idiokit.send(event)
for page in removed:
current.pop(page, None)
event = events.Event()
event.add("id:close", self.page_id(page))
event.add("gwikipagename", page)
event.add("collab url", self.collab_url + page)
yield idiokit.send(event)
if feed_all:
for page in current:
yield idiokit.send(current[page])
yield idiokit.sleep(self.poll_interval)
class RSSBot(bot.PollingBot):
feeds = bot.ListParam("a list of RSS feed URLs")
use_cymru_whois = bot.BoolParam()
http_headers = bot.ListParam("a list of http header (k,v) tuples",
default=[])
def __init__(self, *args, **keys):
bot.PollingBot.__init__(self, *args, **keys)
def feed_keys(self, **_):
for feed in self.feeds:
yield (feed, )
def poll(self, url):
if self.use_cymru_whois:
return self._poll(url) | cymruwhois.augment()
return self._poll(url)
@idiokit.stream
def _poll(self, url):
request = urllib2.Request(url)
for key, value in self.http_headers:
request.add_header(key, value)
try:
self.log.info('Downloading feed from: "%s"', url)
_, fileobj = yield utils.fetch_url(request)
except utils.FetchUrlFailed as e:
self.log.error('Failed to download feed "%s": %r', url, e)
idiokit.stop(False)
self.log.info("Finished downloading the feed.")
byte = fileobj.read(1)
while byte and byte != "<":
byte = fileobj.read(1)
if byte == "<":
fileobj.seek(-1, 1)
try:
for _, elem in etree.iterparse(fileobj):
for event in self._parse(elem, url):
if event:
yield idiokit.send(event)
except ParseError as e:
self.log.error('Invalid format on feed: "%s", "%r"', url, e)
def _parse(self, elem, url):
items = elem.findall("item")
if not items:
return
for item in items:
args = {"source": url}
for element in list(item):
if element.text and element.tag:
args[element.tag] = element.text
event = self.create_event(**args)
yield event
def create_event(self, **keys):
event = events.Event()
for key, value in keys.iteritems():
if value:
event.add(key, value)
return event
class Roomreader(bot.XMPPBot):
bot_name = "roomreader"
xmpp_rooms = bot.ListParam("""
comma separated list of XMPP rooms roomreader should watch.
(e.g. [email protected], [email protected])
""")
show_events = bot.BoolParam("print out events from channels")
@idiokit.stream
def main(self):
try:
xmpp = yield self.xmpp_connect()
rooms = list()
for name in self.xmpp_rooms:
room = yield xmpp.muc.join(name, self.bot_name)
rooms.append(
room | self.xmpp_to_log(room.jid, room.participants))
yield idiokit.pipe(*rooms)
except idiokit.Signal:
pass
@idiokit.stream
def xmpp_to_log(self, own_jid, participants):
in_room = set()
for participant in participants:
in_room.add(participant.name.resource)
while True:
elements = yield idiokit.next()
for message in elements.with_attrs("from"):
sender = JID(elements.get_attr("from"))
if sender == own_jid or sender.resource is None:
continue
resource = sender.resource.encode("unicode-escape")
bare = unicode(sender.bare()).encode("unicode-escape")
type_ = message.get_attr("type", None)
if type_ == "unavailable":
if sender.resource in in_room:
in_room.discard(sender.resource)
self.log.info("* {0} left the room {1}.".format(
resource, bare))
else:
if sender.resource not in in_room:
in_room.add(sender.resource)
self.log.info("* {0} entered the room {1}.".format(
resource, bare))
for body in message.children("body"):
self.log.info("<{0}> {1}".format(
unicode(sender).encode("unicode-escape"),
body.text.encode("unicode-escape")))
if self.show_events:
for event in events.Event.from_elements(message):
self.log.info("<{0}> {1}".format(
unicode(sender).encode("unicode-escape"),
event))