class Tix(bot.PollingBot):
poll_interval = bot.IntParam(default=300)
threshold = bot.IntParam(
default=2,
help="How many % in/out utilization must change before reporting.")
baseurl = "http://tix.estpak.ee/traffic/"
page = "tix-sw1.html"
sw1 = baseurl + page
customers = "http://tix.estpak.ee/?members"
prev_ports = dict()
@threado.stream
def poll(inner, self, something):
yield timer.sleep(3)
self.log.info("Downloading %r", self.sw1)
try:
info, fileobj = yield inner.sub(utils.fetch_url(self.sw1))
except utils.FetchUrlFailed, fuf:
self.log.error("Downloading failed: %r", fuf)
return
self.log.info("Downloaded")
for event in self.parse_sw1(fileobj, self.baseurl):
send = self.enough_delta(event, self.threshold)
if send:
inner.send(event)
class TailBot(bot.FeedBot):
path = bot.Param("path to the followed file")
offset = bot.IntParam("file offset", default=None)
@idiokit.stream
def feed(self):
for result in tail_file(self.path, self.offset):
if result is None:
yield idiokit.sleep(2.0)
continue
mtime, line = result
keys = self.parse(line, mtime)
if keys is None:
continue
event = events.Event()
for key, value in keys.items():
event.add(key, value)
yield idiokit.send(event)
def parse(self, line, mtime):
line = line.rstrip()
if not line:
return
line = utils.force_decode(line)
return {"line": line}
class TallinnAirport(bot.PollingBot):
poll_interval = bot.IntParam(default=300)
@threado.stream
def poll(inner, self, something):
yield timer.sleep(2)
depurl = "http://xtra.tllapt.ee/taru/dep_en.html"
arrurl = "http://xtra.tllapt.ee/taru/arr_en.html"
for (subtype, url) in [('departures', depurl), ('arrivals', arrurl)]:
self.log.info("Downloading %r", url)
try:
info, fileobj = yield inner.sub(utils.fetch_url(url))
except utils.FetchUrlFailed, fuf:
self.log.error("Downloading failed: %r", fuf)
return
self.log.info("Downloaded")
utilization = get_metrics(fileobj)
event = events.Event()
event.add('source', url)
event.add('utilization', utilization)
event.add('service', 'airports')
event.add('subtype', subtype)
event.add('id', create_id(event, 'tallinnairpoirt', 'subtype'))
event.add('longitude', '24.799303')
event.add('latitude', '59.4165212')
yield inner.send(event)
class MeteoAlarmBot(bot.PollingBot):
poll_interval = bot.IntParam(default=60)
url = bot.Param(default="http://www.meteoalarm.eu/documents/rss/ee.rss")
cc = bot.Param(default="EE")
# You can also have file: url for testing,e.g.
# url = "file:///<path>/vsroom/examples/public-sources/custom/ee.rss"
old_events = dict()
@threado.stream
def poll(inner, self,something):
yield timer.sleep(5)
self.log.info("Downloading %r", self.url)
try:
info, fileobj = yield inner.sub(utils.fetch_url(self.url))
except utils.FetchUrlFailed, fuf:
self.log.error("Downloading failed: %r", fuf)
return
self.log.info("Downloaded")
tree = ElementTree()
try:
tree.parse(fileobj)
except ExpatError,e:
self.log.error("Parsing source failed, %r", e)
return
class EmhiEE(bot.PollingBot):
poll_interval = bot.IntParam(default=60)
url = "http://www.emhi.ee/ilma_andmed/xml/forecast.php?lang=eng"
@threado.stream
def poll(inner, self,something):
yield timer.sleep(1)
self.log.info("Downloading %r", self.url)
try:
info, fileobj = yield inner.sub(utils.fetch_url(self.url))
except utils.FetchUrlFailed, fuf:
self.log.error("Downloading failed: %r", fuf)
return
self.log.info("Downloaded")
tree = ElementTree()
tree.parse(fileobj)
tree = tree.findall('forecast')
for event in parse(tree):
inner.send(event)
class EmhiEE(bot.PollingBot):
poll_interval = bot.IntParam(default=60)
url = "http://www.emhi.ee/ilma_andmed/xml/forecast.php?lang=eng"
old_events = dict()
@threado.stream
def poll(inner, self, something):
yield timer.sleep(5)
self.log.info("Downloading %r", self.url)
try:
info, fileobj = yield inner.sub(utils.fetch_url(self.url))
except utils.FetchUrlFailed, fuf:
self.log.error("Downloading failed: %r", fuf)
return
self.log.info("Downloaded")
tree = ElementTree()
new_events = dict()
try:
tree.parse(fileobj)
except (ParseError), e:
self.log.info('Bad data from the source: %s', e)
return
class Receiver(bot.XMPPBot):
room = bot.Param("""
the room for receiving events from
""")
rate_limit = bot.IntParam("""
rate limit for the sent stream
""", default=None)
@idiokit.stream
def main(self):
xmpp = yield self.xmpp_connect()
room = yield xmpp.muc.join(self.room)
yield idiokit.pipe(
self._read_stdin(),
events.events_to_elements(),
_rate_limiter(self.rate_limit),
room,
idiokit.consume()
)
@idiokit.stream
def _read_stdin(self):
loads = json.JSONDecoder(parse_float=unicode, parse_int=unicode).decode
while True:
yield select.select([sys.stdin], [], [])
line = sys.stdin.readline()
if not line:
break
if not line.strip():
continue
in_dict = loads(line)
yield idiokit.send(events.Event(in_dict))
class BridgeBot(bot.Bot):
xmpp_src_jid = bot.Param("the XMPP src JID")
xmpp_src_password = bot.Param("the XMPP src password", default=None)
xmpp_src_room = bot.Param("the XMPP src room")
xmpp_src_host = bot.Param(
"the XMPP src service host (default: autodetect)", default=None)
xmpp_src_port = bot.IntParam(
"the XMPP src service port (default: autodetect)", default=None)
xmpp_src_ignore_cert = bot.BoolParam("""
do not perform any verification for
the XMPP service's SSL certificate
""")
xmpp_src_extra_ca_certs = bot.Param("""
a PEM formatted file of CAs to be
used in addition to the system CAs
""",
default=None)
xmpp_dst_jid = bot.Param("the XMPP dst JID")
xmpp_dst_password = bot.Param("the XMPP dst password", default=None)
xmpp_dst_host = bot.Param(
"the XMPP dst service host (default: autodetect)", default=None)
xmpp_dst_port = bot.IntParam(
"the XMPP dst service port (default: autodetect)", default=None)
xmpp_dst_room = bot.Param("the XMPP dst room")
xmpp_dst_ignore_cert = bot.BoolParam("""
do not perform any verification for
the XMPP service's SSL certificate
""")
xmpp_dst_extra_ca_certs = bot.Param("""
a PEM formatted file of CAs to be
used in addition to the system CAs
""",
default=None)
def __init__(self, **keys):
bot.Bot.__init__(self, **keys)
if self.xmpp_src_password is None:
self.xmpp_src_password = getpass.getpass("XMPP src password: "******"XMPP dst password: "******"Connecting to XMPP %s server with JID %r", _type, jid)
connection = yield xmpp.connect(jid,
password,
host=host,
port=port,
ssl_verify_cert=verify_cert,
ssl_ca_certs=ca_certs)
self.log.info("Connected to XMPP %s server with JID %r", _type, jid)
connection.core.presence()
self.log.info("Joining %s room %r", _type, room_name)
room = yield connection.muc.join(room_name, self.bot_name)
self.log.info("Joined %s room %r", _type, room_name)
idiokit.stop(room)
@idiokit.stream
def main(self):
dst = yield self._join("dst", self.xmpp_dst_jid,
self.xmpp_dst_password, self.xmpp_dst_host,
self.xmpp_dst_port, self.xmpp_dst_ignore_cert,
self.xmpp_dst_extra_ca_certs,
self.xmpp_dst_room)
src = yield self._join("src", self.xmpp_src_jid,
self.xmpp_src_password, self.xmpp_src_host,
self.xmpp_src_port, self.xmpp_src_ignore_cert,
self.xmpp_src_extra_ca_certs,
self.xmpp_src_room)
yield src | peel_messages() | dst | idiokit.consume()
def run(self):
try:
return idiokit.main_loop(self.main())
except idiokit.Signal:
pass
class OpenCollabReader(bot.FeedBot):
poll_interval = bot.IntParam(default=60)
collab_url = bot.Param()
collab_user = bot.Param()
collab_password = bot.Param(default=None)
collab_ignore_cert = bot.BoolParam()
collab_extra_ca_certs = bot.Param(default=None)
def __init__(self, *args, **keys):
bot.FeedBot.__init__(self, *args, **keys)
if self.collab_password is None:
self.collab_password = getpass.getpass("Collab password: "******"utf8") + self.collab_url).hexdigest()
@idiokit.stream
def feed(self, query, feed_all):
collab = wiki.GraphingWiki(self.collab_url,
ssl_verify_cert=not self.collab_ignore_cert,
ssl_ca_certs=self.collab_extra_ca_certs)
yield idiokit.thread(collab.authenticate, self.collab_user,
self.collab_password)
yield idiokit.sleep(5)
token = None
current = dict()
while True:
try:
result = yield idiokit.thread(collab.request, "IncGetMeta",
query, token)
except wiki.WikiFailure as fail:
self.log.error("IncGetMeta failed: {0!r}".format(fail))
else:
incremental, token, (removed, updates) = result
removed = set(removed)
if not incremental:
removed.update(current)
current.clear()
for page, keys in updates.iteritems():
event = current.setdefault(page, events.Event())
event.add("id:open", self.page_id(page))
event.add("gwikipagename", page)
event.add(
"collab url",
self.collab_url + urllib.quote(page.encode("utf8")))
removed.discard(page)
for key, (discarded, added) in keys.iteritems():
for value in map(normalize, discarded):
event.discard(key, value)
for value in map(normalize, added):
event.add(key, value)
if not feed_all:
yield idiokit.send(event)
for page in removed:
current.pop(page, None)
event = events.Event()
event.add("id:close", self.page_id(page))
event.add("gwikipagename", page)
event.add("collab url", self.collab_url + page)
yield idiokit.send(event)
if feed_all:
for page in current:
yield idiokit.send(current[page])
yield idiokit.sleep(self.poll_interval)
class MailDirBot(bot.FeedBot):
handler = handlers.HandlerParam()
input_dir = bot.Param()
work_dir = bot.Param()
concurrency = bot.IntParam(default=1)
poll_interval = bot.FloatParam(default=1)
def __init__(self, *args, **keys):
bot.FeedBot.__init__(self, *args, **keys)
self.handler = handlers.load_handler(self.handler)
self._queue = utils.WaitQueue()
def feed_keys(self, *args, **keys):
for nth_concurrent in range(1, self.concurrency + 1):
yield (nth_concurrent, )
def run(self):
makedirs(self.work_dir)
with lockfile(os.path.join(self.work_dir, ".lock")) as success:
if not success:
self.log.error(
u"Someone else is using the directory {0}".format(
self.work_dir))
else:
bot.FeedBot.run(self)
@idiokit.stream
def _poll_files(self):
in_progress = os.path.join(self.work_dir, "in-progress")
done = os.path.join(self.work_dir, "done")
makedirs(in_progress)
makedirs(done)
for dirname, filename in iter_dir(in_progress):
input_name = os.path.join(dirname, filename)
output_name = os.path.join(done, filename)
yield idiokit.send(input_name, output_name)
while True:
paths = itertools.chain(
iter_dir(os.path.join(self.input_dir, "new")),
iter_dir(os.path.join(self.input_dir, "cur")))
for dirname, filename in paths:
uuid_name = uuid.uuid4().hex + "." + filename
input_name = os.path.join(in_progress, uuid_name)
output_name = os.path.join(done, uuid_name)
if try_rename(os.path.join(dirname, filename), input_name):
yield idiokit.send(input_name, output_name)
yield idiokit.sleep(self.poll_interval)
@idiokit.stream
def _forward_files(self):
while True:
input_name, output_name = yield idiokit.next()
ack = idiokit.Event()
node = yield self._queue.queue(0, (input_name, output_name, ack))
try:
yield ack
finally:
yield self._queue.cancel(node)
@idiokit.stream
def main(self, state):
yield self._poll_files() | self._forward_files()
@idiokit.stream
def feed(self, nth_concurrent):
while True:
input_name, output_name, ack = yield self._queue.wait()
ack.succeed()
msg = try_read_message(input_name)
if msg is None:
continue
subject = escape_whitespace(
msg.get_unicode("Subject", "<no subject>", errors="replace"))
sender = escape_whitespace(
msg.get_unicode("From", "<unknown sender>", errors="replace"))
self.log.info(u"Handler #{0} handling mail '{1}' from {2}".format(
nth_concurrent, subject, sender))
handler = self.handler(log=self.log)
yield handler.handle(msg)
os.rename(input_name, output_name)
self.log.info(u"Handler #{0} done with mail '{1}' from {2}".format(
nth_concurrent, subject, sender))