class MeteoAlarmBot(bot.PollingBot):
poll_interval = bot.IntParam(default=60)
url = bot.Param(default="http://www.meteoalarm.eu/documents/rss/ee.rss")
cc = bot.Param(default="EE")
# You can also have file: url for testing,e.g.
# url = "file:///<path>/vsroom/examples/public-sources/custom/ee.rss"
old_events = dict()
@threado.stream
def poll(inner, self,something):
yield timer.sleep(5)
self.log.info("Downloading %r", self.url)
try:
info, fileobj = yield inner.sub(utils.fetch_url(self.url))
except utils.FetchUrlFailed, fuf:
self.log.error("Downloading failed: %r", fuf)
return
self.log.info("Downloaded")
tree = ElementTree()
try:
tree.parse(fileobj)
except ExpatError,e:
self.log.error("Parsing source failed, %r", e)
return
class TwitterBot(bot.PollingBot):
consumer_key = bot.Param()
consumer_secret = bot.Param()
access_token_key = bot.Param()
access_token_secret = bot.Param()
old_tweets = dict()
def feed_keys(self, user, **keys):
return [user]
@threado.stream
def poll(inner, self, user):
yield timer.sleep(1)
new_tweets = dict()
api = twitter.Api(consumer_key=self.consumer_key,
consumer_secret=self.consumer_secret,
access_token_key=self.access_token_key,
access_token_secret=self.access_token_secret)
try:
userobj = api.GetUser(user)
friends = api.GetFriends()
except (URLError, twitter.TwitterError, ValueError, JSONDecodeError,
socket.error), e:
self.log.error("Twitter error: %s." % (e))
return
self.log.info('Fetching status for %s' % userobj.name)
for friend in [userobj] + friends:
status = friend.GetStatus()
name = friend.screen_name
if status == None: continue
start = status.GetCreatedAtInSeconds()
new_tweets[name] = friend
#dedup
for name, friend in new_tweets.iteritems():
# 1. is this the same event that was sent previously, and
# 2. is this older event than the recent one (twitter api
# sometimes returns non-latest tweet)
old_friend = self.old_tweets.get(name)
if old_friend == None:
event = self.get_event(friend)
inner.send(event)
continue
old_tweet = self.old_tweets.get(name).GetStatus()
new_tweet = friend.GetStatus()
prev_created = old_tweet.GetCreatedAtInSeconds()
new_created = new_tweet.GetCreatedAtInSeconds()
if prev_created < new_created:
event = self.get_event(friend)
inner.send(event)
self.old_tweets = new_tweets
class AutoshunBot(bot.PollingBot):
COLUMNS = ["ip", "time", "info"]
time_offset = 5
feed_url = bot.Param(default=AUTOSHUN_CSV_URL)
use_cymru_whois = bot.BoolParam()
def poll(self):
pipe = self._poll(url=self.feed_url)
if self.use_cymru_whois:
pipe = pipe | cymruwhois.augment("ip")
return pipe | self._normalize()
@idiokit.stream
def _poll(self, url):
self.log.info("Downloading %s" % url)
try:
info, fileobj = yield utils.fetch_url(url)
except utils.FetchUrlFailed, fuf:
self.log.error("Download failed: %r", fuf)
idiokit.stop()
self.log.info("Downloaded")
# Grab time offset from first line of the CSV
header = fileobj.readline()
# Source file header row may sometimes be empty
if header.startswith("Shunlist as of"):
offset = -1 * int(header[-5:]) / 100 # ex: -0500 to 5
self.time_offset = offset if -12 <= offset <= 12 else 5
yield utils.csv_to_events(fileobj,
columns=self.COLUMNS,
charset=info.get_param("charset"))
class DataplaneBot(bot.PollingBot):
url = bot.Param()
use_cymru_whois = bot.BoolParam()
# The first column values (ASN and AS name) are ignored.
COLUMNS = [None, None, "ip", "time", "category"]
def poll(self):
if self.use_cymru_whois:
return self._poll() | cymruwhois.augment("ip")
return self._poll()
@idiokit.stream
def _poll(self):
self.log.info("Downloading %s" % self.url)
try:
info, fileobj = yield utils.fetch_url(self.url)
except utils.FetchUrlFailed, fuf:
self.log.error("Download failed: %r", fuf)
return
self.log.info("Downloaded")
charset = info.get_param("charset")
filtered = (x for x in fileobj if x.strip() and not x.startswith("#"))
yield utils.csv_to_events(filtered,
delimiter="|",
columns=self.COLUMNS,
charset=charset)
class Receiver(bot.XMPPBot):
room = bot.Param("""
the room for receiving events from
""")
@idiokit.stream
def main(self):
xmpp = yield self.xmpp_connect()
room = yield xmpp.muc.join(self.room)
yield idiokit.pipe(
room,
events.stanzas_to_events(),
self._recv()
)
@idiokit.stream
def _recv(self):
dumps = json.JSONEncoder(check_circular=False).encode
while True:
event = yield idiokit.next()
out_dict = {}
for key, value in event.items():
out_dict.setdefault(key, []).append(value)
print dumps(out_dict)
class VxVaultBot(bot.PollingBot):
feed_url = bot.Param(default=FEED_URL)
@idiokit.stream
def poll(self):
self.log.info("Downloading {0}".format(self.feed_url))
try:
info, fileobj = yield utils.fetch_url(self.feed_url)
except utils.FetchUrlFailed as fuf:
raise bot.PollSkipped("failed to download {0} ({1})".format(
self.feed_url, fuf))
self.log.info("Downloaded")
for line in fileobj:
url, netloc = parseURL(line)
if url is None:
continue
event = events.Event()
event.add("url", url)
if i_am_a_name(netloc):
event.add("domain name", netloc)
else:
event.add("ip", netloc)
event.add("feeder", "siri urz")
event.add("feed", "vxvault")
event.add("feed url", self.feed_url)
event.add("type", "malware url")
event.add("description",
"This host is most likely hosting a malware URL.")
yield idiokit.send(event)
class OpenBLBot(bot.PollingBot):
feed_url = bot.Param(default=OPENBL_FEED_URL)
use_cymru_whois = bot.BoolParam()
def poll(self):
pipe = self._poll(url=self.feed_url)
if self.use_cymru_whois:
pipe = pipe | cymruwhois.augment("ip")
return pipe
@idiokit.stream
def _poll(self, url):
self.log.info("Downloading %s" % url)
try:
info, fileobj = yield utils.fetch_url(url)
except utils.FetchUrlFailed, fuf:
raise bot.PollSkipped("failed to download {0!r} ({1})".format(url, fuf))
self.log.info("Downloaded")
for line in fileobj:
event = _parse_line(line)
if event is None:
continue
event.add("feeder", "openbl.org")
event.add("feed", "openbl")
event.add("description url", self.feed_url)
event.add("type", "brute-force")
event.add(
"description",
"This host has most likely been performing brute-force " +
"attacks on one of the following services: FTP, SSH, POP3, " +
"IMAP, IMAPS or POP3S."
)
yield idiokit.send(event)
class MDLBot(bot.PollingBot):
url = bot.Param(default="https://www.malwaredomainlist.com/updatescsv.php")
_columns = ["timestamp", "url", "ip", "reverse", "description", "registrant", "asn"]
@idiokit.stream
def poll(self):
self.log.info("Downloading updates from {0!r}".format(self.url))
try:
info, fileobj = yield utils.fetch_url(self.url)
except utils.FetchUrlFailed as fuf:
raise bot.PollSkipped("Downloading {0!r} failed ({1})".format(self.url, fuf))
self.log.info("Updates downloaded from {0!r}".format(self.url))
yield idiokit.pipe(
utils.csv_to_events(fileobj, columns=self._columns),
idiokit.map(self._normalize))
def _normalize(self, event):
yield events.Event({
"feed": "mdl",
"feed url": self.url,
"type": "malware",
"source time": event.values("timestamp", parse_timestamp),
"url": event.values("url", parse_url),
"domain name": event.values("url", parse_host),
"description url": event.values("url", parse_description_url),
"ip": event.values("ip", parse_ip),
"asn": event.values("asn", parse_valid),
"registrant": event.values("registrant", parse_valid),
"description": event.values("description", parse_valid)
})
class TailBot(bot.FeedBot):
path = bot.Param("path to the followed file")
offset = bot.IntParam("file offset", default=None)
@idiokit.stream
def feed(self):
for result in tail_file(self.path, self.offset):
if result is None:
yield idiokit.sleep(2.0)
continue
mtime, line = result
keys = self.parse(line, mtime)
if keys is None:
continue
event = events.Event()
for key, value in keys.items():
event.add(key, value)
yield idiokit.send(event)
def parse(self, line, mtime):
line = line.rstrip()
if not line:
return
line = utils.force_decode(line)
return {"line": line}
class StressBot(bot.FeedBot):
data = bot.Param("event data")
@idiokit.stream
def feed(self):
event = events.Event.from_unicode(self.data.decode("utf-8"))
while True:
yield idiokit.send(event)
class RollOverArchiveBot(archivebot.ArchiveBot):
rollover = bot.Param("period for doing archive file rollover " +
"(day, week, month or year)")
def archive_path(self, timestamp, room_name, event):
path = archivebot.ArchiveBot.archive_path(self, timestamp, room_name,
event)
path += "." + time.strftime(
"%Y%m%d", current_timestamp(timestamp, self.rollover))
return path
class FakeBig5Bot(bot.XMPPBot):
room_dst = bot.Param("the destination room")
@threado.stream
def main(inner, self):
conn = yield self.xmpp_connect()
dst = yield conn.muc.join(self.room_dst, self.bot_name)
self.log.info("Joined room %r", self.room_dst), dst
yield generate() | events_to_elements_with_delay_element(
) | dst | threado.dev_null()
class HistorianService(bot.ServiceBot):
bot_state_file = bot.Param()
def __init__(self, bot_state_file=None, **keys):
bot.ServiceBot.__init__(self, bot_state_file=None, **keys)
self.rooms = taskfarm.TaskFarm(self.handle_room)
self.db_dir = bot_state_file
try:
os.makedirs(self.db_dir)
except OSError, ose:
if errno.EEXIST != ose.errno:
raise ose
class AccessLogBot(TailBot):
path = bot.Param("access_log file path")
def parse(self, line, _):
line = line.strip()
if not line:
return
facts = dict(parse_log_line(line))
if "timestamp" in facts:
facts["timestamp"] = convert_date(facts["timestamp"])
if "request" in facts:
facts.update(parse_request(facts["request"]))
if "user_agent" in facts:
facts.update(parse_user_agent(facts["user_agent"]))
return events.Event(facts)
class Receiver(bot.XMPPBot):
room = bot.Param("""
the room for receiving events from
""")
rate_limit = bot.IntParam("""
rate limit for the sent stream
""", default=None)
@idiokit.stream
def main(self):
xmpp = yield self.xmpp_connect()
room = yield xmpp.muc.join(self.room)
yield idiokit.pipe(
self._read_stdin(),
events.events_to_elements(),
_rate_limiter(self.rate_limit),
room,
idiokit.consume()
)
@idiokit.stream
def _read_stdin(self):
loads = json.JSONDecoder(parse_float=unicode, parse_int=unicode).decode
while True:
yield select.select([sys.stdin], [], [])
line = sys.stdin.readline()
if not line:
break
if not line.strip():
continue
in_dict = loads(line)
yield idiokit.send(events.Event(in_dict))
class RansomwareTrackerBot(bot.PollingBot):
feed_url = bot.Param(
default="https://ransomwaretracker.abuse.ch/feeds/csv/")
@idiokit.stream
def poll(self):
self.log.info("Downloading {0}".format(self.feed_url))
try:
info, fileobj = yield utils.fetch_url(self.feed_url)
except utils.FetchUrlFailed as fuf:
raise bot.PollSkipped("Download failed: {0}".format(fuf))
lines = []
for line in fileobj:
line = line.strip()
if line and not line.startswith("#"):
lines.append(line)
yield idiokit.pipe(
utils.csv_to_events(tuple(lines),
columns=COLUMNS,
charset=info.get_param("charset", None)),
_parse())
class PhishTankBot(bot.PollingBot):
application_key = bot.Param("registered application key for PhishTank")
feed_url = bot.Param(default="http://data.phishtank.com/data/%s/online-valid.xml.bz2")
def __init__(self, *args, **keys):
bot.PollingBot.__init__(self, *args, **keys)
self._etag = None
@idiokit.stream
def _handle_entry(self, entry, sites):
details = entry.find("details")
if details is None:
return
verification = entry.find("verification")
if verification is None or parse_text(verification, "verified") != "yes":
return
status = entry.find("status")
if status is None or parse_text(status, "online") != "yes":
return
url = parse_text(entry, "url")
if not url:
return
event = events.Event({"feed": "phishtank", "url": url})
domain = urlparse.urlparse(url).netloc
if is_domain(domain):
event.add("domain name", domain)
detail_url = parse_text(entry, "phish_detail_url")
if detail_url:
event.add("description url", detail_url)
target = parse_text(entry, "target")
if target:
event.add("target", target)
history = {}
for detail in details.findall("detail"):
ip = parse_text(detail, "ip_address")
if not ip:
continue
announcer = parse_text(detail, "announcing_network")
if not announcer:
continue
detail_time = parse_text(detail, "detail_time")
try:
ts = datetime.strptime(detail_time, "%Y-%m-%dT%H:%M:%S+00:00")
except (ValueError, TypeError):
continue
history[ts] = (ip, announcer)
if history:
latest = sorted(history.keys())[-1]
ip, announcer = history[latest]
url_data = sites.setdefault(url, set())
if (ip, announcer) in url_data:
return
url_data.add((ip, announcer))
event.add("ip", ip)
event.add("asn", announcer)
event.add("source time", latest.strftime("%Y-%m-%d %H:%M:%SZ"))
yield idiokit.send(event)
@idiokit.stream
def poll(self):
url = self.feed_url % self.application_key
try:
self.log.info("Checking if {0!r} has new data".format(url))
info, _ = yield utils.fetch_url(HeadRequest(url))
etag = info.get("etag", None)
if etag is not None and self._etag == etag:
raise bot.PollSkipped("no new data detected (ETag stayed the same)")
self.log.info("Downloading data from {0!r}".format(url))
_, fileobj = yield utils.fetch_url(url)
except utils.FetchUrlFailed as error:
raise bot.PollSkipped("failed to download {0!r} ({1})".format(url, error))
self.log.info("Downloaded data from {0!r}".format(url))
reader = BZ2Reader(fileobj)
try:
depth = 0
sites = dict()
for event, element in etree.iterparse(reader, events=("start", "end")):
if event == "start" and element.tag == "entry":
depth += 1
if event == "end" and element.tag == "entry":
yield self._handle_entry(element, sites)
depth -= 1
if event == "end" and depth == 0:
element.clear()
except SyntaxError as error:
raise bot.PollSkipped("syntax error in report {0!r} ({1})".format(url, error))
else:
self._etag = etag
def main(self, state):
if state is None:
state = None, None
self._etag, wrapped_state = state
return bot.PollingBot.main(self, wrapped_state) | self._add_etag_to_result()
@idiokit.stream
def _add_etag_to_result(self):
state = yield idiokit.consume()
idiokit.stop(self._etag, state)
class DotBot(bot.Bot):
bot_name = None
config = bot.Param("configuration module")
show_startups = bot.BoolParam()
show_attributes = bot.BoolParam()
def run(self):
print "digraph G {"
print "node [ shape=box, style=filled, color=lightgrey ];"
dot = Dot(config.load_configs(self.config))
services = dot.services()
sessions = dot.sessions()
def missing(x):
return self.show_startups and x not in services
for session in sessions:
conf = dict(session.conf)
path = session.path
src = conf.pop("src_room", None)
dst = conf.pop("dst_room", None)
node = None
if session.service != "roomgraph":
node = "node " + session.service + " " + (src or "@") + " " + (
dst or "@")
if missing(session.service):
print line(node,
label=session.service,
shape="ellipse",
fontsize=12,
fontcolor="white",
color="red")
else:
print line(node,
label=session.service,
shape="ellipse",
fontsize=12,
style="")
label_list = list()
if path:
label_list.append(".".join(path))
if self.show_attributes:
for item in conf.items():
label_list.append(" %r=%r" % item)
label = "\\n".join(label_list)
color = "red" if missing(session.service) else "black"
if src is None:
if node is not None and dst is not None:
print line(node,
dst,
label=label,
color=color,
fontsize=10)
else:
if node is not None and dst is None:
print line(src,
node,
label=label,
color=color,
fontsize=10)
elif node is None and dst is not None:
print line(src, dst, label=label, color=color, fontsize=10)
elif node is not None and dst is not None:
print line(src, node, label="", color=color, fontsize=10)
print line(node,
dst,
label=label,
color=color,
fontsize=10)
print '}'
class ArchiveBot(bot.ServiceBot):
archive_dir = bot.Param("directory where archive files are written")
def __init__(self, *args, **keys):
super(ArchiveBot, self).__init__(*args, **keys)
self.rooms = taskfarm.TaskFarm(self._handle_room, grace_period=0.0)
self.archive_dir = _ensure_dir(self.archive_dir)
@idiokit.stream
def session(self, state, src_room):
src_jid = yield self.xmpp.muc.get_full_room_jid(src_room)
yield self.rooms.inc(src_jid.bare())
@idiokit.stream
def _handle_room(self, name):
msg = "room {0!r}".format(name)
attrs = events.Event({
"type": "room",
"service": self.bot_name,
"room": unicode(name)
})
with self.log.stateful(repr(self.xmpp.jid), "room", repr(name)) as log:
log.open("Joining " + msg, attrs, status="joining")
room = yield self.xmpp.muc.join(name, self.bot_name)
log.open("Joined " + msg, attrs, status="joined")
try:
yield idiokit.pipe(
room,
events.stanzas_to_events(),
self._archive(room.jid.bare())
)
finally:
log.close("Left " + msg, attrs, status="left")
def _archive(self, room_bare_jid):
compress = utils.WaitQueue()
room_name = _encode_room_jid(room_bare_jid)
_dir = os.path.join(self.archive_dir, room_name)
if _dir != os.path.normpath(_dir):
raise ValueError("incorrect room name lands outside the archive directory")
for root, _, filenames in os.walk(_dir):
for filename in filenames:
path = os.path.join(root, filename)
if _is_compress_path(path):
compress.queue(0.0, path)
return idiokit.pipe(
self._collect(room_name, compress),
self._compress(compress)
)
@idiokit.stream
def _collect(self, room_name, compress):
event = yield idiokit.next()
while True:
current = datetime.utcnow().day
with _open_archive(self.archive_dir, time.time(), room_name) as archive:
self.log.info("Opened archive {0!r}".format(archive.name))
while current == datetime.utcnow().day:
json_dict = dict((key, event.values(key)) for key in event.keys())
archive.write(json.dumps(json_dict) + os.linesep)
event = yield idiokit.next()
yield compress.queue(0.0, _rename(archive.name))
@idiokit.stream
def _compress(self, queue):
while True:
compress_path = yield queue.wait()
try:
path = yield idiokit.thread(_compress, compress_path)
self.log.info("Compressed archive {0!r}".format(path))
except ValueError:
self.log.error("Invalid path {0!r}".format(compress_path))
class OpenCollabReader(bot.FeedBot):
poll_interval = bot.IntParam(default=60)
collab_url = bot.Param()
collab_user = bot.Param()
collab_password = bot.Param(default=None)
collab_ignore_cert = bot.BoolParam()
collab_extra_ca_certs = bot.Param(default=None)
def __init__(self, *args, **keys):
bot.FeedBot.__init__(self, *args, **keys)
if self.collab_password is None:
self.collab_password = getpass.getpass("Collab password: "******"utf8") + self.collab_url).hexdigest()
@idiokit.stream
def feed(self, query, feed_all):
collab = wiki.GraphingWiki(self.collab_url,
ssl_verify_cert=not self.collab_ignore_cert,
ssl_ca_certs=self.collab_extra_ca_certs)
yield idiokit.thread(collab.authenticate, self.collab_user,
self.collab_password)
yield idiokit.sleep(5)
token = None
current = dict()
while True:
try:
result = yield idiokit.thread(collab.request, "IncGetMeta",
query, token)
except wiki.WikiFailure as fail:
self.log.error("IncGetMeta failed: {0!r}".format(fail))
else:
incremental, token, (removed, updates) = result
removed = set(removed)
if not incremental:
removed.update(current)
current.clear()
for page, keys in updates.iteritems():
event = current.setdefault(page, events.Event())
event.add("id:open", self.page_id(page))
event.add("gwikipagename", page)
event.add(
"collab url",
self.collab_url + urllib.quote(page.encode("utf8")))
removed.discard(page)
for key, (discarded, added) in keys.iteritems():
for value in map(normalize, discarded):
event.discard(key, value)
for value in map(normalize, added):
event.add(key, value)
if not feed_all:
yield idiokit.send(event)
for page in removed:
current.pop(page, None)
event = events.Event()
event.add("id:close", self.page_id(page))
event.add("gwikipagename", page)
event.add("collab url", self.collab_url + page)
yield idiokit.send(event)
if feed_all:
for page in current:
yield idiokit.send(current[page])
yield idiokit.sleep(self.poll_interval)
class BridgeBot(bot.Bot):
xmpp_src_jid = bot.Param("the XMPP src JID")
xmpp_src_password = bot.Param("the XMPP src password", default=None)
xmpp_src_room = bot.Param("the XMPP src room")
xmpp_src_host = bot.Param(
"the XMPP src service host (default: autodetect)", default=None)
xmpp_src_port = bot.IntParam(
"the XMPP src service port (default: autodetect)", default=None)
xmpp_src_ignore_cert = bot.BoolParam("""
do not perform any verification for
the XMPP service's SSL certificate
""")
xmpp_src_extra_ca_certs = bot.Param("""
a PEM formatted file of CAs to be
used in addition to the system CAs
""",
default=None)
xmpp_dst_jid = bot.Param("the XMPP dst JID")
xmpp_dst_password = bot.Param("the XMPP dst password", default=None)
xmpp_dst_host = bot.Param(
"the XMPP dst service host (default: autodetect)", default=None)
xmpp_dst_port = bot.IntParam(
"the XMPP dst service port (default: autodetect)", default=None)
xmpp_dst_room = bot.Param("the XMPP dst room")
xmpp_dst_ignore_cert = bot.BoolParam("""
do not perform any verification for
the XMPP service's SSL certificate
""")
xmpp_dst_extra_ca_certs = bot.Param("""
a PEM formatted file of CAs to be
used in addition to the system CAs
""",
default=None)
def __init__(self, **keys):
bot.Bot.__init__(self, **keys)
if self.xmpp_src_password is None:
self.xmpp_src_password = getpass.getpass("XMPP src password: "******"XMPP dst password: "******"Connecting to XMPP %s server with JID %r", _type, jid)
connection = yield xmpp.connect(jid,
password,
host=host,
port=port,
ssl_verify_cert=verify_cert,
ssl_ca_certs=ca_certs)
self.log.info("Connected to XMPP %s server with JID %r", _type, jid)
connection.core.presence()
self.log.info("Joining %s room %r", _type, room_name)
room = yield connection.muc.join(room_name, self.bot_name)
self.log.info("Joined %s room %r", _type, room_name)
idiokit.stop(room)
@idiokit.stream
def main(self):
dst = yield self._join("dst", self.xmpp_dst_jid,
self.xmpp_dst_password, self.xmpp_dst_host,
self.xmpp_dst_port, self.xmpp_dst_ignore_cert,
self.xmpp_dst_extra_ca_certs,
self.xmpp_dst_room)
src = yield self._join("src", self.xmpp_src_jid,
self.xmpp_src_password, self.xmpp_src_host,
self.xmpp_src_port, self.xmpp_src_ignore_cert,
self.xmpp_src_extra_ca_certs,
self.xmpp_src_room)
yield src | peel_messages() | dst | idiokit.consume()
def run(self):
try:
return idiokit.main_loop(self.main())
except idiokit.Signal:
pass
class MailDirBot(bot.FeedBot):
handler = handlers.HandlerParam()
input_dir = bot.Param()
work_dir = bot.Param()
concurrency = bot.IntParam(default=1)
poll_interval = bot.FloatParam(default=1)
def __init__(self, *args, **keys):
bot.FeedBot.__init__(self, *args, **keys)
self.handler = handlers.load_handler(self.handler)
self._queue = utils.WaitQueue()
def feed_keys(self, *args, **keys):
for nth_concurrent in range(1, self.concurrency + 1):
yield (nth_concurrent, )
def run(self):
makedirs(self.work_dir)
with lockfile(os.path.join(self.work_dir, ".lock")) as success:
if not success:
self.log.error(
u"Someone else is using the directory {0}".format(
self.work_dir))
else:
bot.FeedBot.run(self)
@idiokit.stream
def _poll_files(self):
in_progress = os.path.join(self.work_dir, "in-progress")
done = os.path.join(self.work_dir, "done")
makedirs(in_progress)
makedirs(done)
for dirname, filename in iter_dir(in_progress):
input_name = os.path.join(dirname, filename)
output_name = os.path.join(done, filename)
yield idiokit.send(input_name, output_name)
while True:
paths = itertools.chain(
iter_dir(os.path.join(self.input_dir, "new")),
iter_dir(os.path.join(self.input_dir, "cur")))
for dirname, filename in paths:
uuid_name = uuid.uuid4().hex + "." + filename
input_name = os.path.join(in_progress, uuid_name)
output_name = os.path.join(done, uuid_name)
if try_rename(os.path.join(dirname, filename), input_name):
yield idiokit.send(input_name, output_name)
yield idiokit.sleep(self.poll_interval)
@idiokit.stream
def _forward_files(self):
while True:
input_name, output_name = yield idiokit.next()
ack = idiokit.Event()
node = yield self._queue.queue(0, (input_name, output_name, ack))
try:
yield ack
finally:
yield self._queue.cancel(node)
@idiokit.stream
def main(self, state):
yield self._poll_files() | self._forward_files()
@idiokit.stream
def feed(self, nth_concurrent):
while True:
input_name, output_name, ack = yield self._queue.wait()
ack.succeed()
msg = try_read_message(input_name)
if msg is None:
continue
subject = escape_whitespace(
msg.get_unicode("Subject", "<no subject>", errors="replace"))
sender = escape_whitespace(
msg.get_unicode("From", "<unknown sender>", errors="replace"))
self.log.info(u"Handler #{0} handling mail '{1}' from {2}".format(
nth_concurrent, subject, sender))
handler = self.handler(log=self.log)
yield handler.handle(msg)
os.rename(input_name, output_name)
self.log.info(u"Handler #{0} done with mail '{1}' from {2}".format(
nth_concurrent, subject, sender))
