def wake(inner, self):
sleeper = timer.sleep(self.throttle_time / 2.0)
while True:
yield inner, sleeper
if sleeper.was_source:
yield inner.sub(self.iteration(set(self.pending)))
sleeper = timer.sleep(self.throttle_time)
def logger(inner,txt2collab, base_time, interval):
sleeper = timer.sleep(interval / 2.0)
while True:
event = yield inner,sleeper
if sleeper.was_source:
sleeper = timer.sleep(interval)
txt2collab.log()
else:
print 'adding', event
txt2collab.add(event)
def poll(inner, self, something):
yield timer.sleep(2)
depurl = "http://xtra.tllapt.ee/taru/dep_en.html"
arrurl = "http://xtra.tllapt.ee/taru/arr_en.html"
for (subtype, url) in [('departures', depurl), ('arrivals', arrurl)]:
self.log.info("Downloading %r", url)
try:
info, fileobj = yield inner.sub(utils.fetch_url(url))
except utils.FetchUrlFailed, fuf:
self.log.error("Downloading failed: %r", fuf)
return
self.log.info("Downloaded")
utilization = get_metrics(fileobj)
event = events.Event()
event.add('source', url)
event.add('utilization', utilization)
event.add('service', 'airports')
event.add('subtype', subtype)
event.add('id', create_id(event, 'tallinnairpoirt', 'subtype'))
event.add('longitude', '24.799303')
event.add('latitude', '59.4165212')
yield inner.send(event)
def report(inner, self, item):
from_addr, to_addr, subject, msg_str = item
# Try to connect to the SMTP server if we're haven't done that yet
while self.server is None:
list(inner)
try:
self.log.info("Connecting %r port %d",
self.smtp_host, self.smtp_port)
self.server = yield inner.thread(self._connect)
except:
self.log.error("Error connecting SMTP server, "+
"retrying in 10 seconds")
yield inner.sub(timer.sleep(10.0))
else:
self.log.info("Connected to the SMTP server")
list(inner)
self.log.info("Sending message %r to %r", subject, to_addr)
try:
yield inner.thread(self.server.sendmail,
from_addr, to_addr, msg_str)
except Exception, exc:
self.log.info("Could not send message to %r: %s", to_addr, exc)
try:
yield inner.thread(self.server.quit)
except:
pass
self.server = None
inner.finish(False)
def poll(inner, self,something):
yield timer.sleep(2)
depurl="http://xtra.tllapt.ee/taru/dep_en.html"
arrurl="http://xtra.tllapt.ee/taru/arr_en.html"
for ( subtype, url ) in [('departures', depurl), ('arrivals',arrurl)]:
self.log.info("Downloading %r", url)
try:
info, fileobj = yield inner.sub(utils.fetch_url(url))
except utils.FetchUrlFailed, fuf:
self.log.error("Downloading failed: %r", fuf)
return
self.log.info("Downloaded")
utilization = get_metrics(fileobj)
event = events.Event()
event.add('source',url)
event.add('utilization',utilization)
event.add('service','airports')
event.add('subtype',subtype)
event.add('id', create_id(event,'tallinnairpoirt','subtype'))
event.add('longitude','24.799303')
event.add('latitude','59.4165212')
yield inner.send(event)
def generate(inner,delay=10):
#make some random for last week
for i in range(0,300):
inner.send(create_a_event(60*60*24*7))
#ddos tartu university for 4 days scaling up ..
for i in range(0,100):
inner.send(create_a_event(60*60*24*4,0,2,1,2,58.36,58.37,26.68,26.69))
for i in range(0,200):
inner.send(create_a_event(60*60*24*3,0,2,1,2,58.36,58.37,26.68,26.69))
for i in range(0,300):
inner.send(create_a_event(60*60*24*2,0,2,1,2,58.36,58.37,26.68,26.69))
#and add some traffic accidents in Parnu road
for i in range(0,100):
inner.send(create_a_event(60*60*24*2,1,1,None,None,58.3,59.4,24.6,24.8))
#more big dynamo load, spin faster.. ;)
for i in range(0,100):
inner.send(create_a_event(60*60*24*4,3,0,0,1, 59.27,59.35,26.75,27.8))
for i in range(0,100):
inner.send(create_a_event(60*60*24*3,3,0,0,1, 59.28,59.34,26.75,27.8))
for i in range(0,100):
inner.send(create_a_event(60*60*24*2,3,0,0,1, 59.29,59.33,26.75,27.8))
while True:
#keep bombing rail between narva and tallinn for last 3 hours ...
inner.send(create_a_event(60*60*3,1,2,1,2, 59.29,59.33,24.75,27.8))
yield timer.sleep(delay)
def poll(inner, self, something):
yield timer.sleep(5)
self.log.info("Downloading %r", self.url)
try:
info, fileobj = yield inner.sub(utils.fetch_url(self.url))
except utils.FetchUrlFailed, fuf:
self.log.error("Downloading failed: %r", fuf)
return
def poll(inner, self,something):
yield timer.sleep(5)
self.log.info("Downloading %r", self.url)
try:
info, fileobj = yield inner.sub(utils.fetch_url(self.url))
except utils.FetchUrlFailed, fuf:
self.log.error("Downloading failed: %r", fuf)
return
def throttle(self, throttle_time):
"""
Ensure that updates for a given event id are sent at least
throttle_time apart.
"""
sleeper = timer.sleep(1.0)
ids = dict()
queue = list()
while True:
yield timer.sleep(1.0)
item = yield idiokit.next()
current_time = time.time()
id, event = item
previous, _, update_time = ids.get(id, (None, None, None))
if update_time is None:
update_time = current_time + 1.0
heapq.heappush(queue, (update_time, id))
ids[id] = previous, event, update_time
while queue and queue[0][0] <= current_time:
_, id = heapq.heappop(queue)
previous, next, _ = ids[id]
if previous == next == None:
del ids[id]
continue
if previous != next:
if next is None:
event = events.Event()
else:
event = events.Event(next)
event.add("id", id)
yield idiokit.send(event)
if previous != next:
update_time = current_time + throttle_time
heapq.heappush(queue, (update_time, id))
else:
update_time = None
ids[id] = next, next, update_time
def poll(inner, self, path):
yield timer.sleep(1)
self.log.info('Fetching population data from %s.' % path)
try:
info, fileobj = yield inner.sub(utils.fetch_url(path))
except utils.FetchUrlFailed, fuf:
self.log.error("Failed to fetch %s." % path)
return
def poll(inner, self, path):
yield timer.sleep(1)
self.log.info('Fetching population data from %s.' % path)
try:
info, fileobj = yield inner.sub(utils.fetch_url(path))
except utils.FetchUrlFailed, fuf:
self.log.error("Failed to fetch %s." % path)
return
def timed(inner, self, interval):
sleeper = timer.sleep(interval)
while True:
for room, events in self.events.iteritems():
self.write_to_wiki(room, events)
self.events[room] = list()
item = yield inner, sleeper
if sleeper.was_source:
inner.finish()
def aggregate(self, group_keys, window_info):
"""
Create aggregated events and ids for them.
"""
group_keys = tuple(set(group_keys))
key_groups = dict()
while True:
yield timer.sleep(1.0)
event = yield idiokit.next()
current_time = time.time()
updated = set()
key = tuple(tuple(sorted(event.values(x))) for x in group_keys)
updated.add(key)
if key not in key_groups:
windows = []
for constructor, keys, output_key in window_info:
windows.append((constructor(**keys), output_key))
key_groups[key] = windows
for window, output_key in key_groups[key]:
window.push(current_time, event)
for key, windows in list(key_groups.iteritems()):
any_expired = False
for window, _ in windows:
any_expired = window.expire(current_time) or any_expired
if not (any_expired or key in updated):
continue
output = None
for window, output_key in windows:
value = window.value()
if value is None:
continue
if output is None:
output = events.Event()
output.add(output_key, unicode(value))
id = hashlib.md5(repr(key)).hexdigest()
if output is not None:
for group_key, group_values in zip(group_keys, key):
output.update(group_key, group_values)
yield idiokit.send(id, output)
if output is None:
del key_groups[key]
def poll(inner,self,user):
yield timer.sleep(1)
new_tweets = dict()
api = twitter.Api(consumer_key=self.consumer_key,
consumer_secret=self.consumer_secret,
access_token_key=self.access_token_key,
access_token_secret=self.access_token_secret)
try:
userobj = api.GetUser(user)
friends = api.GetFriends()
except (URLError,twitter.TwitterError,ValueError,JSONDecodeError,socket.error), e:
self.log.error("Twitter error: %s." % (e))
return
def alert(inner, *times):
while True:
if times:
sleeper = timer.sleep(min(map(next_time, times)))
else:
sleeper = threado.Channel()
while not sleeper.has_result():
try:
yield inner, sleeper
except:
sleeper.rethrow()
raise
inner.send()
def poll(inner, self, user):
yield timer.sleep(1)
new_tweets = dict()
api = twitter.Api(consumer_key=self.consumer_key,
consumer_secret=self.consumer_secret,
access_token_key=self.access_token_key,
access_token_secret=self.access_token_secret)
try:
userobj = api.GetUser(user)
friends = api.GetFriends()
except (URLError, twitter.TwitterError, ValueError, JSONDecodeError,
socket.error), e:
self.log.error("Twitter error: %s." % (e))
return
def feed(inner, self):
while True:
while True:
current_time = time.time()
if self.feed_queue and self.feed_queue[0][0] <= current_time:
break
yield inner, timer.sleep(1.0)
continue
_, feed_key = self.feed_queue.popleft()
if not self._feed_keys.contains(feed_key):
continue
yield inner.sub(self.poll(feed_key))
expire_time = time.time() + self.poll_interval
self.feed_queue.append((expire_time, feed_key))
def configs(inner, self):
conf_path = os.path.abspath(self.config)
last_mtime = None
error_msg = None
while True:
try:
mtime = os.path.getmtime(conf_path)
if last_mtime != mtime:
last_mtime = mtime
inner.send(self.SET, config.load_configs(conf_path))
error_msg = None
except BaseException, exception:
if error_msg != str(exception):
error_msg = str(exception)
self.log.error("Couldn't load module %r: %s",
self.config, error_msg)
yield inner, timer.sleep(self.poll_interval)
def generate(inner, delay=10):
#make some random for last week
for i in range(0, 300):
inner.send(create_a_event(60 * 60 * 24 * 7))
#ddos tartu university for 4 days scaling up ..
for i in range(0, 100):
inner.send(
create_a_event(60 * 60 * 24 * 4, 0, 2, 1, 2, 58.36, 58.37, 26.68,
26.69))
for i in range(0, 200):
inner.send(
create_a_event(60 * 60 * 24 * 3, 0, 2, 1, 2, 58.36, 58.37, 26.68,
26.69))
for i in range(0, 300):
inner.send(
create_a_event(60 * 60 * 24 * 2, 0, 2, 1, 2, 58.36, 58.37, 26.68,
26.69))
#and add some traffic accidents in Parnu road
for i in range(0, 100):
inner.send(
create_a_event(60 * 60 * 24 * 2, 1, 1, None, None, 58.3, 59.4,
24.6, 24.8))
#more big dynamo load, spin faster.. ;)
for i in range(0, 100):
inner.send(
create_a_event(60 * 60 * 24 * 4, 3, 0, 0, 1, 59.27, 59.35, 26.75,
27.8))
for i in range(0, 100):
inner.send(
create_a_event(60 * 60 * 24 * 3, 3, 0, 0, 1, 59.28, 59.34, 26.75,
27.8))
for i in range(0, 100):
inner.send(
create_a_event(60 * 60 * 24 * 2, 3, 0, 0, 1, 59.29, 59.33, 26.75,
27.8))
while True:
#keep bombing rail between narva and tallinn for last 3 hours ...
inner.send(
create_a_event(60 * 60 * 3, 1, 2, 1, 2, 59.29, 59.33, 24.75, 27.8))
yield timer.sleep(delay)
def run_mailbox(inner, self, min_delay=5.0, max_delay=60.0):
mailbox = None
try:
while True:
item = yield inner, self.queue
if inner.was_source:
continue
for _ in inner:
pass
while True:
delay = min(min_delay, max_delay)
while mailbox is None:
try:
mailbox = yield inner.sub(thread(self.connect))
except (imaplib.IMAP4.abort, socket.error), error:
self.log.error("Failed IMAP connection: %r", error)
else:
break
self.log.info("Retrying connection in %.02f seconds", delay)
yield inner, timer.sleep(delay)
delay = min(2 * delay, max_delay)
channel, name, args, keys = item
if channel.has_result():
break
try:
method = getattr(mailbox, name)
result = yield inner.sub(thread(method, *args, **keys))
except (imaplib.IMAP4.abort, socket.error), error:
yield inner.sub(thread(self.disconnect, mailbox))
self.log.error("Lost IMAP connection: %r", error)
mailbox = None
except imaplib.IMAP4.error:
channel.rethrow()
break
else:
channel.finish(result)
break
def run(self, interval=1.0):
try:
while True:
yield self.inner.sub(timer.sleep(interval))
list(self.inner)
if self.keeptime is not None:
cutoff = int(time.time() - self.keeptime)
max_id = self.cursor.execute("SELECT MAX(events.id) FROM events "+
"WHERE events.timestamp <= ?", (cutoff,))
max_id = list(max_id)[0][0]
if max_id is not None:
self.cursor.execute("DELETE FROM events WHERE events.id <= ?",
(max_id,))
self.cursor.execute("DELETE FROM attrs WHERE attrs.eventid <= ?",
(max_id,))
self.conn.commit()
self.cursor = self.conn.cursor()
finally:
self.conn.commit()
self.conn.close()
def noop(inner, self, noop_interval=10.0):
while True:
yield inner.sub(self.call("noop"))
yield inner, timer.sleep(noop_interval)
def commit(self, db, commit_interval=1.0):
while True:
yield timer.sleep(commit_interval)
db.commit()
def _timeout(self, timeout):
yield timer.sleep(timeout)
raise Timeout()
def _timeout(self, timeout):
yield timer.sleep(timeout)
raise Timeout()
def commit(self, db, commit_interval=1.0):
while True:
yield timer.sleep(commit_interval)
db.commit()
def poll(inner, self):
while True:
yield inner.sub(self.fetch_mails(self.filter))
yield inner, timer.sleep(self.poll_interval)
