def wake(inner, self): sleeper = timer.sleep(self.throttle_time / 2.0) while True: yield inner, sleeper if sleeper.was_source: yield inner.sub(self.iteration(set(self.pending))) sleeper = timer.sleep(self.throttle_time)
def logger(inner,txt2collab, base_time, interval): sleeper = timer.sleep(interval / 2.0) while True: event = yield inner,sleeper if sleeper.was_source: sleeper = timer.sleep(interval) txt2collab.log() else: print 'adding', event txt2collab.add(event)
def poll(inner, self, something): yield timer.sleep(2) depurl = "http://xtra.tllapt.ee/taru/dep_en.html" arrurl = "http://xtra.tllapt.ee/taru/arr_en.html" for (subtype, url) in [('departures', depurl), ('arrivals', arrurl)]: self.log.info("Downloading %r", url) try: info, fileobj = yield inner.sub(utils.fetch_url(url)) except utils.FetchUrlFailed, fuf: self.log.error("Downloading failed: %r", fuf) return self.log.info("Downloaded") utilization = get_metrics(fileobj) event = events.Event() event.add('source', url) event.add('utilization', utilization) event.add('service', 'airports') event.add('subtype', subtype) event.add('id', create_id(event, 'tallinnairpoirt', 'subtype')) event.add('longitude', '24.799303') event.add('latitude', '59.4165212') yield inner.send(event)
def report(inner, self, item): from_addr, to_addr, subject, msg_str = item # Try to connect to the SMTP server if we're haven't done that yet while self.server is None: list(inner) try: self.log.info("Connecting %r port %d", self.smtp_host, self.smtp_port) self.server = yield inner.thread(self._connect) except: self.log.error("Error connecting SMTP server, "+ "retrying in 10 seconds") yield inner.sub(timer.sleep(10.0)) else: self.log.info("Connected to the SMTP server") list(inner) self.log.info("Sending message %r to %r", subject, to_addr) try: yield inner.thread(self.server.sendmail, from_addr, to_addr, msg_str) except Exception, exc: self.log.info("Could not send message to %r: %s", to_addr, exc) try: yield inner.thread(self.server.quit) except: pass self.server = None inner.finish(False)
def poll(inner, self,something): yield timer.sleep(2) depurl="http://xtra.tllapt.ee/taru/dep_en.html" arrurl="http://xtra.tllapt.ee/taru/arr_en.html" for ( subtype, url ) in [('departures', depurl), ('arrivals',arrurl)]: self.log.info("Downloading %r", url) try: info, fileobj = yield inner.sub(utils.fetch_url(url)) except utils.FetchUrlFailed, fuf: self.log.error("Downloading failed: %r", fuf) return self.log.info("Downloaded") utilization = get_metrics(fileobj) event = events.Event() event.add('source',url) event.add('utilization',utilization) event.add('service','airports') event.add('subtype',subtype) event.add('id', create_id(event,'tallinnairpoirt','subtype')) event.add('longitude','24.799303') event.add('latitude','59.4165212') yield inner.send(event)
def generate(inner,delay=10): #make some random for last week for i in range(0,300): inner.send(create_a_event(60*60*24*7)) #ddos tartu university for 4 days scaling up .. for i in range(0,100): inner.send(create_a_event(60*60*24*4,0,2,1,2,58.36,58.37,26.68,26.69)) for i in range(0,200): inner.send(create_a_event(60*60*24*3,0,2,1,2,58.36,58.37,26.68,26.69)) for i in range(0,300): inner.send(create_a_event(60*60*24*2,0,2,1,2,58.36,58.37,26.68,26.69)) #and add some traffic accidents in Parnu road for i in range(0,100): inner.send(create_a_event(60*60*24*2,1,1,None,None,58.3,59.4,24.6,24.8)) #more big dynamo load, spin faster.. ;) for i in range(0,100): inner.send(create_a_event(60*60*24*4,3,0,0,1, 59.27,59.35,26.75,27.8)) for i in range(0,100): inner.send(create_a_event(60*60*24*3,3,0,0,1, 59.28,59.34,26.75,27.8)) for i in range(0,100): inner.send(create_a_event(60*60*24*2,3,0,0,1, 59.29,59.33,26.75,27.8)) while True: #keep bombing rail between narva and tallinn for last 3 hours ... inner.send(create_a_event(60*60*3,1,2,1,2, 59.29,59.33,24.75,27.8)) yield timer.sleep(delay)
def poll(inner, self, something): yield timer.sleep(5) self.log.info("Downloading %r", self.url) try: info, fileobj = yield inner.sub(utils.fetch_url(self.url)) except utils.FetchUrlFailed, fuf: self.log.error("Downloading failed: %r", fuf) return
def poll(inner, self,something): yield timer.sleep(5) self.log.info("Downloading %r", self.url) try: info, fileobj = yield inner.sub(utils.fetch_url(self.url)) except utils.FetchUrlFailed, fuf: self.log.error("Downloading failed: %r", fuf) return
def throttle(self, throttle_time): """ Ensure that updates for a given event id are sent at least throttle_time apart. """ sleeper = timer.sleep(1.0) ids = dict() queue = list() while True: yield timer.sleep(1.0) item = yield idiokit.next() current_time = time.time() id, event = item previous, _, update_time = ids.get(id, (None, None, None)) if update_time is None: update_time = current_time + 1.0 heapq.heappush(queue, (update_time, id)) ids[id] = previous, event, update_time while queue and queue[0][0] <= current_time: _, id = heapq.heappop(queue) previous, next, _ = ids[id] if previous == next == None: del ids[id] continue if previous != next: if next is None: event = events.Event() else: event = events.Event(next) event.add("id", id) yield idiokit.send(event) if previous != next: update_time = current_time + throttle_time heapq.heappush(queue, (update_time, id)) else: update_time = None ids[id] = next, next, update_time
def poll(inner, self, path): yield timer.sleep(1) self.log.info('Fetching population data from %s.' % path) try: info, fileobj = yield inner.sub(utils.fetch_url(path)) except utils.FetchUrlFailed, fuf: self.log.error("Failed to fetch %s." % path) return
def timed(inner, self, interval): sleeper = timer.sleep(interval) while True: for room, events in self.events.iteritems(): self.write_to_wiki(room, events) self.events[room] = list() item = yield inner, sleeper if sleeper.was_source: inner.finish()
def aggregate(self, group_keys, window_info): """ Create aggregated events and ids for them. """ group_keys = tuple(set(group_keys)) key_groups = dict() while True: yield timer.sleep(1.0) event = yield idiokit.next() current_time = time.time() updated = set() key = tuple(tuple(sorted(event.values(x))) for x in group_keys) updated.add(key) if key not in key_groups: windows = [] for constructor, keys, output_key in window_info: windows.append((constructor(**keys), output_key)) key_groups[key] = windows for window, output_key in key_groups[key]: window.push(current_time, event) for key, windows in list(key_groups.iteritems()): any_expired = False for window, _ in windows: any_expired = window.expire(current_time) or any_expired if not (any_expired or key in updated): continue output = None for window, output_key in windows: value = window.value() if value is None: continue if output is None: output = events.Event() output.add(output_key, unicode(value)) id = hashlib.md5(repr(key)).hexdigest() if output is not None: for group_key, group_values in zip(group_keys, key): output.update(group_key, group_values) yield idiokit.send(id, output) if output is None: del key_groups[key]
def poll(inner,self,user): yield timer.sleep(1) new_tweets = dict() api = twitter.Api(consumer_key=self.consumer_key, consumer_secret=self.consumer_secret, access_token_key=self.access_token_key, access_token_secret=self.access_token_secret) try: userobj = api.GetUser(user) friends = api.GetFriends() except (URLError,twitter.TwitterError,ValueError,JSONDecodeError,socket.error), e: self.log.error("Twitter error: %s." % (e)) return
def alert(inner, *times): while True: if times: sleeper = timer.sleep(min(map(next_time, times))) else: sleeper = threado.Channel() while not sleeper.has_result(): try: yield inner, sleeper except: sleeper.rethrow() raise inner.send()
def poll(inner, self, user): yield timer.sleep(1) new_tweets = dict() api = twitter.Api(consumer_key=self.consumer_key, consumer_secret=self.consumer_secret, access_token_key=self.access_token_key, access_token_secret=self.access_token_secret) try: userobj = api.GetUser(user) friends = api.GetFriends() except (URLError, twitter.TwitterError, ValueError, JSONDecodeError, socket.error), e: self.log.error("Twitter error: %s." % (e)) return
def feed(inner, self): while True: while True: current_time = time.time() if self.feed_queue and self.feed_queue[0][0] <= current_time: break yield inner, timer.sleep(1.0) continue _, feed_key = self.feed_queue.popleft() if not self._feed_keys.contains(feed_key): continue yield inner.sub(self.poll(feed_key)) expire_time = time.time() + self.poll_interval self.feed_queue.append((expire_time, feed_key))
def configs(inner, self): conf_path = os.path.abspath(self.config) last_mtime = None error_msg = None while True: try: mtime = os.path.getmtime(conf_path) if last_mtime != mtime: last_mtime = mtime inner.send(self.SET, config.load_configs(conf_path)) error_msg = None except BaseException, exception: if error_msg != str(exception): error_msg = str(exception) self.log.error("Couldn't load module %r: %s", self.config, error_msg) yield inner, timer.sleep(self.poll_interval)
def generate(inner, delay=10): #make some random for last week for i in range(0, 300): inner.send(create_a_event(60 * 60 * 24 * 7)) #ddos tartu university for 4 days scaling up .. for i in range(0, 100): inner.send( create_a_event(60 * 60 * 24 * 4, 0, 2, 1, 2, 58.36, 58.37, 26.68, 26.69)) for i in range(0, 200): inner.send( create_a_event(60 * 60 * 24 * 3, 0, 2, 1, 2, 58.36, 58.37, 26.68, 26.69)) for i in range(0, 300): inner.send( create_a_event(60 * 60 * 24 * 2, 0, 2, 1, 2, 58.36, 58.37, 26.68, 26.69)) #and add some traffic accidents in Parnu road for i in range(0, 100): inner.send( create_a_event(60 * 60 * 24 * 2, 1, 1, None, None, 58.3, 59.4, 24.6, 24.8)) #more big dynamo load, spin faster.. ;) for i in range(0, 100): inner.send( create_a_event(60 * 60 * 24 * 4, 3, 0, 0, 1, 59.27, 59.35, 26.75, 27.8)) for i in range(0, 100): inner.send( create_a_event(60 * 60 * 24 * 3, 3, 0, 0, 1, 59.28, 59.34, 26.75, 27.8)) for i in range(0, 100): inner.send( create_a_event(60 * 60 * 24 * 2, 3, 0, 0, 1, 59.29, 59.33, 26.75, 27.8)) while True: #keep bombing rail between narva and tallinn for last 3 hours ... inner.send( create_a_event(60 * 60 * 3, 1, 2, 1, 2, 59.29, 59.33, 24.75, 27.8)) yield timer.sleep(delay)
def run_mailbox(inner, self, min_delay=5.0, max_delay=60.0): mailbox = None try: while True: item = yield inner, self.queue if inner.was_source: continue for _ in inner: pass while True: delay = min(min_delay, max_delay) while mailbox is None: try: mailbox = yield inner.sub(thread(self.connect)) except (imaplib.IMAP4.abort, socket.error), error: self.log.error("Failed IMAP connection: %r", error) else: break self.log.info("Retrying connection in %.02f seconds", delay) yield inner, timer.sleep(delay) delay = min(2 * delay, max_delay) channel, name, args, keys = item if channel.has_result(): break try: method = getattr(mailbox, name) result = yield inner.sub(thread(method, *args, **keys)) except (imaplib.IMAP4.abort, socket.error), error: yield inner.sub(thread(self.disconnect, mailbox)) self.log.error("Lost IMAP connection: %r", error) mailbox = None except imaplib.IMAP4.error: channel.rethrow() break else: channel.finish(result) break
def run(self, interval=1.0): try: while True: yield self.inner.sub(timer.sleep(interval)) list(self.inner) if self.keeptime is not None: cutoff = int(time.time() - self.keeptime) max_id = self.cursor.execute("SELECT MAX(events.id) FROM events "+ "WHERE events.timestamp <= ?", (cutoff,)) max_id = list(max_id)[0][0] if max_id is not None: self.cursor.execute("DELETE FROM events WHERE events.id <= ?", (max_id,)) self.cursor.execute("DELETE FROM attrs WHERE attrs.eventid <= ?", (max_id,)) self.conn.commit() self.cursor = self.conn.cursor() finally: self.conn.commit() self.conn.close()
def noop(inner, self, noop_interval=10.0): while True: yield inner.sub(self.call("noop")) yield inner, timer.sleep(noop_interval)
def commit(self, db, commit_interval=1.0): while True: yield timer.sleep(commit_interval) db.commit()
def _timeout(self, timeout): yield timer.sleep(timeout) raise Timeout()
def poll(inner, self): while True: yield inner.sub(self.fetch_mails(self.filter)) yield inner, timer.sleep(self.poll_interval)