class FeedEngine(Process): """Feed engine. This object will load and update feeds""" def __init__(self, configuration): Process.__init__(self) self.configuration = configuration self.model = Model(self.configuration) self.feeds = {} self.threads = {} self.global_thread = None # self.messenger = FeedsMessenger(self) def run_feed(self, feed_name): # Check if feed exists in list if not self.feeds.get(feed_name): return False # if feed is not already running if not (self.threads.get(feed_name) and self.threads[feed_name].is_alive()): self.threads[feed_name] = threading.Thread(None, self.feeds[feed_name].run, None) self.threads[feed_name].start() return True def run_all_feeds(self, block=False): debug_output("Running all feeds") for feed_name in [f for f in self.feeds if self.feeds[f].enabled]: debug_output('Starting thread for feed %s...' % feed_name) self.run_feed(feed_name) if block: for t in self.threads: if self.threads[t].is_alive(): self.threads[t].join() def stop_all_feeds(self): self.shutdown = True for t in self.threads: if self.threads[t].is_alive(): self.threads[t]._Thread__stop() def run_scheduled_feeds(self): for f in self.feeds: if self.feeds[f].next_run < datetime.utcnow() and self.feeds[f].enabled: debug_output('Starting thread for feed %s...' % self.feeds[f].name) self.run_feed(self.feeds[f].name) for t in self.threads: if self.threads[t].is_alive(): self.threads[t].join() def run(self): self.messenger = FeedsMessenger(self) self.shutdown = False while not self.shutdown: try: debug_output("FeedEngine heartbeat") if self.scheduler: self.run_scheduled_feeds() time.sleep(self.period) # run a new thread every period seconds except KeyboardInterrupt: self.shutdown = True def load_feeds(self, feed_directories): debug_output("Loading feeds in {}...".format(feed_directories)) for d, subd, files in os.walk(feed_directories): if not d.endswith('core'): for f in files: if f.endswith(".py") and f != "__init__.py": full_filename = os.path.join(d, f) module = imp.load_source(f.split('.')[0], full_filename) for name, obj in module.__dict__.items(): try: if issubclass(obj, Feed) and obj != Feed: feed = module.__dict__.get(name)() feed.model = self.model feed.engine = self feed.tags = list(set(feed.tags + [d])) self.feeds[name] = feed debug_output("Loaded feed {}".format(name)) break except TypeError: pass else: debug_output("Something went wrong parsing {}".format(full_filename), type='error') self.load_feed_status() def load_feed_status(self): feed_status = self.model.get_feed_progress([f for f in self.feeds]) for status in feed_status: name = status['name'] self.feeds[name].last_run = status['last_run'] self.feeds[name].next_run = status['last_run'] + self.feeds[name].run_every