class FeedEngine(Process):
"""Feed engine. This object will load and update feeds"""
def __init__(self, configuration):
Process.__init__(self)
self.configuration = configuration
self.model = Model(self.configuration)
self.feeds = {}
self.threads = {}
self.global_thread = None
# self.messenger = FeedsMessenger(self)
def run_feed(self, feed_name):
# Check if feed exists in list
if not self.feeds.get(feed_name):
return False
# if feed is not already running
if not (self.threads.get(feed_name) and self.threads[feed_name].is_alive()):
self.threads[feed_name] = threading.Thread(None, self.feeds[feed_name].run, None)
self.threads[feed_name].start()
return True
def run_all_feeds(self, block=False):
debug_output("Running all feeds")
for feed_name in [f for f in self.feeds if self.feeds[f].enabled]:
debug_output('Starting thread for feed %s...' % feed_name)
self.run_feed(feed_name)
if block:
for t in self.threads:
if self.threads[t].is_alive():
self.threads[t].join()
def stop_all_feeds(self):
self.shutdown = True
for t in self.threads:
if self.threads[t].is_alive():
self.threads[t]._Thread__stop()
def run_scheduled_feeds(self):
for f in self.feeds:
if self.feeds[f].next_run < datetime.utcnow() and self.feeds[f].enabled:
debug_output('Starting thread for feed %s...' % self.feeds[f].name)
self.run_feed(self.feeds[f].name)
for t in self.threads:
if self.threads[t].is_alive():
self.threads[t].join()
def run(self):
self.messenger = FeedsMessenger(self)
self.shutdown = False
while not self.shutdown:
try:
debug_output("FeedEngine heartbeat")
if self.scheduler:
self.run_scheduled_feeds()
time.sleep(self.period) # run a new thread every period seconds
except KeyboardInterrupt:
self.shutdown = True
def load_feeds(self, feed_directories):
debug_output("Loading feeds in {}...".format(feed_directories))
for d, subd, files in os.walk(feed_directories):
if not d.endswith('core'):
for f in files:
if f.endswith(".py") and f != "__init__.py":
full_filename = os.path.join(d, f)
module = imp.load_source(f.split('.')[0], full_filename)
for name, obj in module.__dict__.items():
try:
if issubclass(obj, Feed) and obj != Feed:
feed = module.__dict__.get(name)()
feed.model = self.model
feed.engine = self
feed.tags = list(set(feed.tags + [d]))
self.feeds[name] = feed
debug_output("Loaded feed {}".format(name))
break
except TypeError:
pass
else:
debug_output("Something went wrong parsing {}".format(full_filename), type='error')
self.load_feed_status()
def load_feed_status(self):
feed_status = self.model.get_feed_progress([f for f in self.feeds])
for status in feed_status:
name = status['name']
self.feeds[name].last_run = status['last_run']
self.feeds[name].next_run = status['last_run'] + self.feeds[name].run_every
