def execute_task(self, *args):
"""
Take a feed's url from the cached list and fetch posts
:param args: no arguaments at the moment
:return: None
"""
from flankers.scrawler import Scrawler
RSS_FEEDS_CACHE = memcache.get('RSS_FEEDS_CACHE')
if not RSS_FEEDS_CACHE or len(RSS_FEEDS_CACHE) == 0:
RSS_FEEDS_CACHE = Scrawler.load_links()
memcache.set('RSS_FEEDS_CACHE', RSS_FEEDS_CACHE)
print len(RSS_FEEDS_CACHE)
l = RSS_FEEDS_CACHE.pop()
print l
entries = Scrawler.read_feed(l)
if entries:
for entry in entries:
#
# Store feed
#
store_feed(entry)
memcache.set('RSS_FEEDS_CACHE', RSS_FEEDS_CACHE)
return None
memcache.set('RSS_FEEDS_CACHE', RSS_FEEDS_CACHE)
print "This Feed has no entries"
return None
def execute_task(self, *args):
from flankers.scrawler import Scrawler
RSS_FEEDS_CACHE = memcache.get('RSS_FEEDS_CACHE')
if not RSS_FEEDS_CACHE or len(RSS_FEEDS_CACHE) == 0:
RSS_FEEDS_CACHE = Scrawler.load_links()
memcache.set('RSS_FEEDS_CACHE', RSS_FEEDS_CACHE)
print len(RSS_FEEDS_CACHE)
l = RSS_FEEDS_CACHE.pop()
print l
entries = Scrawler.read_feed(l)
if entries:
for entry in entries:
#
# Store feed
#
store_feed(entry)
memcache.set('RSS_FEEDS_CACHE', RSS_FEEDS_CACHE)
return None
memcache.set('RSS_FEEDS_CACHE', RSS_FEEDS_CACHE)
print "This Feed has no entries"
return None
def execute_task(self, *args):
from flankers.scrawler import Scrawler
RSS_FEEDS_CACHE = memcache.get('RSS_FEEDS_CACHE')
if not RSS_FEEDS_CACHE or len(RSS_FEEDS_CACHE) == 0:
RSS_FEEDS_CACHE = Scrawler.load_links()
memcache.set('RSS_FEEDS_CACHE', RSS_FEEDS_CACHE)
print len(RSS_FEEDS_CACHE)
l = RSS_FEEDS_CACHE.pop()
print l
entries = Scrawler.read_feed(l)
if entries:
for entry in entries:
#
# Store feed
#
store_feed(entry)
memcache.set('RSS_FEEDS_CACHE', RSS_FEEDS_CACHE)
return None
memcache.set('RSS_FEEDS_CACHE', RSS_FEEDS_CACHE)
print "This Feed has no entries"
return None
def run_test(self):
"""
Prints Feed object and multiple attributes if the Item() mock object
"""
s = Scrawler()
items = s.load_links()
print items[0]
for i in xrange(0, 1):
try:
item = s.read_feed(items[i])
except ValueError:
continue
print item # this is a list of feeds
item = self.mock_Item(
str(item["title"]),
str(item["link"]),
str(item["published_parsed"]),
str(item["summary"].encode("utf-8")),
)
for k, v in item.store().items():
print k + ": ", v
print "\n"
